def _iter_file_tree(self):
"""
Iterate over our file tree, yielding the file paths of serialized
elements found.
"""
for fp in file_utils.iter_directory_files(self._root_dir):
m = self.SERIAL_FILE_RE.match(osp.basename(fp))
if m:
# if the path doesn't have the configured split chunking value,
# it doesn't belong to this data set
seg = osp.dirname(osp.relpath(fp, self._root_dir)).split(os.sep)
if len(seg) == self._uuid_chunk:
yield fp
def _iter_file_tree(self):
"""
Iterate over our file tree, yielding the file paths of serialized
elements found.
"""
for fp in file_utils.iter_directory_files(self._root_dir):
m = self.SERIAL_FILE_RE.match(osp.basename(fp))
if m:
# if the path doesn't have the configured split chunking value,
# it doesn't belong to this data set
seg = osp.dirname(osp.relpath(fp, self._root_dir)).split(os.sep)
if not self._uuid_chunk or len(seg) == self._uuid_chunk:
yield fp
def _discover_data_elements(self):
"""
From the set root directory, find serialized files, deserialize them and
store in instance mapping.
"""
if os.path.isdir(self._root_dir):
self._log.debug("Root directory exists, finding existing data "
"elements...")
with self._element_map_lock:
for fpath in iter_directory_files(self._root_dir, True):
m = self.SERIAL_FILE_RE.match(os.path.basename(fpath))
if m:
with open(fpath) as f:
#: :type: smqtk.data_rep.DataElement
de = cPickle.load(f)
self._element_map[de.uuid()] = de
self._log.debug("Found %d elements", len(self._element_map))
else:
self._log.debug("Root dir doesn't exist, can't have existing "
"elements")
def _iter_file_tree(self):
"""
Iterate over our file tree, yielding the file paths of serialized
elements found in the expected sub-directories.
"""
# Select how far we need to descent into root based on chunk level.
recurse = (self._uuid_chunk and self._uuid_chunk - 1) or None
for fp in file_utils.iter_directory_files(self._root_dir, recurse):
m = self.SERIAL_FILE_RE.match(osp.basename(fp))
if m:
# Where file is under root to see if it is a file we care about
# according to our ``uuid_chunk`` value.
# - ``seg`` includes file name, so it will at least be of length
# 1, so its unmodified length should equal ``uuid_chunk``.
# - Exceptions: len(seg) == 1 and uuid_chunk in {None, 0, 1}
seg = osp.relpath(fp, self._root_dir).split(os.sep)
if self._uuid_chunk in {None, 1} and len(seg) == 1:
yield fp
elif len(seg) == self._uuid_chunk:
yield fp
def _iter_file_tree(self):
"""
Iterate over our file tree, yielding the file paths of serialized
elements found in the expected sub-directories.
"""
# Select how far we need to descent into root based on chunk level.
recurse = (self._uuid_chunk and self._uuid_chunk - 1) or None
for fp in file_utils.iter_directory_files(self._root_dir, recurse):
m = self.SERIAL_FILE_RE.match(osp.basename(fp))
if m:
# Where file is under root to see if it is a file we care about
# according to our ``uuid_chunk`` value.
# - ``seg`` includes file name, so it will at least be of length
# 1, so its unmodified length should equal ``uuid_chunk``.
# - Exceptions: len(seg) == 1 and uuid_chunk in {None, 0, 1}
seg = osp.relpath(fp, self._root_dir).split(os.sep)
if self._uuid_chunk in {None, 1} and len(seg) == 1:
yield fp
elif len(seg) == self._uuid_chunk:
yield fp