コード例 #1
0
ファイル: file_set.py プロジェクト: kod3r/SMQTK
 def _iter_file_tree(self):
     """
     Iterate over our file tree, yielding the file paths of serialized
         elements found.
     """
     for fp in file_utils.iter_directory_files(self._root_dir):
         m = self.SERIAL_FILE_RE.match(osp.basename(fp))
         if m:
             # if the path doesn't have the configured split chunking value,
             # it doesn't belong to this data set
             seg = osp.dirname(osp.relpath(fp, self._root_dir)).split(os.sep)
             if len(seg) == self._uuid_chunk:
                 yield fp
コード例 #2
0
ファイル: file_set.py プロジェクト: thezedwards/SMQTK
 def _iter_file_tree(self):
     """
     Iterate over our file tree, yielding the file paths of serialized
         elements found.
     """
     for fp in file_utils.iter_directory_files(self._root_dir):
         m = self.SERIAL_FILE_RE.match(osp.basename(fp))
         if m:
             # if the path doesn't have the configured split chunking value,
             # it doesn't belong to this data set
             seg = osp.dirname(osp.relpath(fp, self._root_dir)).split(os.sep)
             if not self._uuid_chunk or len(seg) == self._uuid_chunk:
                 yield fp
コード例 #3
0
ファイル: file_set.py プロジェクト: gitter-badger/SMQTK
 def _discover_data_elements(self):
     """
     From the set root directory, find serialized files, deserialize them and
     store in instance mapping.
     """
     if os.path.isdir(self._root_dir):
         self._log.debug("Root directory exists, finding existing data "
                         "elements...")
         with self._element_map_lock:
             for fpath in iter_directory_files(self._root_dir, True):
                 m = self.SERIAL_FILE_RE.match(os.path.basename(fpath))
                 if m:
                     with open(fpath) as f:
                         #: :type: smqtk.data_rep.DataElement
                         de = cPickle.load(f)
                     self._element_map[de.uuid()] = de
             self._log.debug("Found %d elements", len(self._element_map))
     else:
         self._log.debug("Root dir doesn't exist, can't have existing "
                         "elements")
コード例 #4
0
    def _iter_file_tree(self):
        """
        Iterate over our file tree, yielding the file paths of serialized
        elements found in the expected sub-directories.
        """
        # Select how far we need to descent into root based on chunk level.
        recurse = (self._uuid_chunk and self._uuid_chunk - 1) or None

        for fp in file_utils.iter_directory_files(self._root_dir, recurse):
            m = self.SERIAL_FILE_RE.match(osp.basename(fp))
            if m:
                # Where file is under root to see if it is a file we care about
                # according to our ``uuid_chunk`` value.
                # - ``seg`` includes file name, so it will at least be of length
                #   1, so its unmodified length should equal ``uuid_chunk``.
                # - Exceptions: len(seg) == 1 and uuid_chunk in {None, 0, 1}
                seg = osp.relpath(fp, self._root_dir).split(os.sep)
                if self._uuid_chunk in {None, 1} and len(seg) == 1:
                    yield fp
                elif len(seg) == self._uuid_chunk:
                    yield fp
コード例 #5
0
ファイル: file_set.py プロジェクト: Kitware/SMQTK
    def _iter_file_tree(self):
        """
        Iterate over our file tree, yielding the file paths of serialized
        elements found in the expected sub-directories.
        """
        # Select how far we need to descent into root based on chunk level.
        recurse = (self._uuid_chunk and self._uuid_chunk - 1) or None

        for fp in file_utils.iter_directory_files(self._root_dir, recurse):
            m = self.SERIAL_FILE_RE.match(osp.basename(fp))
            if m:
                # Where file is under root to see if it is a file we care about
                # according to our ``uuid_chunk`` value.
                # - ``seg`` includes file name, so it will at least be of length
                #   1, so its unmodified length should equal ``uuid_chunk``.
                # - Exceptions: len(seg) == 1 and uuid_chunk in {None, 0, 1}
                seg = osp.relpath(fp, self._root_dir).split(os.sep)
                if self._uuid_chunk in {None, 1} and len(seg) == 1:
                    yield fp
                elif len(seg) == self._uuid_chunk:
                    yield fp