def _is_restaurant_module(entry: DirEntry) -> bool: """Checks if entry is a Python file, ignores if name starts with underscore.""" return all([ entry.is_file(), entry.name.endswith('.py'), # type: ignore not entry.name.startswith('_') # type: ignore ])
def process_one_file(parents: Sequence[str], item: os.DirEntry) -> None: """Process a JSON file during the main walk.""" global _sys_event_tables def is_leaf_dir(path: str) -> bool: for item in os.scandir(path): if item.is_dir(): return False return True # model directory, reset topic if item.is_dir() and is_leaf_dir(item.path): print_events_table_suffix() tblname = file_name_to_table_name(parents, item.name) if item.name == 'sys': _sys_event_tables.append(tblname) print_events_table_prefix(tblname) return # base dir or too deep level = len(parents) if level == 0 or level > 4: return # Ignore other directories. If the file name does not have a .json # extension, ignore it. It could be a readme.txt for instance. if not item.is_file() or not item.name.endswith('.json'): return add_events_table_entries(item, get_topic(item.name))
def filter_alebmic_ini(entry: os.DirEntry) -> bool: if not entry.is_file(): return False if entry.name != "alembic.ini": return False return True
def _component_filter(self, entry: os.DirEntry): if not entry.is_file(): return False base, ext = os.path.splitext(entry.name) if not self._component_mask.fullmatch(base): return False return True
def filter_only_tif_files(item: os.DirEntry) -> bool: if not item.is_file(): return False _, ext = os.path.splitext(item.name) if ext.lower() != ".tif": return False return True
def _process_file_or_dir(context: str, dir_entry: os.DirEntry) -> [[AssetFile]]: if dir_entry.is_file(): return AssetFile.build(dir_entry=dir_entry, context=context) else: new_context = os.path.join(context, dir_entry.name) return [ _process_file_or_dir(new_context, f) for f in os.scandir(dir_entry.path) if f.name[:1] != "." and f.name[:1] != "_" ]
def button_made_from(entry: os.DirEntry) -> Gtk.Button: if entry.is_file(): return Copy(Snippet().load(entry.path)) if entry.is_dir(): return GoTo( name=entry.name, position=os.path.dirname(entry.path), destination=entry.path, )
def _get_entry_attributes(entry: os.DirEntry): attrs = FileAttributes.NONE if entry.is_dir(): attrs |= FileAttributes.IS_DIR elif entry.is_file(): attrs |= FileAttributes.IS_FILE if entry.is_symlink(): attrs |= FileAttributes.IS_LINK if entry.name.startswith('.'): attrs |= FileAttributes.IS_HIDDEN return attrs
def filter_only_trainingdata(item: os.DirEntry) -> bool: if not item.is_file(): return False base, ext = os.path.splitext(item.name) if ext != ".traineddata": return False if base == "osd": return False return True
def filter_ocr_only(entry: os.DirEntry): if not entry.is_file(): return False name, ext = os.path.splitext(entry.name) if ext.lower() != ".xml": return False if name.lower() == "marc": return False return True
def validate_file(self, dir_entry: os.DirEntry) -> bool: """Validates given DirEntry. Returns False if entry should be completely ignored, or True if we want to keep it for further processing. Ignore all zero length files. There are usually there for a purpose like .dummy etc, so there can be tons of it with the same name even, so by default, ignore them completely. Also ignore all symlinks.""" from .log import Log if dir_entry.is_symlink(): Log.vv('{name}: It is the symbolic link. Skipping.'.format( name=dir_entry.name)) return False # NOTE: do not call is_file() on DirEntry. It will fail in endless # recursion for invalid (dead) symbolic links. os.path.isfile() works). if not dir_entry.is_file(): Log.vv('{name}: This is not a file. Skipping.'.format( name=dir_entry.name)) return False item_size = dir_entry.stat().st_size if item_size == 0: Log.vv('{name}: File is 0 bytes long. Skipping.'.format( name=dir_entry.name)) return False if self.min_size > 0 and item_size < self.min_size: Log.vv('{name}: File is shorter than min size ({size}). Skipping.'. format(name=dir_entry.name, size=item_size)) return False if 0 < self.max_size < item_size: Log.vv('{name}: File is biger than max size ({size}). Skipping.'. format(name=dir_entry.name, size=item_size)) return False for list_item in self._file_name_blacklist: match = re.match(list_item, dir_entry.name) if match is not None: Log.vv('File "{name}" blacklisted by "{re}" rule. Skipping.'. format(name=dir_entry.name, re=list_item)) return False return True
def preprocess_one_file(parents: Sequence[str], item: os.DirEntry) -> None: if item.is_dir(): return # base dir or too deep level = len(parents) if level == 0 or level > 4: return # Ignore other directories. If the file name does not have a .json # extension, ignore it. It could be a readme.txt for instance. if not item.is_file() or not item.name.endswith('.json'): return topic = get_topic(item.name) for event in read_json_events(item.path, topic): _bcs.add(event.build_c_string())
def is_info_file(file: DirEntry) -> bool: """Checks if a file is an info file.""" return is_info(file.name) and file.is_file(follow_symlinks=False)
def is_shard_file(file: DirEntry) -> bool: """Checks if a file is a shard.""" return is_shard(file.name) and file.is_file(follow_symlinks=False)
def is_python_file(item: os.DirEntry): return item.is_file() and item.name.endswith('.py')
def _file_can_be_imported(entry: os.DirEntry) -> bool: """ Verify if a directory entry is a non-magic Python file.""" return entry.is_file( ) and not entry.name.startswith("__") and entry.name.endswith(".py")
def is_file(self, entry: DirEntry) -> bool: return entry.is_file(follow_symlinks=self.__follow_symlinks)
def _from_os_dir_entry(original: os.DirEntry) -> fs.DirEntry: return fs.DirEntry( url=urls.URL("file://" + os.path.abspath(original.path)), is_dir=bool(original.is_dir()), is_file=bool(original.is_file()), )
def auto(cls, path: os.DirEntry, asset_data_path, ignore_info=False ): # type: (os.DirEntry, str, bool) -> typing.Tuple[str, Asset] info = {} preview = None id = os.path.splitext(path.name)[0] if os.path.dirname(path.path) != asset_data_path: if id: number = 2 id_path = os.path.join(asset_data_path, id) while True: if os.path.exists(id_path): id_path = os.path.join(asset_data_path, id + f"_{number}") number += 1 else: break else: id_chars = "".join((string.ascii_lowercase, string.digits)) while True: id = ''.join(random.choice(id_chars) for _ in range(11)) id_path = os.path.join(asset_data_path, id) if not os.path.exists(id_path): break id_path = utils.PseudoDirEntry(id_path) else: id_path = path extra_folder = os.path.join(id_path.path, "__extra__") archive_folder = os.path.join(id_path.path, "__archive__") gallery_folder = os.path.join(id_path.path, "__gallery__") def get_info(): is_ok, result = asset_parser.get_web(url, id_path.path) if is_ok: preview = result.pop("preview_path", None) return result, preview return None, None if path.is_file(): file = utils.pathlib.Path(path) url = None auto_folder = file.parent url_files = [ utils.pathlib.Path(auto_folder, file.stem + extension) for extension in utils.URL_EXTENSIONS ] url_files = [ url for url in url_files if url.exists() and url.type == "url" ] for url_file in url_files: url = url_file.data utils.move_to_folder(url_file, extra_folder) if url: info, preview = get_info() if file.type == "zip": utils.extract_zip(file, id_path.path) utils.move_to_folder(file, archive_folder) else: utils.move_to_folder(file, id_path.path) else: id_path = utils.PseudoDirEntry( utils.move_to_folder(path.path, asset_data_path)) files = utils.File_Filter.from_dir(id_path, ignore=("__extra__", "__archive__")) old_info = None for existing_info in files.get_by_type("__info__"): if ignore_info: old_info = existing_info.data break else: return id, cls.default(id_path) zips = files.get_by_type("zip") if zips: for zip in zips: utils.extract_zip(str(zip), path=id_path.path) utils.move_to_folder(zip, archive_folder) files.update() if not info: for url_file in files.get_by_type("url"): url = url_file.data info, preview = get_info() if info: break if not info: for megascan_info in files.get_by_type("megascan_info"): megascan_id = megascan_info.data.get('id') if not megascan_id: continue url = f"https://quixel.com/megascans/home?assetId={megascan_id}" info, preview = get_info() if info: previews = [ str(file) for file in files.get_files() if file.name.lower().endswith("_preview.png") ] if previews: preview = previews[0] break if not info: for blendswap_info in files.get_by_type("blendswap_info"): url = blendswap_info.data info, preview = get_info() if info: break if not info and asset_parser.seven_z: for sbsar in files.get_by_type("sbsar"): is_ok, result = asset_parser.get_info_from_sbsar(str(sbsar)) if is_ok: info = result xml_attrs = info.pop("xml_attrs") if info.get("author") in ("Allegorithmic", "Adobe") or all( map(xml_attrs.get, ("pkgurl", "label", "keywords", "category", "author", "authorurl"))): sbsar_info = info label = info["name"] # if is Adobe name == label info_by_label = asset_parser.get_web_substance_source_info_by_label( label) if info_by_label: info = info_by_label description = sbsar_info.get("description") if description: info["description"] = description # if sketchfab asset --> use folder structure and try to utilize info about the scene if not preview: if not files.get_by_type("__icon__"): posible_icons = [ file for file in files.get_by_extension(('.png', '.jpg', '.jpeg')) if not file.is_meta ] if not posible_icons: # render asset's preview pass if len(posible_icons) == 1: preview = posible_icons[0] if preview: utils.move_to_folder(preview, gallery_folder) asset = cls.new(id_path, exist_ok=True) if ignore_info and old_info: asset.update_info(old_info) asset.update_info(info) asset.standardize_info() id = id.lower() return id, asset