def _is_restaurant_module(entry: DirEntry) -> bool:
"""Checks if entry is a Python file, ignores if name starts with underscore."""
return all([
entry.is_file(),
entry.name.endswith('.py'), # type: ignore
not entry.name.startswith('_') # type: ignore
])
def process_one_file(parents: Sequence[str], item: os.DirEntry) -> None:
"""Process a JSON file during the main walk."""
global _sys_event_tables
def is_leaf_dir(path: str) -> bool:
for item in os.scandir(path):
if item.is_dir():
return False
return True
# model directory, reset topic
if item.is_dir() and is_leaf_dir(item.path):
print_events_table_suffix()
tblname = file_name_to_table_name(parents, item.name)
if item.name == 'sys':
_sys_event_tables.append(tblname)
print_events_table_prefix(tblname)
return
# base dir or too deep
level = len(parents)
if level == 0 or level > 4:
return
# Ignore other directories. If the file name does not have a .json
# extension, ignore it. It could be a readme.txt for instance.
if not item.is_file() or not item.name.endswith('.json'):
return
add_events_table_entries(item, get_topic(item.name))
def filter_alebmic_ini(entry: os.DirEntry) -> bool:
if not entry.is_file():
return False
if entry.name != "alembic.ini":
return False
return True
def _component_filter(self, entry: os.DirEntry):
if not entry.is_file():
return False
base, ext = os.path.splitext(entry.name)
if not self._component_mask.fullmatch(base):
return False
return True
def filter_only_tif_files(item: os.DirEntry) -> bool:
if not item.is_file():
return False
_, ext = os.path.splitext(item.name)
if ext.lower() != ".tif":
return False
return True
def _process_file_or_dir(context: str,
dir_entry: os.DirEntry) -> [[AssetFile]]:
if dir_entry.is_file():
return AssetFile.build(dir_entry=dir_entry, context=context)
else:
new_context = os.path.join(context, dir_entry.name)
return [
_process_file_or_dir(new_context, f)
for f in os.scandir(dir_entry.path)
if f.name[:1] != "." and f.name[:1] != "_"
]
def button_made_from(entry: os.DirEntry) -> Gtk.Button:
if entry.is_file():
return Copy(Snippet().load(entry.path))
if entry.is_dir():
return GoTo(
name=entry.name,
position=os.path.dirname(entry.path),
destination=entry.path,
)
def _get_entry_attributes(entry: os.DirEntry):
attrs = FileAttributes.NONE
if entry.is_dir():
attrs |= FileAttributes.IS_DIR
elif entry.is_file():
attrs |= FileAttributes.IS_FILE
if entry.is_symlink():
attrs |= FileAttributes.IS_LINK
if entry.name.startswith('.'):
attrs |= FileAttributes.IS_HIDDEN
return attrs
def filter_only_trainingdata(item: os.DirEntry) -> bool:
if not item.is_file():
return False
base, ext = os.path.splitext(item.name)
if ext != ".traineddata":
return False
if base == "osd":
return False
return True
def filter_ocr_only(entry: os.DirEntry):
if not entry.is_file():
return False
name, ext = os.path.splitext(entry.name)
if ext.lower() != ".xml":
return False
if name.lower() == "marc":
return False
return True
def validate_file(self, dir_entry: os.DirEntry) -> bool:
"""Validates given DirEntry. Returns False if entry should be completely ignored,
or True if we want to keep it for further processing.
Ignore all zero length files. There are usually there for a purpose like .dummy etc,
so there can be tons of it with the same name even, so by default, ignore them completely.
Also ignore all symlinks."""
from .log import Log
if dir_entry.is_symlink():
Log.vv('{name}: It is the symbolic link. Skipping.'.format(
name=dir_entry.name))
return False
# NOTE: do not call is_file() on DirEntry. It will fail in endless
# recursion for invalid (dead) symbolic links. os.path.isfile() works).
if not dir_entry.is_file():
Log.vv('{name}: This is not a file. Skipping.'.format(
name=dir_entry.name))
return False
item_size = dir_entry.stat().st_size
if item_size == 0:
Log.vv('{name}: File is 0 bytes long. Skipping.'.format(
name=dir_entry.name))
return False
if self.min_size > 0 and item_size < self.min_size:
Log.vv('{name}: File is shorter than min size ({size}). Skipping.'.
format(name=dir_entry.name, size=item_size))
return False
if 0 < self.max_size < item_size:
Log.vv('{name}: File is biger than max size ({size}). Skipping.'.
format(name=dir_entry.name, size=item_size))
return False
for list_item in self._file_name_blacklist:
match = re.match(list_item, dir_entry.name)
if match is not None:
Log.vv('File "{name}" blacklisted by "{re}" rule. Skipping.'.
format(name=dir_entry.name, re=list_item))
return False
return True
def preprocess_one_file(parents: Sequence[str], item: os.DirEntry) -> None:
if item.is_dir():
return
# base dir or too deep
level = len(parents)
if level == 0 or level > 4:
return
# Ignore other directories. If the file name does not have a .json
# extension, ignore it. It could be a readme.txt for instance.
if not item.is_file() or not item.name.endswith('.json'):
return
topic = get_topic(item.name)
for event in read_json_events(item.path, topic):
_bcs.add(event.build_c_string())
def is_info_file(file: DirEntry) -> bool:
"""Checks if a file is an info file."""
return is_info(file.name) and file.is_file(follow_symlinks=False)
def is_shard_file(file: DirEntry) -> bool:
"""Checks if a file is a shard."""
return is_shard(file.name) and file.is_file(follow_symlinks=False)
def is_python_file(item: os.DirEntry):
return item.is_file() and item.name.endswith('.py')
def _file_can_be_imported(entry: os.DirEntry) -> bool:
""" Verify if a directory entry is a non-magic Python file."""
return entry.is_file(
) and not entry.name.startswith("__") and entry.name.endswith(".py")
def is_file(self, entry: DirEntry) -> bool:
return entry.is_file(follow_symlinks=self.__follow_symlinks)
def _from_os_dir_entry(original: os.DirEntry) -> fs.DirEntry:
return fs.DirEntry(
url=urls.URL("file://" + os.path.abspath(original.path)),
is_dir=bool(original.is_dir()),
is_file=bool(original.is_file()),
)
def auto(cls,
path: os.DirEntry,
asset_data_path,
ignore_info=False
): # type: (os.DirEntry, str, bool) -> typing.Tuple[str, Asset]
info = {}
preview = None
id = os.path.splitext(path.name)[0]
if os.path.dirname(path.path) != asset_data_path:
if id:
number = 2
id_path = os.path.join(asset_data_path, id)
while True:
if os.path.exists(id_path):
id_path = os.path.join(asset_data_path,
id + f"_{number}")
number += 1
else:
break
else:
id_chars = "".join((string.ascii_lowercase, string.digits))
while True:
id = ''.join(random.choice(id_chars) for _ in range(11))
id_path = os.path.join(asset_data_path, id)
if not os.path.exists(id_path):
break
id_path = utils.PseudoDirEntry(id_path)
else:
id_path = path
extra_folder = os.path.join(id_path.path, "__extra__")
archive_folder = os.path.join(id_path.path, "__archive__")
gallery_folder = os.path.join(id_path.path, "__gallery__")
def get_info():
is_ok, result = asset_parser.get_web(url, id_path.path)
if is_ok:
preview = result.pop("preview_path", None)
return result, preview
return None, None
if path.is_file():
file = utils.pathlib.Path(path)
url = None
auto_folder = file.parent
url_files = [
utils.pathlib.Path(auto_folder, file.stem + extension)
for extension in utils.URL_EXTENSIONS
]
url_files = [
url for url in url_files if url.exists() and url.type == "url"
]
for url_file in url_files:
url = url_file.data
utils.move_to_folder(url_file, extra_folder)
if url:
info, preview = get_info()
if file.type == "zip":
utils.extract_zip(file, id_path.path)
utils.move_to_folder(file, archive_folder)
else:
utils.move_to_folder(file, id_path.path)
else:
id_path = utils.PseudoDirEntry(
utils.move_to_folder(path.path, asset_data_path))
files = utils.File_Filter.from_dir(id_path,
ignore=("__extra__", "__archive__"))
old_info = None
for existing_info in files.get_by_type("__info__"):
if ignore_info:
old_info = existing_info.data
break
else:
return id, cls.default(id_path)
zips = files.get_by_type("zip")
if zips:
for zip in zips:
utils.extract_zip(str(zip), path=id_path.path)
utils.move_to_folder(zip, archive_folder)
files.update()
if not info:
for url_file in files.get_by_type("url"):
url = url_file.data
info, preview = get_info()
if info:
break
if not info:
for megascan_info in files.get_by_type("megascan_info"):
megascan_id = megascan_info.data.get('id')
if not megascan_id:
continue
url = f"https://quixel.com/megascans/home?assetId={megascan_id}"
info, preview = get_info()
if info:
previews = [
str(file) for file in files.get_files()
if file.name.lower().endswith("_preview.png")
]
if previews:
preview = previews[0]
break
if not info:
for blendswap_info in files.get_by_type("blendswap_info"):
url = blendswap_info.data
info, preview = get_info()
if info:
break
if not info and asset_parser.seven_z:
for sbsar in files.get_by_type("sbsar"):
is_ok, result = asset_parser.get_info_from_sbsar(str(sbsar))
if is_ok:
info = result
xml_attrs = info.pop("xml_attrs")
if info.get("author") in ("Allegorithmic", "Adobe") or all(
map(xml_attrs.get,
("pkgurl", "label", "keywords", "category",
"author", "authorurl"))):
sbsar_info = info
label = info["name"] # if is Adobe name == label
info_by_label = asset_parser.get_web_substance_source_info_by_label(
label)
if info_by_label:
info = info_by_label
description = sbsar_info.get("description")
if description:
info["description"] = description
# if sketchfab asset --> use folder structure and try to utilize info about the scene
if not preview:
if not files.get_by_type("__icon__"):
posible_icons = [
file for file in files.get_by_extension(('.png', '.jpg',
'.jpeg'))
if not file.is_meta
]
if not posible_icons:
# render asset's preview
pass
if len(posible_icons) == 1:
preview = posible_icons[0]
if preview:
utils.move_to_folder(preview, gallery_folder)
asset = cls.new(id_path, exist_ok=True)
if ignore_info and old_info:
asset.update_info(old_info)
asset.update_info(info)
asset.standardize_info()
id = id.lower()
return id, asset
