def create_link(entry: os.DirEntry, dotconfig: bool):
src = entry.path
if dotconfig:
dst = os.path.join(DOT_CONFIG, entry.name)
else:
dst = os.path.join(HOME, entry.name)
# If dst is a symlink
if os.path.islink(dst):
# If dst links to src, all is good
if Path(os.readlink(dst)) == Path(src):
logger.debug(f"{dst} already linked correctly, skipping ...")
return
elif ARGS.remove_symlinks:
# Remove symlinks if arg is set
logger.info(f"Removing {dst}")
os.remove(dst)
elif os.path.exists(dst):
# Not a symlink, but file exists
logger.warning(f"Destination: {dst} already exists (dir)")
logger.warning(f"Moving {dst} to {dst}.backup")
shutil.move(dst, dst + ".backup")
# No case catched -> create symlink
os.symlink(src, dst, target_is_directory=entry.is_dir())
logger.info(f"{src} -> {dst}")
def isToBeIgnored(self,entry:os.DirEntry):
if not entry.is_dir():
return False
if entry.name in self.ignoreDirList:
lgg.info(f" Ignoring {entry.name}",lgg.cR)
return True
return False
def process_one_file(parents: Sequence[str], item: os.DirEntry) -> None:
"""Process a JSON file during the main walk."""
global _sys_event_tables
def is_leaf_dir(path: str) -> bool:
for item in os.scandir(path):
if item.is_dir():
return False
return True
# model directory, reset topic
if item.is_dir() and is_leaf_dir(item.path):
print_events_table_suffix()
tblname = file_name_to_table_name(parents, item.name)
if item.name == 'sys':
_sys_event_tables.append(tblname)
print_events_table_prefix(tblname)
return
# base dir or too deep
level = len(parents)
if level == 0 or level > 4:
return
# Ignore other directories. If the file name does not have a .json
# extension, ignore it. It could be a readme.txt for instance.
if not item.is_file() or not item.name.endswith('.json'):
return
add_events_table_entries(item, get_topic(item.name))
def is_game_dir(self,
entry: os.DirEntry,
ignore_dirs: Optional[List[str]] = None) -> bool:
if ignore_dirs is None:
ignore_dirs = self.ignore_dirs
return (entry.is_dir() and self._is_game_match(entry.name)
and entry.name.lower() not in ignore_dirs)
def _is_backup_entity(backup_entry: os.DirEntry) -> bool:
""" Check if entity_path is a single backup dir. """
if not backup_entry.is_dir():
return False
try:
datetime.strptime(backup_entry.name, BACKUP_ENT_FMT)
return True
except ValueError:
return False
def filter_bib_id_folders(item: os.DirEntry):
if not item.is_dir():
return False
if "v" not in item.name and not isinstance(eval(item.name), int):
return False
return True
def is_editor_file(entry: os.DirEntry) -> bool:
FOLDER_PATTERNS = ('.idea', '.vscode')
FILE_PATTERNS = (r'\.sw.$', r'~$')
if entry.is_dir():
g = (_ == entry.name for _ in FOLDER_PATTERNS)
else:
g = (re.search(_, entry.name) for _ in FILE_PATTERNS)
return any(g)
def __try_read_file(dir_entry: os.DirEntry, scan_dir: str, base_path: str):
"""Attempt to read a file. Internal.
If the file given by ``dir_entry`` is a Maven config object, return
a tuple of (True, JSON dict) representing that object. If the object
could be read but wasn't a part, return (False, None). If the object
couldn't be read, raise an exception.
:param dir_entry: A DirEntry object, as returned by os.scandir
:param scan_dir: The directory on which os.scandir was called. May be
an empty string.
:param base_path: The base path of the config, where "/" is the root of
the config.
"""
name, ext = os.path.splitext(dir_entry.path)
if dir_entry.is_dir() or ext != ".part":
return (False, None)
file_path = os.path.join(base_path, scan_dir, dir_entry.path)
try:
with open(file_path, 'r') as f:
data = json.load(f)
path = ConfigObjectTracker.__norm_path(file_path, base_path)
# return a corrected dict
obj = {
# todo: Should we generate a uuid1 from the file's inode?
# Technically IDs should be unique to a config and
# immutable across a given object's lifetime, however
# inodes are only unique to a block device and may change
# with renames/moves. This means a config with a symlink to
# a network drive or another disk might have duplicate
# inodes, and with uuid1s, they will have duplicate IDs.
"id": data["id"],
"typeName": data["typeName"],
# Because this is a directory store, we can safely ignore
# whatever the file says is true, because the filesystem
# is more accurate in these regards
"lastModified": os.path.getmtime(file_path),
"name": os.path.basename(name),
"path": path,
}
if "data" in data:
# this is a FileConfigObject, attach the data
obj["data"] = data["data"]
if "functionType" in data and "arguments" in data:
# this is a UDF or UDP, attach those args
obj["functionType"] = data["functionType"]
obj["arguments"] = data["arguments"]
return (True, obj)
except (json.JSONDecodeError, KeyError):
# Not a part, or a malformed part
return (False, None)
except UnicodeDecodeError as e:
# Corrupted file, could not read
warnings.warn("Failed to read file" + file_path)
warnings.warn(e)
return (False, None)
def print_dir_entry(entry: DirEntry,
*,
path: bool = False,
path_field_size=70):
stat = entry.stat()
kind = 'DIR' if entry.is_dir() else 'FILE'
data = entry.path if path else entry.name
print(
f"{data:<{path_field_size}s} {kind:<4s} {str(stat.st_size) if entry.is_file() else '':>8s}"
)
def _get_attributes(self, item: os.DirEntry) -> dict:
"""Parse entire item and subdirectories.
Returns:
* Total size in bytes
* Maximum folder depth of item
* Total number of files this item contains
* Access timestamp
* Modification timestamp
* Change timestamp
in the same order as tuple.
:param item: DirEntry object
:type item: posix.DirEntry
:return: Dictionary of {size, depth, num_of_files, atime, mtime, ctime}
:rtype: dict
"""
# it's a file or symlink, size is already on item stat
if not item.is_dir(follow_symlinks=False):
stat = item.stat(follow_symlinks=False)
return {
"size": stat.st_size,
"depth": self._get_depth(item.path) - self._level,
"num_of_files": 1,
"atime": int(stat.st_atime),
"mtime": int(stat.st_mtime),
"ctime": int(stat.st_ctime),
}
# It is a folder, recursive size check
else:
total_size = num_of_files = depth = 0
atime = mtime = ctime = 0
# TODO: try/except catch PermissionError
with os.scandir(item.path) as directory:
for i in directory:
attrs = self._get_attributes(i)
total_size += attrs["size"]
num_of_files += attrs["num_of_files"]
atime = max(atime, attrs["atime"])
mtime = max(mtime, attrs["mtime"])
ctime = max(ctime, attrs["ctime"])
depth = max(depth, attrs["depth"])
return {
"size": total_size,
"depth": depth,
"num_of_files": num_of_files,
"atime": atime,
"mtime": mtime,
"ctime": ctime,
}
def directory_only_filter(item: os.DirEntry):
if not item.is_dir():
return False
if not os.access(item.path, os.F_OK):
return False
if not os.access(item.path, os.R_OK):
return False
return True
def _survives(self, base_path: str, entry: os.DirEntry) -> bool:
"""
Determines if a single file entry survives the ignore filter.
"""
for p in self._processed_patterns:
if (entry.is_dir() and p.endswith("/")
and fnmatch.fnmatch(entry.path, p[:-1])):
return False
if fnmatch.fnmatch(entry.path, p):
return False
return True
def _get_entry_attributes(entry: os.DirEntry):
attrs = FileAttributes.NONE
if entry.is_dir():
attrs |= FileAttributes.IS_DIR
elif entry.is_file():
attrs |= FileAttributes.IS_FILE
if entry.is_symlink():
attrs |= FileAttributes.IS_LINK
if entry.name.startswith('.'):
attrs |= FileAttributes.IS_HIDDEN
return attrs
def button_made_from(entry: os.DirEntry) -> Gtk.Button:
if entry.is_file():
return Copy(Snippet().load(entry.path))
if entry.is_dir():
return GoTo(
name=entry.name,
position=os.path.dirname(entry.path),
destination=entry.path,
)
def os_dir_entry_to_directory_list_entry(
virtual_path: str, dir_entry: os.DirEntry) -> DirectoryListEntry:
"""Convert an `os.DirEntry` instance to a `DirectoryListEntry`."""
s: os.stat_result = dir_entry.stat()
return DirectoryListEntry(
dir_entry.name.decode("utf8"),
utf8_path_join(virtual_path, dir_entry.name),
DirectoryEntryType.DIRECTORY
if dir_entry.is_dir() else DirectoryEntryType.FILE,
DiskSource(),
datetime.datetime.fromtimestamp(s.st_mtime),
)
def __init__(self, entry: DirEntry):
self.name = entry.name.decode()
self.path = entry.path.decode()
self.rel_path = relpath(self.path, models_dir)
self.is_dir = entry.is_dir()
# save created_time and modified_time as date objects to provide
# better date comparison
self.created_time = datetime.fromtimestamp(entry.stat().st_ctime)
self.modified_time = datetime.fromtimestamp(entry.stat().st_mtime)
self.size = self._human_readable_size(self._get_size(entry.path))
def extract_individual_recursive(realm: str, dataset_entry: os.DirEntry,
entry: os.DirEntry,
metalad_arguments: List[str]):
if entry.is_dir():
child_entries = tuple(os.scandir(entry.path))
is_dataset = len(
tuple(filter(lambda entry: entry.name == ".datalad",
child_entries))) == 1
if is_dataset:
extract_individual(entry, metalad_arguments)
for entry in child_entries:
if should_be_ignored(entry.name):
continue
extract_individual_recursive(realm, dataset_entry, entry,
metalad_arguments)
def filter_bib_id_folders(cls, item: os.DirEntry) -> bool:
"""Filter only folders with bibids.
Args:
item:
Returns:
True is the item is a folder with a bibid, else returns false
"""
if not item.is_dir():
return False
if "v" not in item.name and not isinstance(eval(item.name), int):
return False
return True
def preprocess_one_file(parents: Sequence[str], item: os.DirEntry) -> None:
if item.is_dir():
return
# base dir or too deep
level = len(parents)
if level == 0 or level > 4:
return
# Ignore other directories. If the file name does not have a .json
# extension, ignore it. It could be a readme.txt for instance.
if not item.is_file() or not item.name.endswith('.json'):
return
topic = get_topic(item.name)
for event in read_json_events(item.path, topic):
_bcs.add(event.build_c_string())
def extract_recursive(realm: str, dataset_entry: os.DirEntry,
entry: os.DirEntry, metalad_arguments: List[str]):
if entry.is_dir():
child_entries = tuple(os.scandir(entry.path))
is_dataset = len(
tuple(filter(lambda entry: entry.name == ".datalad",
child_entries))) == 1
if is_dataset:
extract_dataset_level_metadata(realm, entry.path,
metalad_arguments)
dataset_entry = entry
for entry in child_entries:
if should_be_ignored(entry.name):
continue
extract_recursive(realm, dataset_entry, entry, metalad_arguments)
else:
extract_file_level_metadata(realm, dataset_entry.path,
entry.path[len(dataset_entry.path) + 1:],
metalad_arguments)
def _is_valid_dir(self, test_dir: os.DirEntry) -> bool:
"""Returns :const:`True` if it's a valid directory
Only some directories are valid, the ones that start with FHBO
(which is our original Head and Neck dataset)
and then only the only ones that have two sub dirs:
- The scan directory
- The mask directory
:param test_dir: Directory to be tested
:return: True or False depending on the folder conditions
"""
# We are looking for directories, only the ones that we have the clinical info are valid
name = str(test_dir.name)
if not test_dir.is_dir() or name not in self.clinical_ids or name not in self.radiomic_ids:
return False
# Check if it has two sub dirs that start with the same name
sub_dirs = list(filter(lambda x: x.is_dir() and str(x.name).startswith(name), os.scandir(test_dir.path)))
return len(sub_dirs) >= 2
def is_python_cache(entry: os.DirEntry) -> bool:
if entry.is_dir():
return entry.name in ('__pycache__', 'site-packages')
else:
return re.search(r'\.py[cod]$', entry.name) is not None
def wrapper(entry: os.DirEntry, *args, **kwargs):
if not entry.is_dir():
return False
return fn(entry, *args, **kwargs)
def is_dir(self, entry: DirEntry) -> bool:
return entry.is_dir(follow_symlinks=self.__follow_symlinks)
def find_dirs(item: os.DirEntry):
if not item.is_dir():
return False
return True
def remove(entry: os.DirEntry):
logging.info(f'Removing old backup: {entry.name}')
if entry.is_dir():
shutil.rmtree(entry.path)
else:
os.remove(entry.path)
def _from_os_dir_entry(original: os.DirEntry) -> fs.DirEntry:
return fs.DirEntry(
url=urls.URL("file://" + os.path.abspath(original.path)),
is_dir=bool(original.is_dir()),
is_file=bool(original.is_file()),
)
def is_table(directory: DirEntry) -> bool:
"""Checks if a directory contains a table."""
return directory.is_dir(follow_symlinks=False) and valid_table_contents(
directory.path)
def _f_dir(entry: os.DirEntry) -> bool:
return entry.is_dir()
