def delete_files(cls, labbook: LabBook, section: str,
relative_paths: List[str]) -> None:
"""Delete file (or directory) from inside lb section.
The list of paths is deleted in series. Only provide "parent" nodes in the file tree. This is because deletes
on directories will remove all child objects, so subsequent deletes of individual files will then fail.
Args:
labbook(LabBook): Subject LabBook
section(str): Section name (code, input, output)
relative_paths(list(str)): a list of relative paths from labbook root to target
Returns:
None
"""
labbook.validate_section(section)
if not isinstance(relative_paths, list):
raise ValueError("Must provide list of paths to remove")
for file_path in relative_paths:
relative_path = LabBook.make_path_relative(file_path)
target_path = os.path.join(labbook.root_dir, section,
relative_path)
if not os.path.exists(target_path):
raise ValueError(
f"Attempted to delete non-existent path at `{target_path}`"
)
else:
labbook.git.remove(target_path, force=True, keep_file=False)
if os.path.exists(target_path):
raise IOError(f"Failed to delete path: {target_path}")
labbook.sweep_uncommitted_changes(show=True)
def put_file(cls,
labbook: LabBook,
section: str,
src_file: str,
dst_path: str,
txid: Optional[str] = None) -> Dict[str, Any]:
"""Move the file at `src_file` to `dst_dir`. Filename removes
upload ID if present. This operation does NOT commit or create an
activity record.
Args:
labbook: Subject LabBook
section: Section name (code, input, output)
src_file: Full path of file to insert into
dst_path: Path within section to insert `src_file`
txid: Optional transaction id
Returns:
Full path to inserted file.
"""
if not os.path.abspath(src_file):
raise ValueError(f"Source file `{src_file}` not an absolute path")
if not os.path.isfile(src_file):
raise ValueError(f"Source file does not exist at `{src_file}`")
labbook.validate_section(section)
r = call_subprocess(
['git', 'check-ignore',
os.path.basename(dst_path)],
cwd=labbook.root_dir,
check=False)
if dst_path and r and os.path.basename(dst_path) in r:
logger.warning(f"File {dst_path} matches gitignore; "
f"not put into {str(labbook)}")
raise FileOperationsException(f"`{dst_path}` matches "
f"ignored pattern")
mdst_dir = _make_path_relative(dst_path)
full_dst = os.path.join(labbook.root_dir, section, mdst_dir)
full_dst = full_dst.replace('..', '')
full_dst = full_dst.replace('~', '')
# Force overwrite if file already exists
if os.path.isfile(os.path.join(full_dst, os.path.basename(src_file))):
os.remove(os.path.join(full_dst, os.path.basename(src_file)))
if not os.path.isdir(os.path.dirname(full_dst)):
os.makedirs(os.path.dirname(full_dst), exist_ok=True)
fdst = shutil.move(src_file, full_dst)
relpath = fdst.replace(os.path.join(labbook.root_dir, section), '')
return cls.get_file_info(labbook, section, relpath)
def walkdir(cls,
labbook: LabBook,
section: str,
show_hidden: bool = False) -> List[Dict[str, Any]]:
"""Return a list of all files and directories in a section of the labbook. Never includes the .git or
.gigantum directory.
Args:
labbook: Subject LabBook
section(str): The labbook section (code, input, output) to walk
show_hidden(bool): If True, include hidden directories (EXCLUDING .git and .gigantum)
Returns:
List[Dict[str, str]]: List of dictionaries containing file and directory metadata
"""
labbook.validate_section(section)
keys: List[str] = list()
# base_dir is the root directory to search, to account for relative paths inside labbook.
base_dir = os.path.join(labbook.root_dir, section)
if not os.path.isdir(base_dir):
raise ValueError(
f"Labbook walkdir base_dir {base_dir} not an existing directory"
)
for root, dirs, files in os.walk(base_dir):
# Remove directories we ignore so os.walk does not traverse into them during future iterations
if '.git' in dirs:
del dirs[dirs.index('.git')]
if '.gigantum' in dirs:
del dirs[dirs.index('.gigantum')]
# For more deterministic responses, sort resulting paths alphabetically.
# Store directories then files, so pagination loads things in an intuitive order
dirs.sort()
keys.extend(
sorted([
os.path.join(root.replace(base_dir, ''), d) for d in dirs
]))
keys.extend(
sorted([
os.path.join(root.replace(base_dir, ''), f) for f in files
]))
# Create stats
stats: List[Dict[str, Any]] = list()
for f_p in keys:
if not show_hidden and any(
[len(p) and p[0] == '.' for p in f_p.split(os.path.sep)]):
continue
stats.append(cls.get_file_info(labbook, section, f_p))
return stats
def listdir(cls,
labbook: LabBook,
section: str,
base_path: Optional[str] = None,
show_hidden: bool = False) -> List[Dict[str, Any]]:
"""Return a list of all files and directories in a directory. Never includes the .git or
.gigantum directory.
Args:
labbook: Subject labbook
section(str): the labbook section to start from
base_path(str): Relative base path, if not listing from labbook's root.
show_hidden(bool): If True, include hidden directories (EXCLUDING .git and .gigantum)
Returns:
List[Dict[str, str]]: List of dictionaries containing file and directory metadata
"""
labbook.validate_section(section)
# base_dir is the root directory to search, to account for relative paths inside labbook.
base_dir = os.path.join(labbook.root_dir, section, base_path or '')
if not os.path.isdir(base_dir):
raise ValueError(
f"Labbook listdir base_dir {base_dir} not an existing directory"
)
stats: List[Dict[str, Any]] = list()
for item in os.listdir(base_dir):
if item in ['.git', '.gigantum']:
# Never include .git or .gigantum
continue
if not show_hidden and any(
[len(p) and p[0] == '.' for p in item.split('/')]):
continue
# Create tuple (isDir, key)
stats.append(
cls.get_file_info(labbook, section,
os.path.join(base_path or "", item)))
# For more deterministic responses, sort resulting paths alphabetically.
return sorted(stats, key=lambda a: a['key'])
def move_file(cls, labbook: LabBook, section: str, src_rel_path: str, dst_rel_path: str) \
-> List[Dict[str, Any]]:
"""Move a file or directory within a labbook, but not outside of it. Wraps
underlying "mv" call.
Args:
labbook: Subject LabBook
section(str): Section name (code, input, output)
src_rel_path(str): Source file or directory
dst_rel_path(str): Target file name and/or directory
"""
# Start with Validations
labbook.validate_section(section)
if not src_rel_path:
raise ValueError("src_rel_path cannot be None or empty")
if dst_rel_path is None:
raise ValueError("dst_rel_path cannot be None or empty")
src_rel_path = LabBook.make_path_relative(src_rel_path)
dst_rel_path = LabBook.make_path_relative(dst_rel_path)
src_abs_path = os.path.join(labbook.root_dir, section,
src_rel_path.replace('..', ''))
dst_abs_path = os.path.join(labbook.root_dir, section,
dst_rel_path.replace('..', ''))
if not os.path.exists(src_abs_path):
raise ValueError(f"No src file exists at `{src_abs_path}`")
try:
src_type = 'directory' if os.path.isdir(src_abs_path) else 'file'
logger.info(
f"Moving {src_type} `{src_abs_path}` to `{dst_abs_path}`")
labbook.git.remove(src_abs_path, keep_file=True)
final_dest = shutil.move(src_abs_path, dst_abs_path)
commit_msg = f"Moved {src_type} `{src_rel_path}` to `{dst_rel_path}`"
cls._make_move_activity_record(labbook, section, dst_abs_path,
commit_msg)
if os.path.isfile(final_dest):
t = final_dest.replace(os.path.join(labbook.root_dir, section),
'')
return [cls.get_file_info(labbook, section, t or "/")]
else:
moved_files = list()
t = final_dest.replace(os.path.join(labbook.root_dir, section),
'')
moved_files.append(
cls.get_file_info(labbook, section, t or "/"))
for root, dirs, files in os.walk(final_dest):
rt = root.replace(os.path.join(labbook.root_dir, section),
'')
rt = _make_path_relative(rt)
for d in sorted(dirs):
dinfo = cls.get_file_info(labbook, section,
os.path.join(rt, d))
moved_files.append(dinfo)
for f in filter(lambda n: n != '.gitkeep', sorted(files)):
finfo = cls.get_file_info(labbook, section,
os.path.join(rt, f))
moved_files.append(finfo)
return moved_files
except Exception as e:
logger.critical(
"Failed moving file in labbook. Repository may be in corrupted state."
)
logger.exception(e)
raise