def is_set_untracked(cls, labbook: LabBook, section: str) -> bool:
""" Return True if the given labbook section is set to be untracked
(to work around git performance issues when files are large).
Args:
labbook: Subject labbook
section: Section one of code, input, or output.
Returns:
bool indicating whether the labbook's section is set as untracked
"""
return in_untracked(labbook.root_dir, section)
def delete_files(cls, labbook: LabBook, section: str,
relative_paths: List[str]) -> None:
"""Delete file (or directory) from inside lb section.
The list of paths is deleted in series. Only provide "parent" nodes in the file tree. This is because deletes
on directories will remove all child objects, so subsequent deletes of individual files will then fail.
Args:
labbook(LabBook): Subject LabBook
section(str): Section name (code, input, output)
relative_paths(list(str)): a list of relative paths from labbook root to target
Returns:
None
"""
labbook.validate_section(section)
is_untracked = in_untracked(labbook.root_dir, section=section)
if not isinstance(relative_paths, list):
raise ValueError("Must provide list of paths to remove")
for file_path in relative_paths:
relative_path = LabBook.make_path_relative(file_path)
target_path = os.path.join(labbook.root_dir, section,
relative_path)
if not os.path.exists(target_path):
raise ValueError(
f"Attempted to delete non-existent path at `{target_path}`"
)
else:
if is_untracked:
if os.path.isdir(target_path):
shutil.rmtree(target_path)
else:
os.remove(target_path)
else:
labbook.git.remove(target_path,
force=True,
keep_file=False)
if os.path.exists(target_path):
raise IOError(f"Failed to delete path: {target_path}")
if not is_untracked:
labbook.sweep_uncommitted_changes(show=True)
def makedir(cls,
labbook: LabBook,
relative_path: str,
make_parents: bool = True,
create_activity_record: bool = False) -> None:
"""Make a new directory inside the labbook directory.
Args:
labbook: Subject LabBook
relative_path(str): Path within the labbook to make directory
make_parents(bool): If true, create intermediary directories
create_activity_record(bool): If true, create commit and activity record
Returns:
str: Absolute path of new directory
"""
if not relative_path:
raise ValueError("relative_path argument cannot be None or empty")
relative_path = LabBook.make_path_relative(relative_path)
new_directory_path = os.path.join(labbook.root_dir, relative_path)
section = relative_path.split(os.sep)[0]
git_untracked = in_untracked(labbook.root_dir, section)
if os.path.exists(new_directory_path):
return
else:
logger.info(f"Making new directory in `{new_directory_path}`")
os.makedirs(new_directory_path, exist_ok=make_parents)
if git_untracked:
logger.warning(
f'New {str(labbook)} untracked directory `{new_directory_path}`'
)
return
new_dir = ''
for d in relative_path.split(os.sep):
new_dir = os.path.join(new_dir, d)
full_new_dir = os.path.join(labbook.root_dir, new_dir)
gitkeep_path = os.path.join(full_new_dir, '.gitkeep')
if not os.path.exists(gitkeep_path):
with open(gitkeep_path, 'w') as gitkeep:
gitkeep.write(
"This file is necessary to keep this directory tracked by Git"
" and archivable by compression tools. Do not delete or modify!"
)
labbook.git.add(gitkeep_path)
if create_activity_record:
# Create detail record
activity_type, activity_detail_type, section_str = labbook.infer_section_from_relative_path(
relative_path)
adr = ActivityDetailRecord(activity_detail_type,
show=False,
importance=0,
action=ActivityAction.CREATE)
msg = f"Created new {section_str} directory `{relative_path}`"
commit = labbook.git.commit(msg)
adr.add_value('text/markdown', msg)
# Create activity record
ar = ActivityRecord(activity_type,
message=msg,
linked_commit=commit.hexsha,
show=True,
importance=255,
tags=['directory-create'])
ar.add_detail_object(adr)
# Store
ars = ActivityStore(labbook)
ars.create_activity_record(ar)
def move_file(cls, labbook: LabBook, section: str, src_rel_path: str, dst_rel_path: str) \
-> List[Dict[str, Any]]:
"""Move a file or directory within a labbook, but not outside of it. Wraps
underlying "mv" call.
Args:
labbook: Subject LabBook
section(str): Section name (code, input, output)
src_rel_path(str): Source file or directory
dst_rel_path(str): Target file name and/or directory
"""
# Start with Validations
labbook.validate_section(section)
if not src_rel_path:
raise ValueError("src_rel_path cannot be None or empty")
if dst_rel_path is None:
raise ValueError("dst_rel_path cannot be None or empty")
is_untracked = in_untracked(labbook.root_dir, section)
src_rel_path = LabBook.make_path_relative(src_rel_path)
dst_rel_path = LabBook.make_path_relative(dst_rel_path)
src_abs_path = os.path.join(labbook.root_dir, section,
src_rel_path.replace('..', ''))
dst_abs_path = os.path.join(labbook.root_dir, section,
dst_rel_path.replace('..', ''))
if not os.path.exists(src_abs_path):
raise ValueError(f"No src file exists at `{src_abs_path}`")
try:
src_type = 'directory' if os.path.isdir(src_abs_path) else 'file'
logger.info(
f"Moving {src_type} `{src_abs_path}` to `{dst_abs_path}`")
if not is_untracked:
labbook.git.remove(src_abs_path, keep_file=True)
final_dest = shutil.move(src_abs_path, dst_abs_path)
if not is_untracked:
commit_msg = f"Moved {src_type} `{src_rel_path}` to `{dst_rel_path}`"
cls._make_move_activity_record(labbook, section, dst_abs_path,
commit_msg)
if os.path.isfile(final_dest):
t = final_dest.replace(os.path.join(labbook.root_dir, section),
'')
return [cls.get_file_info(labbook, section, t or "/")]
else:
moved_files = list()
t = final_dest.replace(os.path.join(labbook.root_dir, section),
'')
moved_files.append(
cls.get_file_info(labbook, section, t or "/"))
for root, dirs, files in os.walk(final_dest):
rt = root.replace(os.path.join(labbook.root_dir, section),
'')
rt = _make_path_relative(rt)
for d in sorted(dirs):
dinfo = cls.get_file_info(labbook, section,
os.path.join(rt, d))
moved_files.append(dinfo)
for f in filter(lambda n: n != '.gitkeep', sorted(files)):
finfo = cls.get_file_info(labbook, section,
os.path.join(rt, f))
moved_files.append(finfo)
return moved_files
except Exception as e:
logger.critical(
"Failed moving file in labbook. Repository may be in corrupted state."
)
logger.exception(e)
raise
def insert_file(cls,
labbook: LabBook,
section: str,
src_file: str,
dst_path: str = '') -> Dict[str, Any]:
""" Move the file at `src_file` into the `dst_dir`, overwriting
if a file already exists there. This calls `put_file()` under-
the-hood, but will create an activity record.
Args:
labbook: Subject labbook
section: Section name (code, input, output)
src_file: Full path of file to insert into
dst_path: Relative path within labbook where `src_file`
should be copied to
Returns:
dict: The inserted file's info
"""
finfo = FileOperations.put_file(labbook=labbook,
section=section,
src_file=src_file,
dst_path=dst_path)
rel_path = os.path.join(section, finfo['key'])
if in_untracked(labbook.root_dir, section):
logger.warning(f"Inserted file {rel_path} ({finfo['size']} bytes)"
f" to untracked section {section}. This will not"
f" be tracked by commits or activity records.")
return finfo
# If we are setting this section to be untracked
activity_type, activity_detail_type, section_str = \
labbook.get_activity_type_from_section(section)
commit_msg = f"Added new {section_str} file {rel_path}"
try:
labbook.git.add(rel_path)
commit = labbook.git.commit(commit_msg)
except Exception as x:
logger.error(x)
os.remove(dst_path)
raise FileOperationsException(x)
# Create Activity record and detail
_, ext = os.path.splitext(rel_path) or 'file'
adr = ActivityDetailRecord(activity_detail_type,
show=False,
importance=0,
action=ActivityAction.CREATE)
adr.add_value('text/plain', commit_msg)
ar = ActivityRecord(activity_type,
message=commit_msg,
show=True,
importance=255,
linked_commit=commit.hexsha,
tags=[ext])
ar.add_detail_object(adr)
ars = ActivityStore(labbook)
ars.create_activity_record(ar)
return finfo