def process(self, result_obj: ActivityRecord, data: List[ExecutionData],
status: Dict[str, Any], metadata: Dict[str,
Any]) -> ActivityRecord:
"""Method to update a result object based on code and result data
Args:
result_obj(ActivityNote): An object containing the note
data(list): A list of ExecutionData instances containing the data for this record
status(dict): A dict containing the result of git status from gitlib
metadata(str): A dictionary containing Dev Env specific or other developer defined data
Returns:
ActivityNote
"""
supported_image_types = [
'image/png', 'image/jpeg', 'image/jpg', 'image/gif', 'image/bmp'
]
# If a supported image exists in the result, grab it and create a detail record
result_cnt = 0
for cell in data:
for result_entry in reversed(cell.result):
if 'data' in result_entry:
for mime_type in result_entry['data']:
if mime_type in supported_image_types:
# You got an image
adr_img = ActivityDetailRecord(
ActivityDetailType.RESULT,
show=True,
action=ActivityAction.CREATE,
importance=max(255 - result_cnt, 0))
adr_img.add_value(mime_type,
result_entry['data'][mime_type])
adr_img.tags = cell.tags
result_obj.add_detail_object(adr_img)
# Set Activity Record Message
result_obj.message = "Executed cell in notebook {} and generated a result".format(
metadata['path'])
result_cnt += 1
return result_obj
def sweep_uncommitted_changes(self,
upload: bool = False,
extra_msg: Optional[str] = None,
show: bool = False) -> None:
""" Sweep all changes into a commit, and create activity record.
NOTE: This method MUST be called inside a lock.
Args:
upload(bool): Flag indicating if this was from a batch upload
extra_msg(str): Optional string used to augment the activity message
show(bool): Optional flag indicating if the result of this sweep is important enough to be shown in the feed
Returns:
"""
result_status = self.git.status()
if any([result_status[k] for k in result_status.keys()]):
self.git.add_all()
self.git.commit("Sweep of uncommitted changes")
ar = ActivityRecord(self._default_activity_type,
message="--overwritten--",
show=show,
importance=255,
linked_commit=self.git.commit_hash,
tags=['save'])
if upload:
ar.tags.append('upload')
ar, newcnt, modcnt, delcnt = self.process_sweep_status(
ar, result_status)
nmsg = f"{newcnt} new file(s). " if newcnt > 0 else ""
mmsg = f"{modcnt} modified file(s). " if modcnt > 0 else ""
dmsg = f"{delcnt} deleted file(s). " if delcnt > 0 else ""
message = f"{extra_msg or ''}" \
f"{'Uploaded ' if upload else ''}" \
f"{nmsg}{mmsg}{dmsg}"
# This is used to handle if you try to delete an empty directory. This shouldn't technically happen, but if
# a user manages to create an empty dir outside the client, we should handle it gracefully
ar.message = "No detected changes" if not message else message
ars = ActivityStore(self)
ars.create_activity_record(ar)
else:
logger.info(f"{str(self)} no changes to sweep.")
def process(self, result_obj: ActivityRecord, data: List[ExecutionData],
status: Dict[str, Any], metadata: Dict[str,
Any]) -> ActivityRecord:
"""Method to update a result object based on code and result data
Args:
result_obj(ActivityNote): An object containing the note
data(list): A list of ExecutionData instances containing the data for this record
status(dict): A dict containing the result of git status from gitlib
metadata(str): A dictionary containing Dev Env specific or other developer defined data
Returns:
ActivityRecord
"""
# If there was some code, assume a cell was executed
result_cnt = 0
for cell_cnt, cell in enumerate(data):
for result_entry in reversed(cell.code):
if result_entry.get('code'):
# Create detail record to capture executed code
adr_code = ActivityDetailRecord(
ActivityDetailType.CODE_EXECUTED,
show=False,
action=ActivityAction.EXECUTE,
importance=max(255 - result_cnt, 0))
adr_code.add_value(
'text/markdown',
f"```\n{result_entry.get('code')}\n```")
adr_code.tags = cell.tags
result_obj.add_detail_object(adr_code)
result_cnt += 1
# Set Activity Record Message
cell_str = f"{cell_cnt} cells" if cell_cnt > 1 else "cell"
result_obj.message = f"Executed {cell_str} in notebook {metadata['path']}"
return result_obj
def sweep_all_changes(self,
upload: bool = False,
extra_msg: str = None) -> None:
"""
Args:
upload:
extra_msg:
Returns:
"""
def _item_type(key):
if key[-1] == os.path.sep:
return 'directory'
else:
return 'file'
previous_revision = self.dataset_revision
# Update manifest
status = self.update()
if len(status.deleted) > 0 or len(status.created) > 0 or len(
status.modified) > 0:
# commit changed manifest file
self.dataset.git.add_all()
self.dataset.git.commit("Commit changes to manifest file.")
ar = ActivityRecord(
ActivityType.DATASET,
message="msg is set below after detail record processing...",
show=True,
importance=255,
linked_commit=self.dataset.git.commit_hash,
tags=[])
if upload:
ar.tags.append('upload')
for cnt, f in enumerate(status.created):
adr = ActivityDetailRecord(ActivityDetailType.DATASET,
show=False,
importance=max(255 - cnt, 0),
action=ActivityAction.CREATE)
msg = f"Created new {_item_type(f)} `{f}`"
adr.add_value('text/markdown', msg)
ar.add_detail_object(adr)
for cnt, f in enumerate(status.modified):
adr = ActivityDetailRecord(ActivityDetailType.DATASET,
show=False,
importance=max(255 - cnt, 0),
action=ActivityAction.EDIT)
msg = f"Modified {_item_type(f)} `{f}`"
adr.add_value('text/markdown', msg)
ar.add_detail_object(adr)
for cnt, f in enumerate(status.deleted):
adr = ActivityDetailRecord(ActivityDetailType.DATASET,
show=False,
importance=max(255 - cnt, 0),
action=ActivityAction.DELETE)
msg = f"Deleted {_item_type(f)} `{f}`"
adr.add_value('text/markdown', msg)
ar.add_detail_object(adr)
nmsg = f"{len(status.created)} new file(s). " if len(
status.created) > 0 else ""
mmsg = f"{len(status.modified)} modified file(s). " if len(
status.modified) > 0 else ""
dmsg = f"{len(status.deleted)} deleted file(s). " if len(
status.deleted) > 0 else ""
ar.message = f"{extra_msg if extra_msg else ''}" \
f"{'Uploaded ' if upload else ''}" \
f"{nmsg}{mmsg}{dmsg}"
ars = ActivityStore(self.dataset)
ars.create_activity_record(ar)
# Re-link new revision, unlink old revision
self.link_revision()
if os.path.isdir(
os.path.join(self.cache_mgr.cache_root, previous_revision)):
shutil.rmtree(
os.path.join(self.cache_mgr.cache_root, previous_revision))
def create_update_activity_record(self, status: StatusResult, upload: bool = False, extra_msg: str = None) -> None:
"""
Args:
status(StatusResult): a StatusResult object after updating the manifest
upload(bool): flag indicating if this is a record for an upload
extra_msg(str): any extra string to add to the activity record
Returns:
None
"""
def _item_type(key):
if key[-1] == os.path.sep:
return 'directory'
else:
return 'file'
if len(status.deleted) > 0 or len(status.created) > 0 or len(status.modified) > 0:
# commit changed manifest file
self.dataset.git.add_all()
self.dataset.git.commit("Commit changes to manifest file.")
ar = ActivityRecord(ActivityType.DATASET,
message="msg is set below after detail record processing...",
show=True,
importance=255,
linked_commit=self.dataset.git.commit_hash,
tags=[])
for cnt, f in enumerate(status.created):
adr = ActivityDetailRecord(ActivityDetailType.DATASET, show=False, importance=max(255 - cnt, 0),
action=ActivityAction.CREATE)
msg = f"Created new {_item_type(f)} `{f}`"
adr.add_value('text/markdown', msg)
ar.add_detail_object(adr)
for cnt, f in enumerate(status.modified):
adr = ActivityDetailRecord(ActivityDetailType.DATASET, show=False, importance=max(255 - cnt, 0),
action=ActivityAction.EDIT)
msg = f"Modified {_item_type(f)} `{f}`"
adr.add_value('text/markdown', msg)
ar.add_detail_object(adr)
for cnt, f in enumerate(status.deleted):
adr = ActivityDetailRecord(ActivityDetailType.DATASET, show=False, importance=max(255 - cnt, 0),
action=ActivityAction.DELETE)
msg = f"Deleted {_item_type(f)} `{f}`"
adr.add_value('text/markdown', msg)
ar.add_detail_object(adr)
num_files_created = sum([_item_type(x) == "file" for x in status.created])
num_files_modified = sum([_item_type(x) == "file" for x in status.modified])
num_files_deleted = sum([_item_type(x) == "file" for x in status.deleted])
upload_str = "Uploaded" if upload else ''
nmsg = f"{upload_str} {num_files_created} new file(s). " if num_files_created > 0 else ""
mmsg = f"{upload_str} {num_files_modified} modified file(s). " if num_files_modified > 0 else ""
dmsg = f"{num_files_deleted} deleted file(s). " if num_files_deleted > 0 else ""
if not nmsg and not mmsg and not dmsg:
# You didn't edit any files, only an empty directory
num_dirs_created = sum([_item_type(x) == "directory" for x in status.created])
num_dirs_modified = sum([_item_type(x) == "directory" for x in status.modified])
num_dirs_deleted = sum([_item_type(x) == "directory" for x in status.deleted])
nmsg = f"{num_dirs_created} new folder(s). " if num_dirs_created > 0 else ""
mmsg = f"{num_dirs_modified} modified folder(s). " if num_dirs_modified > 0 else ""
dmsg = f"{num_dirs_deleted} deleted folder(s). " if num_dirs_deleted > 0 else ""
ar.message = f"{extra_msg if extra_msg else ''}" \
f"{nmsg}{mmsg}{dmsg}"
ars = ActivityStore(self.dataset)
ars.create_activity_record(ar)
def remove_packages(self, package_manager: str,
package_names: List[str]) -> None:
"""Remove yaml files describing a package and its context to the labbook.
Args:
package_manager: The package manager (eg., "apt" or "pip3")
package_names: A list of packages to uninstall
Returns:
None
"""
# Create activity record
ar = ActivityRecord(
ActivityType.ENVIRONMENT,
message="",
show=True,
linked_commit="",
tags=["environment", 'package_manager', package_manager])
for pkg in package_names:
yaml_filename = '{}_{}.yaml'.format(package_manager, pkg)
package_yaml_path = os.path.join(self.env_dir, 'package_manager',
yaml_filename)
# Check for package to exist
if not os.path.exists(package_yaml_path):
raise ValueError(
f"{package_manager} installed package {pkg} does not exist."
)
# Check to make sure package isn't from the base. You cannot remove packages from the base yet.
with open(package_yaml_path, 'rt') as cf:
package_data = yaml.safe_load(cf)
if not package_data:
raise IOError("Failed to load package description")
if package_data['from_base'] is True:
raise ValueError(
"Cannot remove a package installed in the Base")
# Delete the yaml file, which on next Dockerfile gen/rebuild will remove the dependency
os.remove(package_yaml_path)
if os.path.exists(package_yaml_path):
raise ValueError(f"Failed to remove package.")
self.labbook.git.remove(package_yaml_path)
# Create detail record
adr = ActivityDetailRecord(ActivityDetailType.ENVIRONMENT,
show=False,
action=ActivityAction.DELETE)
adr.add_value('text/plain',
f"Removed {package_manager} managed package: {pkg}")
ar.add_detail_object(adr)
logger.info(f"Removed {package_manager} managed package: {pkg}")
# Add to git
short_message = f"Removed {len(package_names)} {package_manager} managed package(s)"
commit = self.labbook.git.commit(short_message)
ar.linked_commit = commit.hexsha
ar.message = short_message
# Store
ars = ActivityStore(self.labbook)
ars.create_activity_record(ar)
def add_packages(self,
package_manager: str,
packages: List[dict],
force: bool = False,
from_base: bool = False) -> None:
"""Add a new yaml file describing the new package and its context to the labbook.
Args:
package_manager: The package manager (eg., "apt" or "pip3")
packages: A dictionary of packages to install (package & version are main keys needed)
force: Force overwriting a component if it already exists (e.g. you want to update the version)
from_base: If a package in a base image, not deletable. Otherwise, can be deleted by LB user.
Returns:
None
"""
if not package_manager:
raise ValueError(
'Argument package_manager cannot be None or empty')
# Create activity record
ar = ActivityRecord(
ActivityType.ENVIRONMENT,
show=True,
message="",
linked_commit="",
tags=["environment", 'package_manager', package_manager])
update_cnt = 0
add_cnt = 0
for pkg in packages:
version_str = f'"{pkg["version"]}"' if pkg["version"] else 'latest'
yaml_lines = [
'# Generated on: {}'.format(str(datetime.datetime.now())),
'manager: "{}"'.format(package_manager),
'package: "{}"'.format(pkg["package"]),
'version: {}'.format(version_str),
f'from_base: {str(from_base).lower()}',
f'schema: {CURRENT_SCHEMA}'
]
yaml_filename = '{}_{}.yaml'.format(package_manager,
pkg["package"])
package_yaml_path = os.path.join(self.env_dir, 'package_manager',
yaml_filename)
# Check if package already exists
if os.path.exists(package_yaml_path):
if force:
# You are updating, since force is set and package already exists.
logger.warning("Updating package file at {}".format(
package_yaml_path))
detail_msg = "Update {} managed package: {} {}".format(
package_manager, pkg["package"], version_str)
adr = ActivityDetailRecord(ActivityDetailType.ENVIRONMENT,
show=False,
action=ActivityAction.EDIT)
update_cnt += 1
else:
raise ValueError(
"The package {} already exists in this LabBook.".
format(pkg["package"]) + " Use `force` to overwrite")
else:
add_cnt += 1
detail_msg = "Add {} managed package: {} {}".format(
package_manager, pkg["package"], version_str)
adr = ActivityDetailRecord(ActivityDetailType.ENVIRONMENT,
show=False,
action=ActivityAction.CREATE)
# Write the YAML to the file
with open(package_yaml_path, 'w') as package_yaml_file:
package_yaml_file.write(os.linesep.join(yaml_lines))
# Create activity record
adr.add_value('text/plain', detail_msg)
ar.add_detail_object(adr)
logger.info("Added package {} to labbook at {}".format(
pkg["package"], self.labbook.root_dir))
# Set activity message
ar_msg = ""
if add_cnt > 0:
ar_msg = f"Added {add_cnt} {package_manager} package(s). "
if update_cnt > 0:
ar_msg = f"{ar_msg}Updated {update_cnt} {package_manager} package(s)"
# Add to git
self.labbook.git.add_all(self.env_dir)
commit = self.labbook.git.commit(ar_msg)
ar.linked_commit = commit.hexsha
ar.message = ar_msg
# Store
ars = ActivityStore(self.labbook)
ars.create_activity_record(ar)