def process(self, result_obj: ActivityRecord, data: List[ExecutionData], status: Dict[str, Any], metadata: Dict[str, Any]) -> ActivityRecord: """Method to update a result object based on code and result data Args: result_obj(ActivityNote): An object containing the note data(list): A list of ExecutionData instances containing the data for this record status(dict): A dict containing the result of git status from gitlib metadata(str): A dictionary containing Dev Env specific or other developer defined data Returns: ActivityNote """ supported_image_types = [ 'image/png', 'image/jpeg', 'image/jpg', 'image/gif', 'image/bmp' ] # If a supported image exists in the result, grab it and create a detail record result_cnt = 0 for cell in data: for result_entry in reversed(cell.result): if 'data' in result_entry: for mime_type in result_entry['data']: if mime_type in supported_image_types: # You got an image adr_img = ActivityDetailRecord( ActivityDetailType.RESULT, show=True, action=ActivityAction.CREATE, importance=max(255 - result_cnt, 0)) adr_img.add_value(mime_type, result_entry['data'][mime_type]) adr_img.tags = cell.tags result_obj.add_detail_object(adr_img) # Set Activity Record Message result_obj.message = "Executed cell in notebook {} and generated a result".format( metadata['path']) result_cnt += 1 return result_obj
def sweep_uncommitted_changes(self, upload: bool = False, extra_msg: Optional[str] = None, show: bool = False) -> None: """ Sweep all changes into a commit, and create activity record. NOTE: This method MUST be called inside a lock. Args: upload(bool): Flag indicating if this was from a batch upload extra_msg(str): Optional string used to augment the activity message show(bool): Optional flag indicating if the result of this sweep is important enough to be shown in the feed Returns: """ result_status = self.git.status() if any([result_status[k] for k in result_status.keys()]): self.git.add_all() self.git.commit("Sweep of uncommitted changes") ar = ActivityRecord(self._default_activity_type, message="--overwritten--", show=show, importance=255, linked_commit=self.git.commit_hash, tags=['save']) if upload: ar.tags.append('upload') ar, newcnt, modcnt, delcnt = self.process_sweep_status( ar, result_status) nmsg = f"{newcnt} new file(s). " if newcnt > 0 else "" mmsg = f"{modcnt} modified file(s). " if modcnt > 0 else "" dmsg = f"{delcnt} deleted file(s). " if delcnt > 0 else "" message = f"{extra_msg or ''}" \ f"{'Uploaded ' if upload else ''}" \ f"{nmsg}{mmsg}{dmsg}" # This is used to handle if you try to delete an empty directory. This shouldn't technically happen, but if # a user manages to create an empty dir outside the client, we should handle it gracefully ar.message = "No detected changes" if not message else message ars = ActivityStore(self) ars.create_activity_record(ar) else: logger.info(f"{str(self)} no changes to sweep.")
def process(self, result_obj: ActivityRecord, data: List[ExecutionData], status: Dict[str, Any], metadata: Dict[str, Any]) -> ActivityRecord: """Method to update a result object based on code and result data Args: result_obj(ActivityNote): An object containing the note data(list): A list of ExecutionData instances containing the data for this record status(dict): A dict containing the result of git status from gitlib metadata(str): A dictionary containing Dev Env specific or other developer defined data Returns: ActivityRecord """ # If there was some code, assume a cell was executed result_cnt = 0 for cell_cnt, cell in enumerate(data): for result_entry in reversed(cell.code): if result_entry.get('code'): # Create detail record to capture executed code adr_code = ActivityDetailRecord( ActivityDetailType.CODE_EXECUTED, show=False, action=ActivityAction.EXECUTE, importance=max(255 - result_cnt, 0)) adr_code.add_value( 'text/markdown', f"```\n{result_entry.get('code')}\n```") adr_code.tags = cell.tags result_obj.add_detail_object(adr_code) result_cnt += 1 # Set Activity Record Message cell_str = f"{cell_cnt} cells" if cell_cnt > 1 else "cell" result_obj.message = f"Executed {cell_str} in notebook {metadata['path']}" return result_obj
def sweep_all_changes(self, upload: bool = False, extra_msg: str = None) -> None: """ Args: upload: extra_msg: Returns: """ def _item_type(key): if key[-1] == os.path.sep: return 'directory' else: return 'file' previous_revision = self.dataset_revision # Update manifest status = self.update() if len(status.deleted) > 0 or len(status.created) > 0 or len( status.modified) > 0: # commit changed manifest file self.dataset.git.add_all() self.dataset.git.commit("Commit changes to manifest file.") ar = ActivityRecord( ActivityType.DATASET, message="msg is set below after detail record processing...", show=True, importance=255, linked_commit=self.dataset.git.commit_hash, tags=[]) if upload: ar.tags.append('upload') for cnt, f in enumerate(status.created): adr = ActivityDetailRecord(ActivityDetailType.DATASET, show=False, importance=max(255 - cnt, 0), action=ActivityAction.CREATE) msg = f"Created new {_item_type(f)} `{f}`" adr.add_value('text/markdown', msg) ar.add_detail_object(adr) for cnt, f in enumerate(status.modified): adr = ActivityDetailRecord(ActivityDetailType.DATASET, show=False, importance=max(255 - cnt, 0), action=ActivityAction.EDIT) msg = f"Modified {_item_type(f)} `{f}`" adr.add_value('text/markdown', msg) ar.add_detail_object(adr) for cnt, f in enumerate(status.deleted): adr = ActivityDetailRecord(ActivityDetailType.DATASET, show=False, importance=max(255 - cnt, 0), action=ActivityAction.DELETE) msg = f"Deleted {_item_type(f)} `{f}`" adr.add_value('text/markdown', msg) ar.add_detail_object(adr) nmsg = f"{len(status.created)} new file(s). " if len( status.created) > 0 else "" mmsg = f"{len(status.modified)} modified file(s). " if len( status.modified) > 0 else "" dmsg = f"{len(status.deleted)} deleted file(s). " if len( status.deleted) > 0 else "" ar.message = f"{extra_msg if extra_msg else ''}" \ f"{'Uploaded ' if upload else ''}" \ f"{nmsg}{mmsg}{dmsg}" ars = ActivityStore(self.dataset) ars.create_activity_record(ar) # Re-link new revision, unlink old revision self.link_revision() if os.path.isdir( os.path.join(self.cache_mgr.cache_root, previous_revision)): shutil.rmtree( os.path.join(self.cache_mgr.cache_root, previous_revision))
def create_update_activity_record(self, status: StatusResult, upload: bool = False, extra_msg: str = None) -> None: """ Args: status(StatusResult): a StatusResult object after updating the manifest upload(bool): flag indicating if this is a record for an upload extra_msg(str): any extra string to add to the activity record Returns: None """ def _item_type(key): if key[-1] == os.path.sep: return 'directory' else: return 'file' if len(status.deleted) > 0 or len(status.created) > 0 or len(status.modified) > 0: # commit changed manifest file self.dataset.git.add_all() self.dataset.git.commit("Commit changes to manifest file.") ar = ActivityRecord(ActivityType.DATASET, message="msg is set below after detail record processing...", show=True, importance=255, linked_commit=self.dataset.git.commit_hash, tags=[]) for cnt, f in enumerate(status.created): adr = ActivityDetailRecord(ActivityDetailType.DATASET, show=False, importance=max(255 - cnt, 0), action=ActivityAction.CREATE) msg = f"Created new {_item_type(f)} `{f}`" adr.add_value('text/markdown', msg) ar.add_detail_object(adr) for cnt, f in enumerate(status.modified): adr = ActivityDetailRecord(ActivityDetailType.DATASET, show=False, importance=max(255 - cnt, 0), action=ActivityAction.EDIT) msg = f"Modified {_item_type(f)} `{f}`" adr.add_value('text/markdown', msg) ar.add_detail_object(adr) for cnt, f in enumerate(status.deleted): adr = ActivityDetailRecord(ActivityDetailType.DATASET, show=False, importance=max(255 - cnt, 0), action=ActivityAction.DELETE) msg = f"Deleted {_item_type(f)} `{f}`" adr.add_value('text/markdown', msg) ar.add_detail_object(adr) num_files_created = sum([_item_type(x) == "file" for x in status.created]) num_files_modified = sum([_item_type(x) == "file" for x in status.modified]) num_files_deleted = sum([_item_type(x) == "file" for x in status.deleted]) upload_str = "Uploaded" if upload else '' nmsg = f"{upload_str} {num_files_created} new file(s). " if num_files_created > 0 else "" mmsg = f"{upload_str} {num_files_modified} modified file(s). " if num_files_modified > 0 else "" dmsg = f"{num_files_deleted} deleted file(s). " if num_files_deleted > 0 else "" if not nmsg and not mmsg and not dmsg: # You didn't edit any files, only an empty directory num_dirs_created = sum([_item_type(x) == "directory" for x in status.created]) num_dirs_modified = sum([_item_type(x) == "directory" for x in status.modified]) num_dirs_deleted = sum([_item_type(x) == "directory" for x in status.deleted]) nmsg = f"{num_dirs_created} new folder(s). " if num_dirs_created > 0 else "" mmsg = f"{num_dirs_modified} modified folder(s). " if num_dirs_modified > 0 else "" dmsg = f"{num_dirs_deleted} deleted folder(s). " if num_dirs_deleted > 0 else "" ar.message = f"{extra_msg if extra_msg else ''}" \ f"{nmsg}{mmsg}{dmsg}" ars = ActivityStore(self.dataset) ars.create_activity_record(ar)
def remove_packages(self, package_manager: str, package_names: List[str]) -> None: """Remove yaml files describing a package and its context to the labbook. Args: package_manager: The package manager (eg., "apt" or "pip3") package_names: A list of packages to uninstall Returns: None """ # Create activity record ar = ActivityRecord( ActivityType.ENVIRONMENT, message="", show=True, linked_commit="", tags=["environment", 'package_manager', package_manager]) for pkg in package_names: yaml_filename = '{}_{}.yaml'.format(package_manager, pkg) package_yaml_path = os.path.join(self.env_dir, 'package_manager', yaml_filename) # Check for package to exist if not os.path.exists(package_yaml_path): raise ValueError( f"{package_manager} installed package {pkg} does not exist." ) # Check to make sure package isn't from the base. You cannot remove packages from the base yet. with open(package_yaml_path, 'rt') as cf: package_data = yaml.safe_load(cf) if not package_data: raise IOError("Failed to load package description") if package_data['from_base'] is True: raise ValueError( "Cannot remove a package installed in the Base") # Delete the yaml file, which on next Dockerfile gen/rebuild will remove the dependency os.remove(package_yaml_path) if os.path.exists(package_yaml_path): raise ValueError(f"Failed to remove package.") self.labbook.git.remove(package_yaml_path) # Create detail record adr = ActivityDetailRecord(ActivityDetailType.ENVIRONMENT, show=False, action=ActivityAction.DELETE) adr.add_value('text/plain', f"Removed {package_manager} managed package: {pkg}") ar.add_detail_object(adr) logger.info(f"Removed {package_manager} managed package: {pkg}") # Add to git short_message = f"Removed {len(package_names)} {package_manager} managed package(s)" commit = self.labbook.git.commit(short_message) ar.linked_commit = commit.hexsha ar.message = short_message # Store ars = ActivityStore(self.labbook) ars.create_activity_record(ar)
def add_packages(self, package_manager: str, packages: List[dict], force: bool = False, from_base: bool = False) -> None: """Add a new yaml file describing the new package and its context to the labbook. Args: package_manager: The package manager (eg., "apt" or "pip3") packages: A dictionary of packages to install (package & version are main keys needed) force: Force overwriting a component if it already exists (e.g. you want to update the version) from_base: If a package in a base image, not deletable. Otherwise, can be deleted by LB user. Returns: None """ if not package_manager: raise ValueError( 'Argument package_manager cannot be None or empty') # Create activity record ar = ActivityRecord( ActivityType.ENVIRONMENT, show=True, message="", linked_commit="", tags=["environment", 'package_manager', package_manager]) update_cnt = 0 add_cnt = 0 for pkg in packages: version_str = f'"{pkg["version"]}"' if pkg["version"] else 'latest' yaml_lines = [ '# Generated on: {}'.format(str(datetime.datetime.now())), 'manager: "{}"'.format(package_manager), 'package: "{}"'.format(pkg["package"]), 'version: {}'.format(version_str), f'from_base: {str(from_base).lower()}', f'schema: {CURRENT_SCHEMA}' ] yaml_filename = '{}_{}.yaml'.format(package_manager, pkg["package"]) package_yaml_path = os.path.join(self.env_dir, 'package_manager', yaml_filename) # Check if package already exists if os.path.exists(package_yaml_path): if force: # You are updating, since force is set and package already exists. logger.warning("Updating package file at {}".format( package_yaml_path)) detail_msg = "Update {} managed package: {} {}".format( package_manager, pkg["package"], version_str) adr = ActivityDetailRecord(ActivityDetailType.ENVIRONMENT, show=False, action=ActivityAction.EDIT) update_cnt += 1 else: raise ValueError( "The package {} already exists in this LabBook.". format(pkg["package"]) + " Use `force` to overwrite") else: add_cnt += 1 detail_msg = "Add {} managed package: {} {}".format( package_manager, pkg["package"], version_str) adr = ActivityDetailRecord(ActivityDetailType.ENVIRONMENT, show=False, action=ActivityAction.CREATE) # Write the YAML to the file with open(package_yaml_path, 'w') as package_yaml_file: package_yaml_file.write(os.linesep.join(yaml_lines)) # Create activity record adr.add_value('text/plain', detail_msg) ar.add_detail_object(adr) logger.info("Added package {} to labbook at {}".format( pkg["package"], self.labbook.root_dir)) # Set activity message ar_msg = "" if add_cnt > 0: ar_msg = f"Added {add_cnt} {package_manager} package(s). " if update_cnt > 0: ar_msg = f"{ar_msg}Updated {update_cnt} {package_manager} package(s)" # Add to git self.labbook.git.add_all(self.env_dir) commit = self.labbook.git.commit(ar_msg) ar.linked_commit = commit.hexsha ar.message = ar_msg # Store ars = ActivityStore(self.labbook) ars.create_activity_record(ar)