def process(self, result_obj: ActivityRecord, data: List[ExecutionData], status: Dict[str, Any], metadata: Dict[str, Any]) -> ActivityRecord: with result_obj.inspect_detail_objects() as old_details: for idx, detail in enumerate(old_details): if detail.type is ActivityDetailType.CODE_EXECUTED: code = detail.data['text/markdown'] res = self.comment_re.search(code) if res: directive = res[1] if directive == 'auto': # This is the default behavior pass elif directive in ['show', 'hide', 'ignore']: for tag in detail.tags: # currently, the 'ex1' type tags are the only ones... if tag.startswith('ex'): result_obj.modify_tag_visibility( tag, directive) else: # TODO DC Currently we don't have a mechanism to highlight any problem to users. # But this is where we could signal an unknown comment directive. Another place # would be in the jupyterlab extension. We need to think through this UX! pass return result_obj
def mutate_and_get_payload(cls, root, info, owner, dataset_name, description, client_mutation_id=None): username = get_logged_in_username() ds = InventoryManager().load_dataset(username, owner, dataset_name, author=get_logged_in_author()) ds.description = description with ds.lock(): ds.git.add(os.path.join(ds.root_dir, '.gigantum/gigantum.yaml')) commit = ds.git.commit('Updating description') adr = ActivityDetailRecord(ActivityDetailType.LABBOOK, show=False) adr.add_value('text/plain', f"Updated Dataset description: {description}") ar = ActivityRecord(ActivityType.LABBOOK, message="Updated Dataset description", linked_commit=commit.hexsha, tags=["dataset"], show=False) ar.add_detail_object(adr) ars = ActivityStore(ds) ars.create_activity_record(ar) return SetDatasetDescription( updated_dataset=Dataset(owner=owner, name=dataset_name))
def remove_docker_snippet(self, name: str) -> None: """Remove a custom docker snippet Args: name: Name or identifer of snippet to remove Returns: None """ docker_dir = os.path.join(self.labbook.root_dir, '.gigantum', 'env', 'docker') docker_file = os.path.join(docker_dir, f'{name}.yaml') if not os.path.exists(docker_file): raise ValueError(f'Docker snippet name `{name}` does not exist') self.labbook.git.remove(docker_file, keep_file=False) short_message = f"Removed custom Docker snippet `{name}`" logger.info(short_message) commit = self.labbook.git.commit(short_message) adr = ActivityDetailRecord(ActivityDetailType.ENVIRONMENT, show=False, action=ActivityAction.DELETE) adr.add_value('text/plain', short_message) ar = ActivityRecord(ActivityType.ENVIRONMENT, message=short_message, show=False, linked_commit=commit.hexsha, tags=["environment", "docker", "snippet"]) ar.add_detail_object(adr) ars = ActivityStore(self.labbook) ars.create_activity_record(ar)
def remove_bundled_app(self, name: str) -> None: """Remove a bundled app from this labbook Args: name(str): name of the bundled app Returns: None """ data = self.get_bundled_apps() if name not in data: raise ValueError(f"App {name} does not exist. Cannot remove.") del data[name] with open(self.bundled_app_file, 'wt') as baf: json.dump(data, baf) # Commit the changes self.labbook.git.add(self.bundled_app_file) commit = self.labbook.git.commit(f"Committing bundled app") adr = ActivityDetailRecord(ActivityDetailType.ENVIRONMENT, show=False, action=ActivityAction.CREATE) adr.add_value('text/plain', f"Removed bundled application: {name}") ar = ActivityRecord(ActivityType.ENVIRONMENT, message=f"Removed bundled application: {name}", show=True, linked_commit=commit.hexsha, tags=["environment", "docker", "bundled_app"]) ar.add_detail_object(adr) ars = ActivityStore(self.labbook) ars.create_activity_record(ar)
def mutate_and_get_payload(cls, root, info, owner, labbook_name, description_content, client_mutation_id=None): username = get_logged_in_username() lb = InventoryManager().load_labbook(username, owner, labbook_name, author=get_logged_in_author()) lb.description = description_content with lb.lock(): lb.git.add(os.path.join(lb.config_path)) commit = lb.git.commit('Updating description') adr = ActivityDetailRecord(ActivityDetailType.LABBOOK, show=False) adr.add_value('text/plain', "Updated description of Project") ar = ActivityRecord(ActivityType.LABBOOK, message="Updated description of Project", linked_commit=commit.hexsha, tags=["labbook"], show=False) ar.add_detail_object(adr) ars = ActivityStore(lb) ars.create_activity_record(ar) return SetLabbookDescription(success=True)
def migrate_labbook_schema(labbook: LabBook) -> None: # Fallback point in case of a problem initial_commit = labbook.git.commit_hash try: migrate_schema_to_current(labbook.root_dir) except Exception as e: logger.exception(e) call_subprocess(f'git reset --hard {initial_commit}'.split(), cwd=labbook.root_dir) raise msg = f"Migrate schema to {CURRENT_LABBOOK_SCHEMA}" labbook.git.add(labbook.config_path) cmt = labbook.git.commit(msg, author=labbook.author, committer=labbook.author) adr = ActivityDetailRecord(ActivityDetailType.LABBOOK, show=True, importance=100, action=ActivityAction.EDIT) adr.add_value('text/plain', msg) ar = ActivityRecord(ActivityType.LABBOOK, message=msg, show=True, importance=255, linked_commit=cmt.hexsha, tags=['schema', 'update', 'migration']) ar.add_detail_object(adr) ars = ActivityStore(labbook) ars.create_activity_record(ar)
def process(self, result_obj: ActivityRecord, data: List[ExecutionData], status: Dict[str, Any], metadata: Dict[str, Any]) -> ActivityRecord: """Method to update a result object based on code and result data Args: result_obj(ActivityNote): An object containing the note data(list): A list of ExecutionData instances containing the data for this record status(dict): A dict containing the result of git status from gitlib metadata(str): A dictionary containing Dev Env specific or other developer defined data Returns: ActivityNote """ with result_obj.inspect_detail_objects() as detail_objs: orig_num = result_obj.num_detail_objects if result_obj.num_detail_objects > 255: result_obj.trim_detail_objects(255) adr = ActivityDetailRecord(ActivityDetailType.NOTE, show=True, importance=0, action=ActivityAction.NOACTION) adr.add_value('text/markdown', f"This activity produced {orig_num} detail records, " f"but was truncated to the top 255 items. Inspect your code to make " f"sure that this was not accidental. In Jupyter for example, you can" f" use a `;` at the end of a line to suppress output from functions" f" that print excessively.") result_obj.add_detail_object(adr) return result_obj
def backend_config(self, data: dict) -> None: """Save storage config data""" if self._backend: self._backend.configuration = {**self._backend.configuration, **data} # Remove defaults set at runtime that shouldn't be persisted if "username" in data: del data["username"] if "gigantum_bearer_token" in data: del data["gigantum_bearer_token"] if "gigantum_id_token" in data: del data["gigantum_id_token"] config_file = os.path.join(self.root_dir, ".gigantum", "backend.json") with open(config_file, 'wt') as sf: json.dump(data, sf, indent=2) self.git.add(config_file) cm = self.git.commit("Updating backend config") ar = ActivityRecord(ActivityType.DATASET, message="Updated Dataset storage backend configuration", show=True, importance=255, linked_commit=cm.hexsha, tags=['config']) adr = ActivityDetailRecord(ActivityDetailType.DATASET, show=False, importance=255, action=ActivityAction.EDIT) d = json.dumps(data, indent=2) adr.add_value('text/markdown', f"Updated dataset storage backend configuration:\n\n ```{d}```") ar.add_detail_object(adr) ars = ActivityStore(self) ars.create_activity_record(ar)
def mutate_and_get_payload(cls, root, info, name, description, repository, base_id, revision, is_untracked=False, client_mutation_id=None): username = get_logged_in_username() inv_manager = InventoryManager() if is_untracked: lb = inv_manager.create_labbook_disabled_lfs( username=username, owner=username, labbook_name=name, description=description, author=get_logged_in_author()) else: lb = inv_manager.create_labbook(username=username, owner=username, labbook_name=name, description=description, author=get_logged_in_author()) if is_untracked: FileOperations.set_untracked(lb, 'input') FileOperations.set_untracked(lb, 'output') input_set = FileOperations.is_set_untracked(lb, 'input') output_set = FileOperations.is_set_untracked(lb, 'output') if not (input_set and output_set): raise ValueError( f'{str(lb)} untracking for input/output in malformed state' ) if not lb.is_repo_clean: raise ValueError( f'{str(lb)} should have clean Git state after setting for untracked' ) adr = ActivityDetailRecord(ActivityDetailType.LABBOOK, show=False, importance=0) adr.add_value('text/plain', f"Created new LabBook: {username}/{name}") # Create activity record ar = ActivityRecord(ActivityType.LABBOOK, message=f"Created new LabBook: {username}/{name}", show=True, importance=255, linked_commit=lb.git.commit_hash) ar.add_detail_object(adr) store = ActivityStore(lb) store.create_activity_record(ar) cm = ComponentManager(lb) cm.add_base(repository, base_id, revision) return CreateLabbook(labbook=Labbook(owner=username, name=lb.name))
def insert_file(cls, labbook: LabBook, section: str, src_file: str, dst_path: str = '') -> Dict[str, Any]: """ Move the file at `src_file` into the `dst_dir`, overwriting if a file already exists there. This calls `copy_into_container()` under- the-hood, but will create an activity record. Args: labbook: Subject labbook section: Section name (code, input, output) src_file: Full path of file to insert into dst_path: Relative path within labbook where `src_file` should be copied to Returns: dict: The inserted file's info """ finfo = FileOperations.put_file(labbook=labbook, section=section, src_file=src_file, dst_path=dst_path) rel_path = os.path.join(section, finfo['key']) # If we are setting this section to be untracked activity_type, activity_detail_type, section_str = \ labbook.get_activity_type_from_section(section) commit_msg = f"Added new {section_str} file {rel_path}" try: labbook.git.add(rel_path) commit = labbook.git.commit(commit_msg) except Exception as x: logger.error(x) os.remove(dst_path) raise FileOperationsException(x) # Create Activity record and detail _, ext = os.path.splitext(rel_path) or 'file' adr = ActivityDetailRecord(activity_detail_type, show=False, importance=0, action=ActivityAction.CREATE) adr.add_value('text/plain', commit_msg) ar = ActivityRecord(activity_type, message=commit_msg, show=True, importance=255, linked_commit=commit.hexsha, tags=[ext]) ar.add_detail_object(adr) ars = ActivityStore(labbook) ars.create_activity_record(ar) return finfo
def add_docker_snippet(self, name: str, docker_content: List[str], description: Optional[str] = None) -> None: """ Add a custom docker snippet to the environment (replacing custom dependency). Args: name: Name or identifier of the custom docker snippet docker_content: Content of the docker material (May make this a list of strings instead) description: Human-readable verbose description of what the snippet is intended to accomplish. Returns: None """ if not name: raise ValueError('Argument `name` cannot be None or empty') if not name.replace('-', '').replace('_', '').isalnum(): raise ValueError( 'Argument `name` must be alphanumeric string (- and _ accepted)' ) if not docker_content: docker_content = [] file_data = { 'name': name, 'timestamp_utc': datetime.datetime.utcnow().isoformat(), 'description': description or "", 'content': docker_content } docker_dir = os.path.join(self.labbook.root_dir, '.gigantum', 'env', 'docker') docker_file = os.path.join(docker_dir, f'{name}.yaml') os.makedirs(docker_dir, exist_ok=True) yaml_dump = yaml.safe_dump(file_data, default_flow_style=False) with open(docker_file, 'w') as df: df.write(yaml_dump) logger.info( f"Wrote custom Docker snippet `{name}` to {str(self.labbook)}") short_message = f"Wrote custom Docker snippet `{name}`" self.labbook.git.add(docker_file) commit = self.labbook.git.commit(short_message) adr = ActivityDetailRecord(ActivityDetailType.ENVIRONMENT, show=False, action=ActivityAction.CREATE) adr.add_value('text/plain', '\n'.join(docker_content)) ar = ActivityRecord(ActivityType.ENVIRONMENT, message=short_message, show=True, linked_commit=commit.hexsha, tags=["environment", "docker", "snippet"]) ar.add_detail_object(adr) ars = ActivityStore(self.labbook) ars.create_activity_record(ar)
def process(self, result_obj: ActivityRecord, data: List[ExecutionData], status: Dict[str, Any], metadata: Dict[str, Any]) -> ActivityRecord: """Method to update a result object based on code and result data Args: result_obj(ActivityNote): An object containing the note data(list): A list of ExecutionData instances containing the data for this record status(dict): A dict containing the result of git status from gitlib metadata(str): A dictionary containing Dev Env specific or other developer defined data Returns: ActivityRecord """ for cnt, filename in enumerate(status['untracked']): # skip any file in .git or .gigantum dirs if ".git" in filename or ".gigantum" in filename: continue activity_type, activity_detail_type, section = LabBook.infer_section_from_relative_path(filename) adr = ActivityDetailRecord(activity_detail_type, show=False, importance=max(255-cnt, 0), action=ActivityAction.CREATE) # We use a "private" attribute here, but it's better than the silent breakage that happened before # cf. https://github.com/gigantum/gigantum-client/issues/436 if section == LabBook._default_activity_section: msg = f'Created new file `{filename}` in the Project Root. Note, it is best practice to use the Code, ' \ 'Input, and Output sections exclusively. ' else: msg = f"Created new {section} file `{filename}`" adr.add_value('text/markdown', msg) result_obj.add_detail_object(adr) cnt = 0 for filename, change in status['unstaged']: # skip any file in .git or .gigantum dirs if ".git" in filename or ".gigantum" in filename: continue activity_type, activity_detail_type, section = LabBook.infer_section_from_relative_path(filename) if change == "deleted": action = ActivityAction.DELETE elif change == "added": action = ActivityAction.CREATE elif change == "modified": action = ActivityAction.EDIT elif change == "renamed": action = ActivityAction.EDIT else: action = ActivityAction.NOACTION adr = ActivityDetailRecord(activity_detail_type, show=False, importance=max(255-cnt, 0), action=action) adr.add_value('text/markdown', f"{change[0].upper() + change[1:]} {section} file `{filename}`") result_obj.add_detail_object(adr) cnt += 1 return result_obj
def create_directory(self, path: str) -> Dict[str, Any]: """Method to create an empty directory in a dataset Args: path: Relative path to the directory Returns: dict """ relative_path = self.dataset.make_path_relative(path) new_directory_path = os.path.join(self.cache_mgr.cache_root, self.dataset_revision, relative_path) previous_revision = self.dataset_revision if os.path.exists(new_directory_path): raise ValueError(f"Directory already exists: `{relative_path}`") else: logger.info(f"Creating new empty directory in `{new_directory_path}`") if os.path.isdir(Path(new_directory_path).parent) is False: raise ValueError(f"Parent directory does not exist. Failed to create `{new_directory_path}` ") # create dir os.makedirs(new_directory_path) self.update() if relative_path not in self.manifest: raise ValueError("Failed to add directory to manifest") # Create detail record adr = ActivityDetailRecord(ActivityDetailType.DATASET, show=False, importance=0, action=ActivityAction.CREATE) msg = f"Created new empty directory `{relative_path}`" adr.add_value('text/markdown', msg) commit = self.dataset.git.commit(msg) # Create activity record ar = ActivityRecord(ActivityType.DATASET, message=msg, linked_commit=commit.hexsha, show=True, importance=255, tags=['directory-create']) ar.add_detail_object(adr) # Store ars = ActivityStore(self.dataset) ars.create_activity_record(ar) # Relink after the commit self.link_revision() if os.path.isdir(os.path.join(self.cache_mgr.cache_root, previous_revision)): shutil.rmtree(os.path.join(self.cache_mgr.cache_root, previous_revision)) return self.gen_file_info(relative_path)
def process(self, result_obj: ActivityRecord, data: List[ExecutionData], status: Dict[str, Any], metadata: Dict[str, Any]) -> ActivityRecord: """Method to update a result object based on code and result data Args: result_obj(ActivityNote): An object containing the note data(list): A list of ExecutionData instances containing the data for this record status(dict): A dict containing the result of git status from gitlib metadata(str): A dictionary containing Dev Env specific or other developer defined data Returns: ActivityNote """ # Only store up to 64kB of plain text result data (if the user printed a TON don't save it all) truncate_at = 64 * 1000 max_show_len = 280 result_cnt = 0 for cell in data: for result_entry in reversed(cell.result): if 'metadata' in result_entry: if 'source' in result_entry['metadata']: if result_entry['metadata'][ 'source'] == "display_data": # Don't save plain-text representations of displayed data by default. continue if 'data' in result_entry: if 'text/plain' in result_entry['data']: text_data = result_entry['data']['text/plain'] if len(text_data) > 0: adr = ActivityDetailRecord( ActivityDetailType.RESULT, show=True if len(text_data) < max_show_len else False, action=ActivityAction.CREATE, importance=max(255 - result_cnt - 100, 0)) if len(text_data) <= truncate_at: adr.add_value("text/plain", text_data) else: adr.add_value( "text/plain", text_data[:truncate_at] + " ...\n\n <result truncated>") # Set cell data to tag adr.tags = cell.tags result_obj.add_detail_object(adr) result_cnt += 1 return result_obj
def write_readme(self, contents: str) -> None: """Method to write a string to the readme file within the repository. Must write ENTIRE document at once. Args: contents(str): entire readme document in markdown format Returns: None """ # Validate readme data if len(contents) > (1000000 * 5): raise ValueError("Readme file is larger than the 5MB limit") if type(contents) is not str: raise TypeError("Invalid content. Must provide string") readme_file = os.path.join(self.root_dir, 'README.md') readme_exists = os.path.exists(readme_file) # Write file to disk with open(readme_file, 'wt') as rf: rf.write(contents) # Create commit if readme_exists: commit_msg = f"Updated README file" action = ActivityAction.EDIT else: commit_msg = f"Added README file" action = ActivityAction.CREATE self.git.add(readme_file) commit = self.git.commit(commit_msg) # Create detail record adr = ActivityDetailRecord(self._default_activity_detail_type, show=False, importance=0, action=action) adr.add_value('text/plain', commit_msg) # Create activity record ar = ActivityRecord(self._default_activity_type, message=commit_msg, show=False, importance=255, linked_commit=commit.hexsha, tags=['readme']) ar.add_detail_object(adr) # Store ars = ActivityStore(self) ars.create_activity_record(ar)
def unlink_dataset_from_labbook(self, dataset_namespace: str, dataset_name: str, labbook: LabBook) -> None: """Method to removed a dataset reference from a labbook Args: dataset_namespace: dataset_name: labbook: Returns: """ submodule_dir = os.path.join('.gigantum', 'datasets', dataset_namespace, dataset_name) call_subprocess(['git', 'rm', '-f', submodule_dir], cwd=labbook.root_dir) git_module_dir = os.path.join(labbook.root_dir, '.git', 'modules', f"{dataset_namespace}&{dataset_name}") if os.path.exists(git_module_dir): shutil.rmtree(git_module_dir) absolute_submodule_dir = os.path.join(labbook.root_dir, '.gigantum', 'datasets', dataset_namespace, dataset_name) if os.path.exists(absolute_submodule_dir): shutil.rmtree(absolute_submodule_dir) labbook.git.add_all() commit = labbook.git.commit("removing submodule ref") # Add Activity Record adr = ActivityDetailRecord(ActivityDetailType.DATASET, show=False, action=ActivityAction.DELETE) adr.add_value( 'text/markdown', f"Unlinked Dataset `{dataset_namespace}/{dataset_name}` from project" ) ar = ActivityRecord( ActivityType.DATASET, message= f"Unlinked Dataset {dataset_namespace}/{dataset_name} from project.", linked_commit=commit.hexsha, tags=["dataset"], show=True) ar.add_detail_object(adr) ars = ActivityStore(labbook) ars.create_activity_record(ar)
def update_linked_dataset_reference(self, dataset_namespace: str, dataset_name: str, labbook: LabBook) -> Dataset: """Method to update a linked dataset reference to the latest revision Args: dataset_namespace: owner (namespace) of the dateset dataset_name: name of the dataset labbook: labbook instance to which the dataset is linked Returns: none1 """ # Load dataset from inside Project directory submodule_dir = os.path.join(labbook.root_dir, '.gigantum', 'datasets', dataset_namespace, dataset_name) ds = self.load_dataset_from_directory(submodule_dir, author=labbook.author) ds.namespace = dataset_namespace # Update the submodule reference with the latest changes original_revision = ds.git.repo.head.object.hexsha ds.git.pull() revision = ds.git.repo.head.object.hexsha # If the submodule has changed, commit the changes. if original_revision != revision: labbook.git.add_all() commit = labbook.git.commit("Updating submodule ref") # Add Activity Record adr = ActivityDetailRecord(ActivityDetailType.DATASET, show=False, action=ActivityAction.DELETE) adr.add_value( 'text/markdown', f"Updated Dataset `{dataset_namespace}/{dataset_name}` link to {revision}" ) msg = f"Updated Dataset `{dataset_namespace}/{dataset_name}` link to version {revision[0:8]}" ar = ActivityRecord(ActivityType.DATASET, message=msg, linked_commit=commit.hexsha, tags=["dataset"], show=True) ar.add_detail_object(adr) ars = ActivityStore(labbook) ars.create_activity_record(ar) return ds
def _create_user_note(cls, lb, title, body, tags): store = ActivityStore(lb) adr = ActivityDetailRecord(ActivityDetailType.NOTE, show=True, importance=255) if body: adr.add_value('text/markdown', body) ar = ActivityRecord(ActivityType.NOTE, message=title, linked_commit="no-linked-commit", importance=255, tags=tags) ar.add_detail_object(adr) ar = store.create_activity_record(ar) return ar
def helper_gen_record(): return ActivityRecord(ActivityType.LABBOOK, message="--overwritten--", show=False, importance=255, linked_commit="", tags=['save'])
def _update_branch_description(cls, lb: LabBook, description: str): # Update the description on branch creation lb.description = description lb.git.add(lb.config_path) commit = lb.git.commit('Updating description') adr = ActivityDetailRecord(ActivityDetailType.LABBOOK, show=False) adr.add_value('text/plain', description) ar = ActivityRecord(ActivityType.LABBOOK, message="Updated description of Project", linked_commit=commit.hexsha, tags=["labbook"], show=False) ar.add_detail_object(adr) ars = ActivityStore(lb) ars.create_activity_record(ar)
def process(self, result_obj: ActivityRecord, data: List[ExecutionData], status: Dict[str, Any], metadata: Dict[str, Any]) -> ActivityRecord: """Method to update a result object based on code and result data Args: result_obj(ActivityNote): An object containing the note data(list): A list of ExecutionData instances containing the data for this record status(dict): A dict containing the result of git status from gitlib metadata(str): A dictionary containing Dev Env specific or other developer defined data Returns: ActivityNote """ supported_image_types = [ 'image/png', 'image/jpeg', 'image/jpg', 'image/gif', 'image/bmp' ] # If a supported image exists in the result, grab it and create a detail record result_cnt = 0 for cell in data: for result_entry in reversed(cell.result): if 'data' in result_entry: for mime_type in result_entry['data']: if mime_type in supported_image_types: # You got an image adr_img = ActivityDetailRecord( ActivityDetailType.RESULT, show=True, action=ActivityAction.CREATE, importance=max(255 - result_cnt, 0)) adr_img.add_value(mime_type, result_entry['data'][mime_type]) adr_img.tags = cell.tags result_obj.add_detail_object(adr_img) # Set Activity Record Message result_obj.message = "Executed cell in notebook {} and generated a result".format( metadata['path']) result_cnt += 1 return result_obj
def sweep_uncommitted_changes(self, upload: bool = False, extra_msg: Optional[str] = None, show: bool = False) -> None: """ Sweep all changes into a commit, and create activity record. NOTE: This method MUST be called inside a lock. Args: upload(bool): Flag indicating if this was from a batch upload extra_msg(str): Optional string used to augment the activity message show(bool): Optional flag indicating if the result of this sweep is important enough to be shown in the feed Returns: """ result_status = self.git.status() if any([result_status[k] for k in result_status.keys()]): self.git.add_all() self.git.commit("Sweep of uncommitted changes") ar = ActivityRecord(self._default_activity_type, message="--overwritten--", show=show, importance=255, linked_commit=self.git.commit_hash, tags=['save']) if upload: ar.tags.append('upload') ar, newcnt, modcnt, delcnt = self.process_sweep_status( ar, result_status) nmsg = f"{newcnt} new file(s). " if newcnt > 0 else "" mmsg = f"{modcnt} modified file(s). " if modcnt > 0 else "" dmsg = f"{delcnt} deleted file(s). " if delcnt > 0 else "" message = f"{extra_msg or ''}" \ f"{'Uploaded ' if upload else ''}" \ f"{nmsg}{mmsg}{dmsg}" # This is used to handle if you try to delete an empty directory. This shouldn't technically happen, but if # a user manages to create an empty dir outside the client, we should handle it gracefully ar.message = "No detected changes" if not message else message ars = ActivityStore(self) ars.create_activity_record(ar) else: logger.info(f"{str(self)} no changes to sweep.")
def _record_remove_activity(cls, secret_store, filename, lb): """Make an activity record for the removal of the secret. """ lb.git.add(secret_store.secret_path) lb.git.commit("Removed entry from secrets registry.") commit = lb.git.commit_hash adr = ActivityDetailRecord(ActivityDetailType.LABBOOK, show=True, action=ActivityAction.DELETE) adr.add_value('text/markdown', f"Removed entry for secrets file {filename}") ar = ActivityRecord( ActivityType.LABBOOK, message=f"Removed entry for secrets file {filename}", linked_commit=commit, tags=["labbook", "secrets"], show=True) ar.add_detail_object(adr) ars = ActivityStore(lb) ars.create_activity_record(ar)
def _record_insert_activity(cls, secret_store, filename, lb, mount_path): """Make an activity record for the insertion of the secret. """ lb.git.add(secret_store.secret_path) lb.git.commit("Updated secrets registry.") commit = lb.git.commit_hash adr = ActivityDetailRecord(ActivityDetailType.LABBOOK, show=True, action=ActivityAction.CREATE) adr.add_value( 'text/markdown', f"Created new entry for secrets file {filename}" f"to map to {mount_path}") ar = ActivityRecord( ActivityType.LABBOOK, message=f"Created entry for secrets file {filename}", linked_commit=commit, tags=["labbook", "secrets"], show=True) ar.add_detail_object(adr) ars = ActivityStore(lb) ars.create_activity_record(ar)
def process(self, result_obj: ActivityRecord, data: List[ExecutionData], status: Dict[str, Any], metadata: Dict[str, Any]) -> ActivityRecord: """Method to update a result object based on code and result data Args: result_obj(ActivityNote): An object containing the note data(list): A list of ExecutionData instances containing the data for this record status(dict): A dict containing the result of git status from gitlib metadata(str): A dictionary containing Dev Env specific or other developer defined data Returns: ActivityNote """ result_obj.show = False with result_obj.inspect_detail_objects() as details: for detail in details: if detail.show: result_obj.show = True break return result_obj
def process(self, result_obj: ActivityRecord, data: List[ExecutionData], status: Dict[str, Any], metadata: Dict[str, Any]) -> ActivityRecord: """Method to update a result object based on code and result data Args: result_obj(ActivityNote): An object containing the note data(list): A list of ExecutionData instances containing the data for this record status(dict): A dict containing the result of git status from gitlib metadata(str): A dictionary containing Dev Env specific or other developer defined data Returns: ActivityRecord """ # If there was some code, assume a cell was executed result_cnt = 0 for cell_cnt, cell in enumerate(data): for result_entry in reversed(cell.code): if result_entry.get('code'): # Create detail record to capture executed code adr_code = ActivityDetailRecord( ActivityDetailType.CODE_EXECUTED, show=False, action=ActivityAction.EXECUTE, importance=max(255 - result_cnt, 0)) adr_code.add_value( 'text/markdown', f"```\n{result_entry.get('code')}\n```") adr_code.tags = cell.tags result_obj.add_detail_object(adr_code) result_cnt += 1 # Set Activity Record Message cell_str = f"{cell_cnt} cells" if cell_cnt > 1 else "cell" result_obj.message = f"Executed {cell_str} in notebook {metadata['path']}" return result_obj
def mutate_and_get_payload(cls, root, info, name, description, repository, base_id, revision, is_untracked=False, client_mutation_id=None): username = get_logged_in_username() inv_manager = InventoryManager() lb = inv_manager.create_labbook(username=username, owner=username, labbook_name=name, description=description, author=get_logged_in_author()) adr = ActivityDetailRecord(ActivityDetailType.LABBOOK, show=False, importance=0) adr.add_value('text/plain', f"Created new LabBook: {username}/{name}") # Create activity record ar = ActivityRecord(ActivityType.LABBOOK, message=f"Created new LabBook: {username}/{name}", show=True, importance=255, linked_commit=lb.git.commit_hash) ar.add_detail_object(adr) store = ActivityStore(lb) store.create_activity_record(ar) cm = ComponentManager(lb) cm.add_base(repository, base_id, revision) return CreateLabbook(labbook=Labbook(owner=username, name=lb.name))
def _make_move_activity_record(cls, labbook: LabBook, section: str, dst_abs_path: str, commit_msg: str) -> None: if os.path.isdir(dst_abs_path): labbook.git.add_all(dst_abs_path) else: labbook.git.add(dst_abs_path) commit = labbook.git.commit(commit_msg) activity_type, activity_detail_type, section_str = labbook.get_activity_type_from_section( section) adr = ActivityDetailRecord(activity_detail_type, show=False, importance=0, action=ActivityAction.EDIT) adr.add_value('text/markdown', commit_msg) ar = ActivityRecord(activity_type, message=commit_msg, linked_commit=commit.hexsha, show=True, importance=255, tags=['file-move']) ar.add_detail_object(adr) ars = ActivityStore(labbook) ars.create_activity_record(ar)
def store_activity_record( self, linked_commit: str, activity_record: ActivityRecord) -> Optional[str]: """Method to commit changes to a file Args: linked_commit(str): Git commit this ActivityRecord is related to activity_record(ActivityNote): The populated ActivityNote object returned by the processing pipeline Returns: str """ activity_record.linked_commit = linked_commit # Create a activity record record = self.activity_store.create_activity_record(activity_record) return record.commit
def process(self, activity_type: ActivityType, data: List[ExecutionData], metadata: Dict[str, Any]) -> ActivityRecord: """Method to update the result ActivityRecord object based on code and result data Args: activity_type(ActivityType): A ActivityType object indicating the activity type data(list): A list of ExecutionData instances containing the data for this record metadata(str): A dictionary containing Dev Env specific or other developer defined data Returns: ActivityRecord """ # Initialize empty record activity_record = ActivityRecord(activity_type=activity_type) # Get git status for tracking file changes status = self.labbook.git.status() # Run processors to populate the record for p in self.processors: activity_record = p.process(activity_record, data, status, metadata) return activity_record