def makedir(cls, labbook: LabBook, relative_path: str, make_parents: bool = True, create_activity_record: bool = False) -> None: """Make a new directory inside the labbook directory. Args: labbook: Subject LabBook relative_path(str): Path within the labbook to make directory make_parents(bool): If true, create intermediary directories create_activity_record(bool): If true, create commit and activity record Returns: str: Absolute path of new directory """ if not relative_path: raise ValueError("relative_path argument cannot be None or empty") relative_path = LabBook.make_path_relative(relative_path) new_directory_path = os.path.join(labbook.root_dir, relative_path) section = relative_path.split(os.sep)[0] git_untracked = in_untracked(labbook.root_dir, section) if os.path.exists(new_directory_path): return else: logger.info(f"Making new directory in `{new_directory_path}`") os.makedirs(new_directory_path, exist_ok=make_parents) if git_untracked: logger.warning( f'New {str(labbook)} untracked directory `{new_directory_path}`' ) return new_dir = '' for d in relative_path.split(os.sep): new_dir = os.path.join(new_dir, d) full_new_dir = os.path.join(labbook.root_dir, new_dir) gitkeep_path = os.path.join(full_new_dir, '.gitkeep') if not os.path.exists(gitkeep_path): with open(gitkeep_path, 'w') as gitkeep: gitkeep.write( "This file is necessary to keep this directory tracked by Git" " and archivable by compression tools. Do not delete or modify!" ) labbook.git.add(gitkeep_path) if create_activity_record: # Create detail record activity_type, activity_detail_type, section_str = labbook.infer_section_from_relative_path( relative_path) adr = ActivityDetailRecord(activity_detail_type, show=False, importance=0, action=ActivityAction.CREATE) msg = f"Created new {section_str} directory `{relative_path}`" commit = labbook.git.commit(msg) adr.add_value('text/markdown', msg) # Create activity record ar = ActivityRecord(activity_type, message=msg, linked_commit=commit.hexsha, show=True, importance=255, tags=['directory-create']) ar.add_detail_object(adr) # Store ars = ActivityStore(labbook) ars.create_activity_record(ar)
def add_bundled_app(self, port: int, name: str, description: str, command: Optional[str] = None) -> Dict[str, Any]: """Add a "bundled app" configuration to this labbook Args: port(int): port number to expose from the container (will be routed to the browser) name(str): name of the bundled app description(str): description of the bundled app command(str): command to run in the container if needed to start the app Returns: dict """ # Check if a reserved application name, currently: if name.lower() in self.reserved_names: raise ValueError( f"{name} is a reserved application name. Try again.") if len(name) > 10 or len(name) < 1: raise ValueError(f"{name} must be 10 characters or less.") if len(description) > 240: raise ValueError(f"{description} must be 240 characters or less.") if command: if len(command) > 1024: raise ValueError(f"{command} must be 1024 characters or less.") # Check if a reserved port currently if port in self.reserved_ports: raise ValueError(f"Port {port} is a in reserved port. Try again.") data = self.get_bundled_apps() # Check for port already in use for app in data: if data[app].get('port') == port: raise ValueError(f"Port {port} is already in use. Try again.") data[name] = { 'port': port, 'description': description, 'command': command } with open(self.bundled_app_file, 'wt') as baf: json.dump(data, baf) # Commit the changes self.labbook.git.add(self.bundled_app_file) commit = self.labbook.git.commit(f"Committing bundled app") adr = ActivityDetailRecord(ActivityDetailType.ENVIRONMENT, show=False, action=ActivityAction.CREATE) adr.add_value('text/plain', f"Added bundled application: {json.dumps(data[name])}") ar = ActivityRecord(ActivityType.ENVIRONMENT, message=f"Added bundled app '{name}'", show=True, linked_commit=commit.hexsha, tags=["environment", "docker", "bundled_app"]) ar.add_detail_object(adr) ars = ActivityStore(self.labbook) ars.create_activity_record(ar) return data
def test_multiple_cells(self, redis_client, mock_labbook, mock_kernel): """Test processing notebook activity""" dummy_file = os.path.join(mock_labbook[2].root_dir, 'code', 'Test.ipynb') with open(dummy_file, 'wt') as tf: tf.write("Dummy file") monitor_key = "dev_env_monitor:{}:{}:{}:{}:activity_monitor:{}".format( 'test', 'test', 'labbook1', 'jupyterlab-ubuntu1604', uuid.uuid4()) monitor = JupyterLabNotebookMonitor("test", "test", mock_labbook[2].name, monitor_key, config_file=mock_labbook[0]) # Setup monitoring metadata metadata = { "kernel_id": "XXXX", "kernel_name": 'python', "kernel_type": 'notebook', "path": 'code/Test.ipynb' } # Perform an action mock_kernel[0].execute("print('Hello, World')") mock_kernel[0].execute("print('Cell number 2!')") # Check lab book repo state status = mock_labbook[2].git.status() assert len(status["untracked"]) == 1 assert status["untracked"][0] == 'code/Test.ipynb' # Process messages msg1a = mock_kernel[0].get_iopub_msg() msg2a = mock_kernel[0].get_iopub_msg() msg3a = mock_kernel[0].get_iopub_msg() msg4a = mock_kernel[0].get_iopub_msg() msg1b = mock_kernel[0].get_iopub_msg() msg2b = mock_kernel[0].get_iopub_msg() msg3b = mock_kernel[0].get_iopub_msg() msg4b = mock_kernel[0].get_iopub_msg() # Process first state change message assert monitor.kernel_status == 'idle' assert monitor.can_store_activity_record is False monitor.handle_message(msg1a) assert monitor.kernel_status == 'busy' # Process input message monitor.handle_message(msg2a) assert len(monitor.current_cell.code) > 0 assert len(monitor.cell_data) == 0 assert monitor.can_store_activity_record is False # Process output message monitor.handle_message(msg3a) assert len(monitor.current_cell.result) > 0 assert len(monitor.cell_data) == 0 assert monitor.can_store_activity_record is False # Process final state change message monitor.handle_message(msg4a) assert monitor.kernel_status == 'idle' assert monitor.can_store_activity_record is True assert len(monitor.cell_data) == 1 # Process first state change message assert monitor.kernel_status == 'idle' monitor.handle_message(msg1b) assert monitor.can_store_activity_record is False assert monitor.kernel_status == 'busy' # Process input message monitor.handle_message(msg2b) assert len(monitor.current_cell.code) > 0 assert len(monitor.cell_data) == 1 assert monitor.can_store_activity_record is False # Process output message monitor.handle_message(msg3b) assert len(monitor.current_cell.result) > 0 assert len(monitor.cell_data) == 1 assert monitor.can_store_activity_record is False # Process final state change message monitor.handle_message(msg4b) assert monitor.kernel_status == 'idle' assert monitor.can_store_activity_record is True assert len(monitor.cell_data) == 2 # Store the record manually for this test monitor.store_record(metadata) assert monitor.can_store_activity_record is False assert len(monitor.cell_data) == 0 # Check lab book repo state status = mock_labbook[2].git.status() assert len(status["untracked"]) == 0 assert len(status["staged"]) == 0 assert len(status["unstaged"]) == 0 # Check activity entry log = mock_labbook[2].git.log() assert len(log) == 4 assert 'code/Test.ipynb' in log[0]['message'] a_store = ActivityStore(mock_labbook[2]) record = a_store.get_activity_record(log[0]['commit']) assert record.type == ActivityType.CODE assert record.show is True assert record.importance == 0 assert not record.tags assert record.message == 'Executed cell in notebook code/Test.ipynb' assert len(record._detail_objects) == 5 assert record._detail_objects[0][0] is True assert record._detail_objects[0][1] == ActivityDetailType.RESULT.value assert record._detail_objects[0][2] == 155 assert record._detail_objects[1][0] is True assert record._detail_objects[1][1] == ActivityDetailType.RESULT.value assert record._detail_objects[1][2] == 154 assert record._detail_objects[2][0] is False assert record._detail_objects[2][1] == ActivityDetailType.CODE.value assert record._detail_objects[2][2] == 255 assert record._detail_objects[3][0] is False assert record._detail_objects[3][ 1] == ActivityDetailType.CODE_EXECUTED.value assert record._detail_objects[3][2] == 255 assert record._detail_objects[4][0] is False assert record._detail_objects[4][ 1] == ActivityDetailType.CODE_EXECUTED.value assert record._detail_objects[4][2] == 254
def insert_file(cls, labbook: LabBook, section: str, src_file: str, dst_path: str = '') -> Dict[str, Any]: """ Move the file at `src_file` into the `dst_dir`, overwriting if a file already exists there. This calls `put_file()` under- the-hood, but will create an activity record. Args: labbook: Subject labbook section: Section name (code, input, output) src_file: Full path of file to insert into dst_path: Relative path within labbook where `src_file` should be copied to Returns: dict: The inserted file's info """ finfo = FileOperations.put_file(labbook=labbook, section=section, src_file=src_file, dst_path=dst_path) rel_path = os.path.join(section, finfo['key']) if in_untracked(labbook.root_dir, section): logger.warning(f"Inserted file {rel_path} ({finfo['size']} bytes)" f" to untracked section {section}. This will not" f" be tracked by commits or activity records.") return finfo # If we are setting this section to be untracked activity_type, activity_detail_type, section_str = \ labbook.get_activity_type_from_section(section) commit_msg = f"Added new {section_str} file {rel_path}" try: labbook.git.add(rel_path) commit = labbook.git.commit(commit_msg) except Exception as x: logger.error(x) os.remove(dst_path) raise FileOperationsException(x) # Create Activity record and detail _, ext = os.path.splitext(rel_path) or 'file' adr = ActivityDetailRecord(activity_detail_type, show=False, importance=0, action=ActivityAction.CREATE) adr.add_value('text/plain', commit_msg) ar = ActivityRecord(activity_type, message=commit_msg, show=True, importance=255, linked_commit=commit.hexsha, tags=[ext]) ar.add_detail_object(adr) ars = ActivityStore(labbook) ars.create_activity_record(ar) return finfo
def test_no_show(self, redis_client, mock_labbook, mock_kernel): """Test processing notebook activity that doesn't have any important detail items""" monitor_key = "dev_env_monitor:{}:{}:{}:{}:activity_monitor:{}".format( 'test', 'test', 'labbook1', 'jupyterlab-ubuntu1604', uuid.uuid4()) monitor = JupyterLabNotebookMonitor("test", "test", mock_labbook[2].name, monitor_key, config_file=mock_labbook[0]) # Setup monitoring metadata metadata = { "kernel_id": "XXXX", "kernel_name": 'python', "kernel_type": 'notebook', "path": 'code/Test.ipynb' } # Perform an action mock_kernel[0].execute("a=1") # Check lab book repo state status = mock_labbook[2].git.status() assert len(status["untracked"]) == 0 # Process messages msg1 = mock_kernel[0].get_iopub_msg() msg2 = mock_kernel[0].get_iopub_msg() msg3 = mock_kernel[0].get_iopub_msg() # Process first state change message assert monitor.kernel_status == 'idle' assert monitor.can_store_activity_record is False monitor.handle_message(msg1) assert monitor.kernel_status == 'busy' # Process input message monitor.handle_message(msg2) assert len(monitor.current_cell.code) > 0 assert len(monitor.cell_data) == 0 assert monitor.can_store_activity_record is False # Process output message monitor.handle_message(msg3) assert monitor.kernel_status == 'idle' assert monitor.can_store_activity_record is True assert len(monitor.cell_data) == 1 # Store the record manually for this test monitor.store_record(metadata) assert monitor.can_store_activity_record is False assert len(monitor.cell_data) == 0 # Check activity entry log = mock_labbook[2].git.log() assert len(log) == 4 assert 'code/Test.ipynb' in log[0]['message'] a_store = ActivityStore(mock_labbook[2]) record = a_store.get_activity_record(log[0]['commit']) assert record.type == ActivityType.CODE assert record.show is False assert record.importance == 0 assert not record.tags assert record.message == 'Executed cell in notebook code/Test.ipynb' assert len(record._detail_objects) == 1 assert record._detail_objects[0][0] is False assert record._detail_objects[0][ 1] == ActivityDetailType.CODE_EXECUTED.value assert record._detail_objects[0][2] == 255
def test_add_many_files(self, redis_client, mock_labbook, mock_kernel): """Test processing notebook activity when lots of output files have been created""" for file_number in range(0, 260): with open( os.path.join(mock_labbook[2].root_dir, 'output', f"{file_number}.dat"), 'wt') as tf: tf.write("blah") monitor_key = "dev_env_monitor:{}:{}:{}:{}:activity_monitor:{}".format( 'test', 'test', 'labbook1', 'jupyterlab-ubuntu1604', uuid.uuid4()) monitor = JupyterLabNotebookMonitor("test", "test", mock_labbook[2].name, monitor_key, config_file=mock_labbook[0]) # Setup monitoring metadata metadata = { "kernel_id": "XXXX", "kernel_name": 'python', "kernel_type": 'notebook', "path": 'code/Test.ipynb' } # Perform an action mock_kernel[0].execute("print('Generated 260 output files')") # Check lab book repo state status = mock_labbook[2].git.status() assert len(status["untracked"]) == 260 # Process messages msg1 = mock_kernel[0].get_iopub_msg() msg2 = mock_kernel[0].get_iopub_msg() msg3 = mock_kernel[0].get_iopub_msg() msg4 = mock_kernel[0].get_iopub_msg() # Process first state change message assert monitor.kernel_status == 'idle' assert monitor.can_store_activity_record is False monitor.handle_message(msg1) assert monitor.kernel_status == 'busy' # Process input message monitor.handle_message(msg2) assert len(monitor.current_cell.code) > 0 assert len(monitor.cell_data) == 0 assert monitor.can_store_activity_record is False # Process output message monitor.handle_message(msg3) assert len(monitor.current_cell.result) > 0 assert len(monitor.cell_data) == 0 assert monitor.can_store_activity_record is False # Process final state change message monitor.handle_message(msg4) assert monitor.kernel_status == 'idle' assert monitor.can_store_activity_record is True assert len(monitor.cell_data) == 1 # Store the record manually for this test monitor.store_record(metadata) assert monitor.can_store_activity_record is False assert len(monitor.cell_data) == 0 # Check lab book repo state status = mock_labbook[2].git.status() assert len(status["untracked"]) == 0 assert len(status["staged"]) == 0 assert len(status["unstaged"]) == 0 # Check activity entry log = mock_labbook[2].git.log() assert len(log) == 4 assert 'code/Test.ipynb' in log[0]['message'] a_store = ActivityStore(mock_labbook[2]) record = a_store.get_activity_record(log[0]['commit']) assert record.type == ActivityType.CODE assert record.show is True assert record.importance == 0 assert not record.tags assert record.message == 'Executed cell in notebook code/Test.ipynb' assert len(record._detail_objects) == 256 assert record.num_detail_objects == 256 assert record._detail_objects[0][0] is True assert record._detail_objects[0][1] == ActivityDetailType.NOTE.value assert record._detail_objects[0][2] == 0 assert record._detail_objects[1][0] is True assert record._detail_objects[1][1] == ActivityDetailType.RESULT.value assert record._detail_objects[1][2] == 155 assert record._detail_objects[2][0] is False assert record._detail_objects[2][ 1] == ActivityDetailType.CODE_EXECUTED.value assert record._detail_objects[2][2] == 255 assert record._detail_objects[3][0] is False assert record._detail_objects[3][ 1] == ActivityDetailType.OUTPUT_DATA.value assert record._detail_objects[3][2] == 255 assert record._detail_objects[4][0] is False assert record._detail_objects[4][ 1] == ActivityDetailType.OUTPUT_DATA.value assert record._detail_objects[4][2] == 254 assert record._detail_objects[48][0] is False assert record._detail_objects[48][ 1] == ActivityDetailType.OUTPUT_DATA.value assert record._detail_objects[48][2] == 210 assert record._detail_objects[254][0] is False assert record._detail_objects[254][ 1] == ActivityDetailType.OUTPUT_DATA.value assert record._detail_objects[254][2] == 4 assert record._detail_objects[255][0] is False assert record._detail_objects[255][ 1] == ActivityDetailType.OUTPUT_DATA.value assert record._detail_objects[255][2] == 3
def test_start_modify(self, redis_client, mock_labbook, mock_kernel): """Test processing notebook activity and have it modify an existing file & create some files""" dummy_file = os.path.join(mock_labbook[2].root_dir, 'code', 'Test.ipynb') dummy_output = os.path.join(mock_labbook[2].root_dir, 'output', 'result.bin') with open(dummy_file, 'wt') as tf: tf.write("Dummy file") monitor_key = "dev_env_monitor:{}:{}:{}:{}:activity_monitor:{}".format( 'test', 'test', 'labbook1', 'jupyterlab-ubuntu1604', uuid.uuid4()) monitor = JupyterLabNotebookMonitor("test", "test", mock_labbook[2].name, monitor_key, config_file=mock_labbook[0]) # Setup monitoring metadata metadata = { "kernel_id": "XXXX", "kernel_name": 'python', "kernel_type": 'notebook', "path": 'code/Test.ipynb' } # Perform an action mock_kernel[0].execute("print('Hello, World')") # Check lab book repo state status = mock_labbook[2].git.status() assert len(status["untracked"]) == 1 assert status["untracked"][0] == 'code/Test.ipynb' # Process messages msg1 = mock_kernel[0].get_iopub_msg() msg2 = mock_kernel[0].get_iopub_msg() msg3 = mock_kernel[0].get_iopub_msg() msg4 = mock_kernel[0].get_iopub_msg() # Process first state change message assert monitor.kernel_status == 'idle' assert monitor.can_store_activity_record is False monitor.handle_message(msg1) assert monitor.kernel_status == 'busy' # Process input message monitor.handle_message(msg2) assert len(monitor.current_cell.code) > 0 assert len(monitor.cell_data) == 0 assert monitor.can_store_activity_record is False # Process output message monitor.handle_message(msg3) assert len(monitor.current_cell.result) > 0 assert len(monitor.cell_data) == 0 assert monitor.can_store_activity_record is False # Check lab book repo state status = mock_labbook[2].git.status() assert len(status["untracked"]) == 1 assert status["untracked"][0] == 'code/Test.ipynb' # Process final state change message monitor.handle_message(msg4) assert monitor.kernel_status == 'idle' assert monitor.can_store_activity_record is True assert len(monitor.cell_data) == 1 # Store the record manually for this test monitor.store_record(metadata) assert monitor.can_store_activity_record is False assert len(monitor.cell_data) == 0 # Check lab book repo state status = mock_labbook[2].git.status() assert len(status["untracked"]) == 0 assert len(status["staged"]) == 0 assert len(status["unstaged"]) == 0 # Check activity entry log = mock_labbook[2].git.log() assert len(log) == 4 assert 'code/Test.ipynb' in log[0]['message'] # Mock Performing an action AGAIN, faking editing the file and generating some output files mock_kernel[0].execute("a=100\nprint('Hello, World 2')") with open(dummy_file, 'wt') as tf: tf.write("change the fake notebook") with open(dummy_output, 'wt') as tf: tf.write("some result data") # Process messages msg1 = mock_kernel[0].get_iopub_msg() msg2 = mock_kernel[0].get_iopub_msg() msg3 = mock_kernel[0].get_iopub_msg() msg4 = mock_kernel[0].get_iopub_msg() # Process first state change message assert monitor.kernel_status == 'idle' assert monitor.can_store_activity_record is False monitor.handle_message(msg1) assert monitor.kernel_status == 'busy' # Process input message monitor.handle_message(msg2) assert len(monitor.current_cell.code) > 0 assert len(monitor.cell_data) == 0 assert monitor.can_store_activity_record is False # Process output message monitor.handle_message(msg3) assert len(monitor.current_cell.result) > 0 assert len(monitor.cell_data) == 0 assert monitor.can_store_activity_record is False # Check lab book repo state status = mock_labbook[2].git.status() assert len(status["staged"]) == 0 assert len(status["untracked"]) == 1 assert len(status["unstaged"]) == 1 assert status["unstaged"][0][0] == 'code/Test.ipynb' assert status["unstaged"][0][1] == 'modified' # Process final state change message monitor.handle_message(msg4) assert monitor.kernel_status == 'idle' assert monitor.can_store_activity_record is True assert len(monitor.cell_data) == 1 # Store the record manually for this test monitor.store_record(metadata) assert monitor.can_store_activity_record is False assert len(monitor.cell_data) == 0 # Check lab book repo state status = mock_labbook[2].git.status() assert len(status["untracked"]) == 0 assert len(status["staged"]) == 0 assert len(status["unstaged"]) == 0 # Check activity entry log = mock_labbook[2].git.log() assert len(log) == 6 assert 'code/Test.ipynb' in log[0]['message'] a_store = ActivityStore(mock_labbook[2]) record = a_store.get_activity_record(log[0]['commit']) assert record.type == ActivityType.CODE assert record.show is True assert record.importance == 0 assert not record.tags assert record.message == 'Executed cell in notebook code/Test.ipynb' assert len(record._detail_objects) == 4 assert record._detail_objects[0][0] is True assert record._detail_objects[0][1] == ActivityDetailType.RESULT.value assert record._detail_objects[0][2] == 155 assert record._detail_objects[1][0] is False assert record._detail_objects[1][1] == ActivityDetailType.CODE.value assert record._detail_objects[1][2] == 255 assert record._detail_objects[2][0] is False assert record._detail_objects[2][ 1] == ActivityDetailType.CODE_EXECUTED.value assert record._detail_objects[2][2] == 255 assert record._detail_objects[3][0] is False assert record._detail_objects[3][ 1] == ActivityDetailType.OUTPUT_DATA.value assert record._detail_objects[3][2] == 255 detail = a_store.get_detail_record(record._detail_objects[3][3].key) assert len(detail.data) == 1 assert detail.data[ 'text/markdown'] == 'Created new Output Data file `output/result.bin`' detail = a_store.get_detail_record(record._detail_objects[1][3].key) assert len(detail.data) == 1 assert detail.data[ 'text/markdown'] == 'Modified Code file `code/Test.ipynb`'
def change_base(self, repository: str, base_id: str, revision: int) -> None: """Delete existing base, create an activity record, call add_base Note that all packages that were installed by the current base will be removed from the environment (in env/package_manager). Even if the new base installs a newer version of a user-installed package, that package will remain in effect - this avoids actively breaking a working package selection and is easy enough for a user to update. In case it's useful, this method is robust to multiple base images (this might happen, for example, after a merge). If multiple base image files are found, all will be removed prior to installing the specified base. Args: repository: name of git repo for base images, e.g. 'gigantum_base-images' base_id: name of base image, e.g. 'python3-minimal' revision: The revision number specified INSIDE the YAML file for that base image """ # We'll populate detail records as we go detail_records: List[ActivityDetailRecord] = [] current_base_dir = Path(self.env_dir) / "base" matching_fnames = list(current_base_dir.glob('*.yaml')) short_message = '' if len(matching_fnames) != 1: logger.warning( f"Project misconfigured. Found {len(matching_fnames)} base configuration files." ) if len(matching_fnames) > 1: # We provide brief details regarding these files short_message = self.remove_all_bases(matching_fnames, detail_records) else: # We have a properly configured Labbook, we'll report more detail about the base short_message = self.remove_base(matching_fnames[0], detail_records) logger.info(short_message) if short_message: # We did something above - we commit and create an activity record commit = self.labbook.git.commit(short_message) # Create activity record - we populated detail_records above ar = ActivityRecord(ActivityType.ENVIRONMENT, message=short_message, linked_commit=commit.hexsha, tags=["environment", "base"], show=True) for adr in detail_records: ar.add_detail_object(adr) # Store the activity record. ars = ActivityStore(self.labbook) ars.create_activity_record(ar) # We construct a list of packages with `from_base` == True for each package manager packages_to_rm: Dict[str, List[str]] = {} for package in self.get_component_list("package_manager"): # Build dictionary of packages if package['from_base']: # We are removing the base - so the package isn't guaranteed packages_to_rm.setdefault(package['manager'], []).append(package["package"]) for p_manager, package_names in packages_to_rm.items(): # Package removal will also create activity records self.remove_packages(p_manager, package_names, remove_from_base=True) # add_base currently returns None, but this will incorporate any future changes return self.add_base(repository, base_id, revision)
def add_base(self, repository: str, base_id: str, revision: int) -> None: """Method to add a base to a LabBook's environment Note that if this is run after packages have been configured, it will leave user-specified packages alone (and not override them with the base-installed package) even if the base provides a newer version of the package. Args: repository: The Environment Component repository the component is in base_id: The name of the component revision: The revision to use, specified *inside* yaml file. Returns: None """ if not repository: raise ValueError('repository cannot be None or empty') if not base_id: raise ValueError('component cannot be None or empty') # Get the base base_data = self.bases.get_base(repository, base_id, revision) base_filename = f"{repository}_{base_id}.yaml" base_final_path = os.path.join(self.env_dir, 'base', base_filename) short_message = f"Added base: {base_id} r{revision}" # Count number of YAML files in our base dir - should be 0 existing_bases = sum( 1 for base_path in Path(self.env_dir, 'base').iterdir() if base_path.suffix == '.yaml') if existing_bases: # This shouldn't ever happen - but we don't trust the front-end raise ValueError( f"Found {existing_bases} base(s) already in this project") with open(base_final_path, 'wt') as cf: cf.write(yaml.safe_dump(base_data, default_flow_style=False)) # We construct records of packages installed by the user grouped by package manager # This can happen, for example, when we're changing bases installed_packages: Dict[str, List[str]] = {} for package in self.get_component_list("package_manager"): if package['from_base']: # Packages from the base to be added are NOT yet installed, but there are package # files that are marked `from_base`. This should never happen! logger.warning( 'Residual packages remain that are listed as installed by base - converting to user' ) self.add_packages(package_manager=package['manager'], packages=[package], force=True, from_base=False) # Build dictionary of packages installed_packages.setdefault(package['manager'], []).append(package["package"]) for manager in base_data['package_managers']: packages = list() # Build dictionary of packages for p_manager in manager.keys(): if manager[p_manager]: for pkg in manager[p_manager]: pkg_name, pkg_version = strip_package_and_version( p_manager, pkg) if pkg_name in installed_packages.get(p_manager, []): # If package is already installed by this package manager, we expect it gets overwritten # If it's a different package manger, it won't. continue packages.append({ "package": pkg_name, "version": pkg_version, "manager": p_manager }) self.add_packages(package_manager=p_manager, packages=packages, force=True, from_base=True) self.labbook.git.add(base_final_path) commit = self.labbook.git.commit(short_message) logger.info(f"Added base from {repository}: {base_id} rev{revision}") # Create a ActivityRecord long_message = "\n".join( (f"Added base {base_id}\n", f"{base_data['description']}\n", f" - repository: {repository}", f" - component: {base_id}", f" - revision: {revision}\n")) # Create detail record adr = ActivityDetailRecord(ActivityDetailType.ENVIRONMENT, show=False, action=ActivityAction.CREATE) adr.add_value('text/plain', long_message) # Create activity record ar = ActivityRecord(ActivityType.ENVIRONMENT, message=short_message, linked_commit=commit.hexsha, tags=["environment", "base"], show=True) ar.add_detail_object(adr) # Store ars = ActivityStore(self.labbook) ars.create_activity_record(ar)
def remove_packages(self, package_manager: str, package_names: List[str], remove_from_base: bool = False) -> None: """Remove yaml files describing a package and its context to the labbook. Args: package_manager: The package manager (eg., "apt" or "pip3") package_names: A list of packages to uninstall remove_from_base: Usually we won't do this, specify `True` when you are changing out a base """ # Create activity record ar = ActivityRecord( ActivityType.ENVIRONMENT, message="", show=True, linked_commit="", tags=["environment", 'package_manager', package_manager]) for pkg in package_names: yaml_filename = '{}_{}.yaml'.format(package_manager, pkg) package_yaml_path = os.path.join(self.env_dir, 'package_manager', yaml_filename) # Check for package to exist if not os.path.exists(package_yaml_path): raise ValueError( f"{package_manager} installed package {pkg} does not exist." ) # Check to make sure package isn't from the base. You cannot remove packages from the base yet. with open(package_yaml_path, 'rt') as cf: package_data = yaml.safe_load(cf) if not package_data: raise IOError("Failed to load package description") if package_data['from_base'] is True and not remove_from_base: raise ValueError( "Won't remove a package installed in the Base, without `remove_from_base=True`" ) # Delete the yaml file, which on next Dockerfile gen/rebuild will remove the dependency os.remove(package_yaml_path) if os.path.exists(package_yaml_path): raise ValueError(f"Failed to remove package.") self.labbook.git.remove(package_yaml_path) # Create detail record adr = ActivityDetailRecord(ActivityDetailType.ENVIRONMENT, show=False, action=ActivityAction.DELETE) adr.add_value('text/plain', f"Removed {package_manager} managed package: {pkg}") ar.add_detail_object(adr) logger.info(f"Removed {package_manager} managed package: {pkg}") # Add to git short_message = f"Removed {len(package_names)} {package_manager} managed package(s)" commit = self.labbook.git.commit(short_message) ar.linked_commit = commit.hexsha ar.message = short_message # Store ars = ActivityStore(self.labbook) ars.create_activity_record(ar)
def add_packages(self, package_manager: str, packages: List[dict], force: bool = False, from_base: bool = False) -> None: """Add a new yaml file describing the new package and its context to the labbook. Args: package_manager: The package manager (eg., "apt" or "pip3") packages: A dictionary of packages to install (package & version are main keys needed) force: Force overwriting a component if it already exists (e.g. you want to update the version) from_base: If a package in a base image, not deletable. Otherwise, can be deleted by LB user. Returns: None """ if not package_manager: raise ValueError( 'Argument package_manager cannot be None or empty') # Create activity record ar = ActivityRecord( ActivityType.ENVIRONMENT, show=True, message="", linked_commit="", tags=["environment", 'package_manager', package_manager]) update_cnt = 0 add_cnt = 0 for pkg in packages: version_str = f'"{pkg["version"]}"' if pkg["version"] else 'latest' yaml_lines = [ f'# Generated on: {datetime.datetime.now()}', f'manager: "{package_manager}"', f'package: "{pkg["package"]}"', f'version: {version_str}', f'from_base: {str(from_base).lower()}', f'schema: {CURRENT_SCHEMA}' ] yaml_filename = f'{package_manager}_{pkg["package"]}.yaml' package_yaml_path = os.path.join(self.env_dir, 'package_manager', yaml_filename) # Check if package already exists if os.path.exists(package_yaml_path): if force: # You are updating, since force is set and package already exists. logger.warning("Updating package file at {}".format( package_yaml_path)) detail_msg = "Update {} managed package: {} {}".format( package_manager, pkg["package"], version_str) adr = ActivityDetailRecord(ActivityDetailType.ENVIRONMENT, show=False, action=ActivityAction.EDIT) update_cnt += 1 else: raise ValueError( "The package {} already exists in this LabBook.". format(pkg["package"]) + " Use `force` to overwrite") else: add_cnt += 1 detail_msg = "Add {} managed package: {} {}".format( package_manager, pkg["package"], version_str) adr = ActivityDetailRecord(ActivityDetailType.ENVIRONMENT, show=False, action=ActivityAction.CREATE) # Write the YAML to the file with open(package_yaml_path, 'w') as package_yaml_file: package_yaml_file.write(os.linesep.join(yaml_lines)) # Create activity record adr.add_value('text/plain', detail_msg) ar.add_detail_object(adr) logger.info("Added package {} to labbook at {}".format( pkg["package"], self.labbook.root_dir)) # Set activity message ar_msg = "" if add_cnt > 0: ar_msg = f"Added {add_cnt} {package_manager} package(s). " if update_cnt > 0: ar_msg = f"{ar_msg}Updated {update_cnt} {package_manager} package(s)" # Add to git self.labbook.git.add_all(self.env_dir) commit = self.labbook.git.commit(ar_msg) ar.linked_commit = commit.hexsha ar.message = ar_msg # Store ars = ActivityStore(self.labbook) ars.create_activity_record(ar)