def mutate_and_get_payload(cls, root, info, owner, dataset_name, transaction_id, cancel=False, rollback=False, client_mutation_id=None): username = get_logged_in_username() ds = InventoryManager().load_dataset(username, owner, dataset_name, author=get_logged_in_author()) with ds.lock(): if cancel and rollback: logger.warning( f"Cancelled tx {transaction_id}, doing git reset") # TODO: Add ability to reset else: logger.info( f"Done batch upload {transaction_id}, cancelled={cancel}") if cancel: logger.warning("Sweeping aborted batch upload.") m = "Cancelled upload `{transaction_id}`. " if cancel else '' # Sweep up and process all files added during upload manifest = Manifest(ds, username) manifest.sweep_all_changes(upload=True, extra_msg=m) return CompleteDatasetUploadTransaction(success=True)
def mutate_and_get_payload(cls, root, info, owner, labbook_name, other_branch_name, override_method="abort", client_mutation_id=None): username = get_logged_in_username() lb = InventoryManager().load_labbook(username, owner, labbook_name, author=get_logged_in_author()) with lb.lock(): override = MergeOverride(override_method) bm = BranchManager(lb, username=username) if override == MergeOverride.ABORT: bm.merge_from(other_branch=other_branch_name) elif override == MergeOverride.OURS: bm.merge_use_ours(other_branch=other_branch_name) elif override == MergeOverride.THEIRS: bm.merge_use_theirs(other_branch=other_branch_name) else: raise ValueError(f"Unknown override method {override}") return MergeFromBranch( Labbook(id="{}&{}".format(owner, labbook_name), name=labbook_name, owner=owner))
def mutate_and_get_payload(cls, root, info, owner, labbook_name, branch_name, revision=None, description=None, client_mutation_id=None): username = get_logged_in_username() lb = InventoryManager().load_labbook(username, owner, labbook_name, author=get_logged_in_author()) with lb.lock(): bm = BranchManager(lb, username=username) full_branch_title = bm.create_branch(title=branch_name, revision=revision) logger.info( f"In {str(lb)} created new experimental feature branch: " f"{full_branch_title}") if description: cls._update_branch_description(lb, description) return CreateExperimentalBranch( Labbook(id="{}&{}".format(owner, labbook_name), name=labbook_name, owner=owner))
def mutate_and_get_payload(cls, root, info, owner, labbook_name, filename, mount_path, client_mutation_id=None): if len(mount_path) > 0 and mount_path[0] == '~': mount_path = mount_path.replace('~', '/home/giguser') if not cls._is_target_valid(mount_path): raise ValueError(f"Mount path {mount_path} is not a valid path") username = get_logged_in_username() lb = InventoryManager().load_labbook(username, owner, labbook_name, author=get_logged_in_author()) with lb.lock(): secstore = SecretStore(lb, username) secstore[filename] = mount_path cls._record_insert_activity(secstore, filename, lb, mount_path) env = Environment(owner=owner, name=lb.name) return InsertSecretsEntry(environment=env)
def mutate_and_get_payload(cls, root, info, owner, labbook_name, client_mutation_id=None): username = get_logged_in_username() lb = InventoryManager().load_labbook(username, owner, labbook_name, author=get_logged_in_author()) migrated = False with lb.lock(): t0 = time.time() workflow = LabbookWorkflow(lb) migrated = workflow.migrate() tf = time.time() if migrated: logger.info(f"Migrated {str(lb)} in {tf-t0:.2}sec") else: logger.info(f"No migration needed for {str(lb)}") return MigrateLabbookSchema( Labbook(id=f"{owner}&{labbook_name}", name=labbook_name, owner=owner))
def savehook(username, owner, labbook_name): try: changed_file = request.args.get('file') jupyter_token = request.args.get('jupyter_token') logger.debug(f"Received save hook for {changed_file} in {username}/{owner}/{labbook_name}") redis_conn = redis.Redis(db=1) lb_jupyter_token_key = '-'.join(['gmlb', username, owner, labbook_name, 'jupyter-token']) lb_active_key = f"{'|'.join([username, owner, labbook_name])}&is-busy*" r = redis_conn.get(lb_jupyter_token_key.encode()) if r is None: logger.error(f"Could not find jupyter token for {username}/{owner}/{labbook_name}") abort(400) if r.decode() != jupyter_token: raise ValueError("Incoming jupyter token must be valid") if len(redis_conn.keys(lb_active_key.encode())) > 0: # A kernel in this project is still active. Don't save auto-commit because it can blow up the # repository size depending on what the user is doing logger.info(f"Skipping jupyter savehook for {username}/{owner}/{labbook_name} due to active kernel") return 'success' lb = InventoryManager().load_labbook(username, owner, labbook_name, author=get_logged_in_author()) with lb.lock(): lb.sweep_uncommitted_changes() logger.info(f"Jupyter save hook saved {changed_file} from {str(lb)}") return 'success' except Exception as err: logger.error(err) return abort(400)
def mutate_and_get_payload(cls, root, info, owner, labbook_name, description_content, client_mutation_id=None): username = get_logged_in_username() lb = InventoryManager().load_labbook(username, owner, labbook_name, author=get_logged_in_author()) lb.description = description_content with lb.lock(): lb.git.add(os.path.join(lb.config_path)) commit = lb.git.commit('Updating description') adr = ActivityDetailRecord(ActivityDetailType.LABBOOK, show=False) adr.add_value('text/plain', "Updated description of Project") ar = ActivityRecord(ActivityType.LABBOOK, message="Updated description of Project", linked_commit=commit.hexsha, tags=["labbook"], show=False) ar.add_detail_object(adr) ars = ActivityStore(lb) ars.create_activity_record(ar) return SetLabbookDescription(success=True)
def mutate_and_get_payload(cls, root, info, owner, labbook_name, section, directory, client_mutation_id=None): username = get_logged_in_username() lb = InventoryManager().load_labbook(username, owner, labbook_name, author=get_logged_in_author()) with lb.lock(): FileOperations.makedir(lb, os.path.join(section, directory), create_activity_record=True) # Prime dataloader with labbook you already loaded dataloader = LabBookLoader() dataloader.prime(f"{owner}&{labbook_name}&{lb.name}", lb) # Create data to populate edge file_info = FileOperations.get_file_info(lb, section, directory) create_data = {'owner': owner, 'name': labbook_name, 'section': section, 'key': file_info['key'], '_file_info': file_info} # TODO: Fix cursor implementation, this currently doesn't make sense cursor = base64.b64encode(f"{0}".encode('utf-8')) return MakeLabbookDirectory( new_labbook_file_edge=LabbookFileConnection.Edge( node=LabbookFile(**create_data), cursor=cursor))
def mutate_and_get_payload(cls, root, info, owner, labbook_name, section, key, description=None, is_dir=False, client_mutation_id=None): username = get_logged_in_username() lb = InventoryManager().load_labbook(username, owner, labbook_name, author=get_logged_in_author()) # Add Favorite if is_dir: is_dir = is_dir # Make sure trailing slashes are always present when favoriting a dir if key[-1] != "/": key = f"{key}/" with lb.lock(): new_favorite = lb.create_favorite(section, key, description=description, is_dir=is_dir) # Create data to populate edge create_data = {"id": f"{owner}&{labbook_name}&{section}&{key}", "owner": owner, "section": section, "name": labbook_name, "key": key, "index": new_favorite['index'], "_favorite_data": new_favorite} # Create cursor cursor = base64.b64encode(f"{str(new_favorite['index'])}".encode('utf-8')) return AddLabbookFavorite(new_favorite_edge=LabbookFavoriteConnection.Edge(node=LabbookFavorite(**create_data), cursor=cursor))
def mutate_and_get_payload(cls, root, info, owner, labbook_name, section, src_path, dst_path, client_mutation_id=None, **kwargs): username = get_logged_in_username() lb = InventoryManager().load_labbook(username, owner, labbook_name, author=get_logged_in_author()) with lb.lock(): mv_results = FileOperations.move_file(lb, section, src_path, dst_path) file_edges = list() for file_dict in mv_results: file_edges.append(LabbookFile(owner=owner, name=labbook_name, section=section, key=file_dict['key'], is_dir=file_dict['is_dir'], is_favorite=file_dict['is_favorite'], modified_at=file_dict['modified_at'], size=str(file_dict['size']))) cursors = [base64.b64encode("{}".format(cnt).encode("UTF-8")).decode("UTF-8") for cnt, x in enumerate(file_edges)] edge_objs = [LabbookFileConnection.Edge(node=e, cursor=c) for e, c in zip(file_edges, cursors)] return MoveLabbookFile(updated_edges=edge_objs)
def mutate_and_get_payload(cls, root, info, owner, dataset_name, description, client_mutation_id=None): username = get_logged_in_username() ds = InventoryManager().load_dataset(username, owner, dataset_name, author=get_logged_in_author()) ds.description = description with ds.lock(): ds.git.add(os.path.join(ds.root_dir, '.gigantum/gigantum.yaml')) commit = ds.git.commit('Updating description') adr = ActivityDetailRecord(ActivityDetailType.LABBOOK, show=False) adr.add_value('text/plain', f"Updated Dataset description: {description}") ar = ActivityRecord(ActivityType.LABBOOK, message="Updated Dataset description", linked_commit=commit.hexsha, tags=["dataset"], show=False) ar.add_detail_object(adr) ars = ActivityStore(ds) ars.create_activity_record(ar) return SetDatasetDescription( updated_dataset=Dataset(owner=owner, name=dataset_name))
def mutate_and_process_upload(cls, info, upload_file_path, upload_filename, **kwargs): if not upload_file_path: logger.error('No file uploaded') raise ValueError('No file uploaded') username = get_logged_in_username() owner = kwargs.get('owner') dataset_name = kwargs.get('dataset_name') file_path = kwargs.get('file_path') try: ds = InventoryManager().load_dataset(username, owner, dataset_name, author=get_logged_in_author()) with ds.lock(): if not os.path.abspath(upload_file_path): raise ValueError( f"Source file `{upload_file_path}` not an absolute path" ) if not os.path.isfile(upload_file_path): raise ValueError( f"Source file does not exist at `{upload_file_path}`") manifest = Manifest(ds, username) full_dst = manifest.get_abs_path(file_path) # If file (hard link) already exists, remove it first so you don't write to all files with same content if os.path.isfile(full_dst): os.remove(full_dst) full_dst_base = os.path.dirname(full_dst) if not os.path.isdir(full_dst_base): pathlib.Path(full_dst_base).mkdir(parents=True, exist_ok=True) shutil.move(upload_file_path, full_dst) file_info = manifest.gen_file_info(file_path) finally: try: logger.debug(f"Removing temp file {upload_file_path}") os.remove(upload_file_path) except FileNotFoundError: pass # Create data to populate edge create_data = { 'owner': owner, 'name': dataset_name, 'key': file_info['key'], '_file_info': file_info } # TODO: Fix cursor implementation. this currently doesn't make sense when adding edges cursor = base64.b64encode(f"{0}".encode('utf-8')) return AddDatasetFile(new_dataset_file_edge=DatasetFileConnection.Edge( node=DatasetFile(**create_data), cursor=cursor))
def mutate_and_get_payload(cls, root, info, owner, labbook_name, client_mutation_id=None): username = get_logged_in_username() lb = InventoryManager().load_labbook(username, owner, labbook_name, author=get_logged_in_author()) with lb.lock(): cls._stop_container(lb, username) return StopContainer(environment=Environment(owner=owner, name=labbook_name))
def mutate_and_get_payload(cls, root, info, owner, labbook_name, branch_name, client_mutation_id=None): username = get_logged_in_username() lb = InventoryManager().load_labbook(username, owner, labbook_name, author=get_logged_in_author()) with lb.lock(): wf = LabbookWorkflow(lb) wf.checkout(username, branch_name) return WorkonBranch(Labbook(id="{}&{}".format(owner, labbook_name), name=labbook_name, owner=owner))
def mutate_and_get_payload(cls, root, info, owner, labbook_name, section, file_paths, client_mutation_id=None): username = get_logged_in_username() lb = InventoryManager().load_labbook(username, owner, labbook_name, author=get_logged_in_author()) with lb.lock(): FileOperations.delete_files(lb, section, file_paths) return DeleteLabbookFiles(success=True)
def mutate_and_get_payload(cls, root, info, owner, labbook_name, content, client_mutation_id=None): username = get_logged_in_username() lb = InventoryManager().load_labbook(username, owner, labbook_name, author=get_logged_in_author()) # Write data with lb.lock(): lb.write_readme(content) return WriteLabbookReadme(updated_labbook=Labbook(owner=owner, name=labbook_name))
def mutate_and_get_payload(cls, root, info, owner, labbook_name, transaction_id, cancel=False, rollback=False, client_mutation_id=None): username = get_logged_in_username() lb = InventoryManager().load_labbook(username, owner, labbook_name, author=get_logged_in_author()) with lb.lock(): FileOperations.complete_batch(lb, transaction_id, cancel=cancel, rollback=rollback) return CompleteBatchUploadTransaction(success=True)
def mutate_and_get_payload(cls, root: str, info: str, owner: str, labbook_name: str, dev_tool: str, container_override_id: str = None, client_mutation_id: str = None): username = get_logged_in_username() labbook = InventoryManager().load_labbook(username, owner, labbook_name, author=get_logged_in_author()) with labbook.lock(failfast=True): path = cls._start_dev_tool(labbook, username, dev_tool.lower(), container_override_id) return StartDevTool(path=path)
def mutate_and_get_payload(cls, root, info, owner, dataset_name, visibility, client_mutation_id=None): # Load Dataset username = get_logged_in_username() ds = InventoryManager().load_dataset(username, owner, dataset_name, author=get_logged_in_author()) # Extract valid Bearer token token = None if hasattr(info.context.headers, 'environ'): if "HTTP_AUTHORIZATION" in info.context.headers.environ: token = parse_token( info.context.headers.environ["HTTP_AUTHORIZATION"]) if not token: raise ValueError( "Authorization header not provided. Must have a valid session to query for collaborators" ) default_remote = ds.client_config.config['git']['default_remote'] admin_service = None for remote in ds.client_config.config['git']['remotes']: if default_remote == remote: admin_service = ds.client_config.config['git']['remotes'][ remote]['admin_service'] break if not admin_service: raise ValueError('admin_service could not be found') # Configure git creds mgr = GitLabManager(default_remote, admin_service, access_token=token) mgr.configure_git_credentials(default_remote, username) if visibility not in ['public', 'private']: raise ValueError( f'Visibility must be either "public" or "private";' f'("{visibility}" invalid)') with ds.lock(): mgr.set_visibility(namespace=owner, repository_name=dataset_name, visibility=visibility) cursor = base64.b64encode(f"{0}".encode('utf-8')) dsedge = DatasetConnection.Edge(node=DatasetObject(owner=owner, name=dataset_name), cursor=cursor) return SetDatasetVisibility(new_dataset_edge=dsedge)
def mutate_and_get_payload(cls, root, info, owner, labbook_name, client_mutation_id=None): username = get_logged_in_username() lb = InventoryManager().load_labbook(username, owner, labbook_name, author=get_logged_in_author()) with lb.lock(): lb, container_id = ContainerOperations.start_container( labbook=lb, username=username) logger.info(f'Started new {lb} container ({container_id})') return StartContainer(environment=Environment(owner=owner, name=labbook_name))
def mutate_and_get_payload(cls, root, info, owner, labbook_name, branch_name, delete_local=False, delete_remote=False, client_mutation_id=None): username = get_logged_in_username() lb = InventoryManager().load_labbook(username, owner, labbook_name, author=get_logged_in_author()) with lb.lock(): bm = BranchManager(lb, username=username) if delete_local: bm.remove_branch(target_branch=branch_name) if delete_remote: bm.remove_remote_branch(target_branch=branch_name) return DeleteExperimentalBranch(Labbook(id="{}&{}".format(owner, labbook_name), name=labbook_name, owner=owner))
def mutate_and_get_payload(cls, root, info, owner, labbook_name, section, key, client_mutation_id=None): username = get_logged_in_username() lb = InventoryManager().load_labbook(username, owner, labbook_name, author=get_logged_in_author()) # Manually generate the Node ID for now. This simplifies the connection between the file browser and favorites # widgets in the UI favorite_node_id = f"LabbookFavorite:{owner}&{labbook_name}&{section}&{key}" favorite_node_id = base64.b64encode(favorite_node_id.encode()).decode() with lb.lock(): lb.remove_favorite(section, key) return RemoveLabbookFavorite(success=True, removed_node_id=favorite_node_id)
def export_dataset_as_zip(dataset_path: str, ds_export_directory: str) -> str: """Return path to archive file of exported dataset. """ p = os.getpid() logger = LMLogger.get_logger() logger.info(f"(Job {p}) Starting export_dataset_as_zip({dataset_path})") try: ds = InventoryManager().load_dataset_from_directory(dataset_path) with ds.lock(): path = ZipExporter.export_dataset(ds.root_dir, ds_export_directory) return path except Exception as e: logger.exception(f"(Job {p}) Error on export_dataset_as_zip: {e}") raise
def export_labbook_as_zip(labbook_path: str, lb_export_directory: str) -> str: """Return path to archive file of exported labbook. """ p = os.getpid() logger = LMLogger.get_logger() logger.info(f"(Job {p}) Starting export_labbook_as_zip({labbook_path})") try: lb = InventoryManager().load_labbook_from_directory(labbook_path) with lb.lock(): path = ZipExporter.export_labbook(lb.root_dir, lb_export_directory) return path except Exception as e: logger.exception(f"(Job {p}) Error on export_labbook_as_zip: {e}") raise
def mutate_and_get_payload(cls, root, info, owner, labbook_name, remote_name, remote_url, client_mutation_id=None): username = get_logged_in_username() lb = InventoryManager().load_labbook(username, owner, labbook_name, author=get_logged_in_author()) with lb.lock(): lb.add_remote(remote_name, remote_url) return AddLabbookRemote(success=True)
def mutate_and_get_payload(cls, root, info, owner, labbook_name, client_mutation_id=None): username = get_logged_in_username() lb = InventoryManager().load_labbook(username, owner, labbook_name, author=get_logged_in_author()) # TODO - Should we cehck if a custom docker component already exists? with lb.lock(): cm = ComponentManager(lb) cm.remove_docker_snippet(cm.DEFAULT_CUSTOM_DOCKER_NAME) return RemoveCustomDocker( updated_environment=Environment(owner=owner, name=labbook_name))
def mutate_and_process_upload(cls, info, upload_file_path, upload_filename, **kwargs): if not upload_file_path: logger.error('No file uploaded') raise ValueError('No file uploaded') owner = kwargs.get('owner') labbook_name = kwargs.get('labbook_name') section = kwargs.get('section') transaction_id = kwargs.get('transaction_id') file_path = kwargs.get('file_path') try: username = get_logged_in_username() lb = InventoryManager().load_labbook(username, owner, labbook_name, author=get_logged_in_author()) dst_path = os.path.join(os.path.dirname(file_path), upload_filename) with lb.lock(): fops = FileOperations.put_file(labbook=lb, section=section, src_file=upload_file_path, dst_path=dst_path, txid=transaction_id) finally: try: logger.debug(f"Removing temp file {upload_file_path}") os.remove(upload_file_path) except FileNotFoundError: pass # Create data to populate edge create_data = { 'owner': owner, 'name': labbook_name, 'section': section, 'key': fops['key'], '_file_info': fops } # TODO: Fix cursor implementation..this currently doesn't make sense when adding edges without a refresh cursor = base64.b64encode(f"{0}".encode('utf-8')) return AddLabbookFile(new_labbook_file_edge=LabbookFileConnection.Edge( node=LabbookFile(**create_data), cursor=cursor))
def mutate_and_get_payload(cls, root, info, owner, labbook_name, packages, client_mutation_id=None): username = get_logged_in_username() lb = InventoryManager().load_labbook(username, owner, labbook_name, author=get_logged_in_author()) with lb.lock(): new_edges = cls._add_package_components(lb, packages) return AddPackageComponents(new_package_component_edges=new_edges)
def mutate_and_get_payload(cls, root, info, owner, labbook_name, filename, client_mutation_id=None): username = get_logged_in_username() lb = InventoryManager().load_labbook(username, owner, labbook_name, author=get_logged_in_author()) with lb.lock(): secstore = SecretStore(lb, username) secstore.delete_file(filename) env = Environment(owner=owner, name=lb.name) return DeleteSecretsFile(environment=env)
def mutate_and_get_payload(cls, root, info, owner, labbook_name, app_name, client_mutation_id=None): username = get_logged_in_username() lb = InventoryManager().load_labbook(username, owner, labbook_name, author=get_logged_in_author()) bam = BundledAppManager(lb) with lb.lock(): bam.remove_bundled_app(app_name) return SetBundledApp( environment=Environment(name=labbook_name, owner=owner))