def mutate_and_get_payload(cls, root, info, owner, labbook_name, section, key, updated_index=None, updated_description=None, client_mutation_id=None): username = get_logged_in_username() lb = InventoryManager().load_labbook(username, owner, labbook_name, author=get_logged_in_author()) with lb.lock(): new_favorite = lb.update_favorite( section, key, new_description=updated_description, new_index=updated_index) # Create data to populate edge create_data = { "id": f"{owner}&{labbook_name}&{section}&{key}", "owner": owner, "section": section, "key": key, "_favorite_data": new_favorite } # Create dummy cursor cursor = base64.b64encode( f"{str(new_favorite['index'])}".encode('utf-8')) return UpdateLabbookFavorite( updated_favorite_edge=LabbookFavoriteConnection.Edge( node=LabbookFavorite(**create_data), cursor=cursor))
def mutate_and_get_payload(cls, root, info, owner, labbook_name, section, directory, client_mutation_id=None): username = get_logged_in_username() working_directory = Configuration().config['git']['working_directory'] inferred_lb_directory = os.path.join(working_directory, username, owner, 'labbooks', labbook_name) lb = LabBook(author=get_logged_in_author()) lb.from_directory(inferred_lb_directory) lb.makedir(os.path.join(section, directory), create_activity_record=True) logger.info(f"Made new directory in `{directory}`") # Prime dataloader with labbook you already loaded dataloader = LabBookLoader() dataloader.prime(f"{owner}&{labbook_name}&{lb.name}", lb) # Create data to populate edge file_info = lb.get_file_info(section, directory) create_data = { 'owner': owner, 'name': labbook_name, 'section': section, 'key': file_info['key'], '_file_info': file_info } # TODO: Fix cursor implementation, this currently doesn't make sense cursor = base64.b64encode(f"{0}".encode('utf-8')) return MakeLabbookDirectory( new_labbook_file_edge=LabbookFileConnection.Edge( node=LabbookFile(**create_data), cursor=cursor))
def mutate_and_get_payload(cls, root, info, owner, labbook_name, visibility, client_mutation_id=None): # Load LabBook username = get_logged_in_username() lb = InventoryManager().load_labbook(username, owner, labbook_name, author=get_logged_in_author()) # Extract valid Bearer token token = None if hasattr(info.context.headers, 'environ'): if "HTTP_AUTHORIZATION" in info.context.headers.environ: token = parse_token(info.context.headers.environ["HTTP_AUTHORIZATION"]) if not token: raise ValueError("Authorization header not provided. Must have a valid session to query for collaborators") default_remote = lb.client_config.config['git']['default_remote'] admin_service = None for remote in lb.client_config.config['git']['remotes']: if default_remote == remote: admin_service = lb.client_config.config['git']['remotes'][remote]['admin_service'] break if not admin_service: raise ValueError('admin_service could not be found') # Configure git creds mgr = GitLabManager(default_remote, admin_service, access_token=token) mgr.configure_git_credentials(default_remote, username) if visibility not in ['public', 'private']: raise ValueError(f'Visibility must be either "public" or "private";' f'("{visibility}" invalid)') with lb.lock(): mgr.set_visibility(namespace=owner, repository_name=labbook_name, visibility=visibility) cursor = base64.b64encode(f"{0}".encode('utf-8')) lbedge = LabbookConnection.Edge(node=LabbookObject(owner=owner, name=labbook_name), cursor=cursor) return SetVisibility(new_labbook_edge=lbedge)
def mutate_and_get_payload(cls, root, info, owner, labbook_name, section, key, client_mutation_id=None): username = get_logged_in_username() lb = LabBook(author=get_logged_in_author()) lb.from_name(username, owner, labbook_name) # Manually generate the Node ID for now. This simplifies the connection between the file browser and favorites # widgets in the UI favorite_node_id = f"LabbookFavorite:{owner}&{labbook_name}&{section}&{key}" favorite_node_id = base64.b64encode(favorite_node_id.encode()).decode() # Remove Favorite lb.remove_favorite(section, key) return RemoveLabbookFavorite(success=True, removed_node_id=favorite_node_id)
def mutate_and_get_payload(cls, root, info, owner, labbook_name, username, client_mutation_id=None): logged_in_username = get_logged_in_username() lb = InventoryManager().load_labbook(logged_in_username, owner, labbook_name, author=get_logged_in_author()) # TODO: Future work will look up remote in LabBook data, allowing user to select remote. default_remote = lb.client_config.config['git']['default_remote'] admin_service = None for remote in lb.client_config.config['git']['remotes']: if default_remote == remote: admin_service = lb.client_config.config['git']['remotes'][ remote]['admin_service'] break # Extract valid Bearer token if "HTTP_AUTHORIZATION" in info.context.headers.environ: token = parse_token( info.context.headers.environ["HTTP_AUTHORIZATION"]) else: raise ValueError( "Authorization header not provided. Must have a valid session to query for collaborators" ) # Add collaborator to remote service mgr = GitLabManager(default_remote, admin_service, token) mgr.delete_collaborator(owner, labbook_name, username) create_data = {"owner": owner, "name": labbook_name} return DeleteLabbookCollaborator(updated_labbook=Labbook( **create_data))
def _start_jupyter_tool(cls, labbook: LabBook, router: ProxyRouter, username: str, container_override_id: str = None): tool_port = 8888 labbook_ip = ContainerOperations.get_labbook_ip(labbook, username) labbook_endpoint = f'http://{labbook_ip}:{tool_port}' matched_routes = router.get_matching_routes(labbook_endpoint, 'jupyter') run_start_jupyter = True suffix = None if len(matched_routes) == 1: logger.info(f'Found existing Jupyter instance in route table for {str(labbook)}.') suffix = matched_routes[0] # wait for jupyter to be up try: check_jupyter_reachable(labbook_ip, tool_port, suffix) run_start_jupyter = False except GigantumException: logger.warning(f'Detected stale route. Attempting to restart Jupyter and clean up route table.') router.remove(suffix[1:]) elif len(matched_routes) > 1: raise ValueError(f"Multiple Jupyter instances found in route table for {str(labbook)}! Restart container.") if run_start_jupyter: rt_prefix = unique_id() rt_prefix, _ = router.add(labbook_endpoint, f'jupyter/{rt_prefix}') # Start jupyterlab suffix = start_jupyter(labbook, username, tag=container_override_id, proxy_prefix=rt_prefix) # Ensure we start monitor IFF jupyter isn't already running. start_labbook_monitor(labbook, username, 'jupyterlab', url=f'{labbook_endpoint}/{rt_prefix}', author=get_logged_in_author()) return suffix
def mutate_and_get_payload(cls, root, info, owner, labbook_name, set_public=False, client_mutation_id=None): # Load LabBook username = get_logged_in_username() lb = InventoryManager().load_labbook(username, owner, labbook_name, author=get_logged_in_author()) # Extract valid Bearer token if "HTTP_AUTHORIZATION" in info.context.headers.environ: token = parse_token( info.context.headers.environ["HTTP_AUTHORIZATION"]) else: raise ValueError( "Authorization header not provided. Must have a valid session to query for collaborators" ) job_metadata = {'method': 'publish_labbook', 'labbook': lb.key} job_kwargs = { 'repository': lb, 'username': username, 'access_token': token, 'public': set_public } dispatcher = Dispatcher() job_key = dispatcher.dispatch_task(jobs.publish_repository, kwargs=job_kwargs, metadata=job_metadata) logger.info( f"Publishing LabBook {lb.root_dir} in background job with key {job_key.key_str}" ) return PublishLabbook(job_key=job_key.key_str)
def mutate_and_get_payload(cls, root, info, owner, labbook_name, section, directory, client_mutation_id=None): username = get_logged_in_username() lb = InventoryManager().load_labbook(username, owner, labbook_name, author=get_logged_in_author()) with lb.lock(): FileOperations.makedir(lb, os.path.join(section, directory), create_activity_record=True) # Prime dataloader with labbook you already loaded dataloader = LabBookLoader() dataloader.prime(f"{owner}&{labbook_name}&{lb.name}", lb) # Create data to populate edge file_info = FileOperations.get_file_info(lb, section, directory) create_data = { 'owner': owner, 'name': labbook_name, 'section': section, 'key': file_info['key'], '_file_info': file_info } # TODO: Fix cursor implementation, this currently doesn't make sense cursor = base64.b64encode(f"{0}".encode('utf-8')) return MakeLabbookDirectory( new_labbook_file_edge=LabbookFileConnection.Edge( node=LabbookFile(**create_data), cursor=cursor))
def mutate_and_get_payload(cls, root, info, name, description, repository, base_id, revision, is_untracked=False, client_mutation_id=None): username = get_logged_in_username() inv_manager = InventoryManager() lb = inv_manager.create_labbook(username=username, owner=username, labbook_name=name, description=description, author=get_logged_in_author()) adr = ActivityDetailRecord(ActivityDetailType.LABBOOK, show=False, importance=0) adr.add_value('text/plain', f"Created new LabBook: {username}/{name}") # Create activity record ar = ActivityRecord(ActivityType.LABBOOK, message=f"Created new LabBook: {username}/{name}", show=True, importance=255, linked_commit=lb.git.commit_hash) ar.add_detail_object(adr) store = ActivityStore(lb) store.create_activity_record(ar) cm = ComponentManager(lb) cm.add_base(repository, base_id, revision) return CreateLabbook(labbook=Labbook(owner=username, name=lb.name))
def mutate_and_get_payload(cls, root, info, owner, labbook_name, title, body=None, tags=None, client_mutation_id=None): username = get_logged_in_username() # Load LabBook instance lb = LabBook(author=get_logged_in_author()) lb.from_name(username, owner, labbook_name) # Create a Activity Store instance store = ActivityStore(lb) # Create detail record adr = ActivityDetailRecord(ActivityDetailType.NOTE, show=True, importance=255) if body: adr.add_value('text/markdown', body) # Create activity record ar = ActivityRecord(ActivityType.NOTE, message=title, linked_commit="no-linked-commit", importance=255, tags=tags) ar.add_detail_object(adr) ar = store.create_activity_record(ar) return CreateUserNote(new_activity_record_edge=ActivityConnection.Edge( node=ActivityRecordObject( owner=owner, name=labbook_name, commit=ar.commit), cursor=ar.commit))
def mutate_and_get_payload(cls, root, info, owner, labbook_name, confirm, client_mutation_id=None): username = get_logged_in_username() working_directory = Configuration().config['git']['working_directory'] inferred_lb_directory = os.path.join(working_directory, username, owner, 'labbooks', labbook_name) lb = LabBook(author=get_logged_in_author()) lb.from_directory(inferred_lb_directory) if confirm: logger.warning(f"Deleting {str(lb)}...") try: lb, stopped = ContainerOperations.stop_container( labbook=lb, username=username) except OSError: pass lb, docker_removed = ContainerOperations.delete_image( labbook=lb, username=username) if not docker_removed: raise ValueError( f'Cannot delete docker image for {str(lb)} - unable to delete LB from disk' ) shutil.rmtree(lb.root_dir, ignore_errors=True) if os.path.exists(lb.root_dir): logger.error( f'Deleted {str(lb)} but root directory {lb.root_dir} still exists!' ) return DeleteLabbook(success=False) else: return DeleteLabbook(success=True) else: logger.info(f"Dry run in deleting {str(lb)} -- not deleted.") return DeleteLabbook(success=False)
def mutate_and_get_payload(cls, root, info, owner, labbook_name, branch_name, client_mutation_id=None): """Method to perform mutation Args: input: context: info: Returns: """ username = get_logged_in_username() # Load an existing LabBook labbook_obj = LabBook(author=get_logged_in_author()) labbook_obj.from_name(username, owner, labbook_name) # Checkout labbook_obj.checkout_branch(branch_name) return CheckoutBranch(labbook=Labbook(owner=owner, name=labbook_name))
def mutate_and_get_payload(cls, root, info, owner, labbook_name, other_branch_name, override_method="abort", client_mutation_id=None): username = get_logged_in_username() lb = InventoryManager().load_labbook(username, owner, labbook_name, author=get_logged_in_author()) with lb.lock(): override = MergeOverride(override_method) bm = BranchManager(lb, username=username) if override == MergeOverride.ABORT: bm.merge_from(other_branch=other_branch_name) elif override == MergeOverride.OURS: bm.merge_use_ours(other_branch=other_branch_name) elif override == MergeOverride.THEIRS: bm.merge_use_theirs(other_branch=other_branch_name) else: raise ValueError(f"Unknown override method {override}") # Run update_linked_datasets() to initialize and cleanup dataset submodules. You don't need to schedule # auto-import jobs because the user will have switched to the branch to pull it before merging. InventoryManager().update_linked_datasets(lb, username) return MergeFromBranch(Labbook(id="{}&{}".format(owner, labbook_name), name=labbook_name, owner=owner))
def mutate_and_get_payload(cls, root, info, owner, labbook_name, branch_name, client_mutation_id=None): """Method to perform mutation Args: input: context: info: Returns: """ username = get_logged_in_username() # Load an existing LabBook labbook_obj = LabBook(author=get_logged_in_author()) labbook_obj.from_name(username, owner, labbook_name) # Create Branch labbook_obj.checkout_branch(branch_name, new=True) if labbook_obj.active_branch != branch_name: raise ValueError( f"Create branch failed, could not switch to new branch {branch_name}" ) # Create a LabbookRef to the branch create_data = { "owner": owner, "name": labbook_name, "prefix": None, "branch": branch_name, } return CreateBranch(branch=LabbookRef(**create_data))
def mutate_and_get_payload(cls, root, info, dataset_owner, dataset_name, key, client_mutation_id=None): logged_in_username = get_logged_in_username() ds = InventoryManager().load_dataset(logged_in_username, dataset_owner, dataset_name, author=get_logged_in_author()) ds.namespace = dataset_owner m = Manifest(ds, logged_in_username) if key[-1] != '/': raise ValueError( "Provided relative path must end in `/` to indicate it is a directory" ) with ds.lock(): file_info = m.create_directory(key) create_data = { 'owner': dataset_owner, 'name': dataset_name, 'key': file_info['key'], '_file_info': file_info } # TODO: Fix cursor implementation, this currently doesn't make sense cursor = base64.b64encode(f"{0}".encode('utf-8')) return MakeDatasetDirectory( new_dataset_file_edge=DatasetFileConnection.Edge( node=DatasetFile(**create_data), cursor=cursor))
def mutate_and_get_payload(cls, root, info, owner, dataset_name, client_mutation_id=None): username = get_logged_in_username() working_directory = Configuration().config['git']['working_directory'] ds = InventoryManager().load_dataset(username, owner, dataset_name, author=get_logged_in_author()) job_metadata = {'method': 'export_dataset_as_zip', 'dataset': ds.key} job_kwargs = { 'dataset_path': ds.root_dir, 'ds_export_directory': os.path.join(working_directory, 'export') } dispatcher = Dispatcher() job_key = dispatcher.dispatch_task(jobs.export_dataset_as_zip, kwargs=job_kwargs, metadata=job_metadata) return ExportDataset(job_key=job_key.key_str)
def mutate_and_get_payload(cls, root, info, owner, labbook_name, branch_name, delete_local=False, delete_remote=False, client_mutation_id=None): username = get_logged_in_username() lb = InventoryManager().load_labbook(username, owner, labbook_name, author=get_logged_in_author()) with lb.lock(): bm = BranchManager(lb, username=username) if delete_local: bm.remove_branch(target_branch=branch_name) if delete_remote: bm.remove_remote_branch(target_branch=branch_name) return DeleteExperimentalBranch( Labbook(id="{}&{}".format(owner, labbook_name), name=labbook_name, owner=owner))
def helper_resolve_linked_datasets(labbook, info): submodules = labbook.git.list_submodules() datasets = list() for submodule in submodules: try: namespace, dataset_name = submodule['name'].split("&") submodule_dir = os.path.join(labbook.root_dir, '.gigantum', 'datasets', namespace, dataset_name) ds = InventoryManager().load_dataset_from_directory(submodule_dir, author=get_logged_in_author()) ds.namespace = namespace info.context.dataset_loader.prime(f"{get_logged_in_username()}&{namespace}&{dataset_name}", ds) datasets.append(Dataset(owner=namespace, name=dataset_name)) except InventoryException: continue return datasets
def mutate_and_get_payload(cls, root, info, dataset_owner, dataset_name, from_local=False, from_remote=False, client_mutation_id=None): logged_in_username = get_logged_in_username() im = InventoryManager() ds = im.load_dataset(logged_in_username, dataset_owner, dataset_name, get_logged_in_author()) ds.backend.set_default_configuration(logged_in_username, bearer_token=flask.g.access_token, id_token=flask.g.id_token) if not ds.backend.is_configured: raise ValueError("Dataset is not fully configured. Cannot update.") d = Dispatcher() kwargs = { 'logged_in_username': logged_in_username, 'access_token': flask.g.access_token, 'id_token': flask.g.id_token, 'dataset_owner': dataset_owner, 'dataset_name': dataset_name, } background_job_key = None if from_remote is True: if ds.backend.can_update_from_remote: # Gen unique keys for tracking jobs metadata = { 'dataset': f"{logged_in_username}|{dataset_owner}|{dataset_name}", 'method': 'update_unmanaged_dataset_from_remote' } job_response = d.dispatch_task( jobs.update_unmanaged_dataset_from_remote, kwargs=kwargs, metadata=metadata) background_job_key = job_response.key_str else: raise ValueError( "This dataset type does not support automatic update via querying its remote" ) elif from_local is True: # Gen unique keys for tracking jobs metadata = { 'dataset': f"{logged_in_username}|{dataset_owner}|{dataset_name}", 'method': 'update_unmanaged_dataset_from_local' } job_response = d.dispatch_task( jobs.update_unmanaged_dataset_from_local, kwargs=kwargs, metadata=metadata) background_job_key = job_response.key_str else: ValueError("Either `fromRemote` or `fromLocal` must be True.") return UpdateUnmanagedDataset(dataset=Dataset(id="{}&{}".format( dataset_owner, dataset_name), name=dataset_name, owner=dataset_owner), background_job_key=background_job_key)
def mutate_and_get_payload(cls, root, info, labbook_owner, labbook_name, dataset_owner, dataset_name, action, dataset_url=None, client_mutation_id=None): logged_in_username = get_logged_in_username() im = InventoryManager() lb = im.load_labbook(logged_in_username, labbook_owner, labbook_name, author=get_logged_in_author()) with lb.lock(): if action == 'link': if dataset_url: remote_domain = cls._get_remote_domain( dataset_url, dataset_owner, dataset_name) if remote_domain: # Make sure git creds are configured for the remote admin_service = None for remote in lb.client_config.config['git'][ 'remotes']: if remote_domain == remote: admin_service = lb.client_config.config['git'][ 'remotes'][remote]['admin_service'] break if "HTTP_AUTHORIZATION" in info.context.headers.environ: token = parse_token(info.context.headers. environ["HTTP_AUTHORIZATION"]) else: raise ValueError( "Authorization header not provided." " Must have a valid session to query for collaborators" ) mgr = GitLabManager(remote_domain, admin_service, token) mgr.configure_git_credentials(remote_domain, logged_in_username) else: # Link to local dataset ds = im.load_dataset(logged_in_username, dataset_owner, dataset_name) dataset_url = f"{ds.root_dir}/.git" # Link the dataset to the labbook ds = im.link_dataset_to_labbook(dataset_url, dataset_owner, dataset_name, lb) ds.namespace = dataset_owner # Preload the dataloader info.context.dataset_loader.prime( f"{get_logged_in_username()}&{dataset_owner}&{dataset_name}", ds) # Relink the revision m = Manifest(ds, logged_in_username) m.link_revision() elif action == 'unlink': im.unlink_dataset_from_labbook(dataset_owner, dataset_name, lb) elif action == 'update': ds = im.update_linked_dataset_reference( dataset_owner, dataset_name, lb) m = Manifest(ds, logged_in_username) m.force_reload() info.context.dataset_loader.prime( f"{get_logged_in_username()}&{dataset_owner}&{dataset_name}", ds) else: raise ValueError( "Unsupported action. Use `link`, `unlink`, or `update`") info.context.labbook_loader.prime( f"{get_logged_in_username()}&{labbook_owner}&{labbook_name}", lb) edge = LabbookConnection.Edge(node=Labbook(owner=labbook_owner, name=labbook_name), cursor=base64.b64encode( f"{0}".encode('utf-8'))) return ModifyDatasetLink(new_labbook_edge=edge)
def mutate_and_get_payload(cls, root, info, owner, dataset_name, local=False, remote=False, client_mutation_id=None): logged_in_user = get_logged_in_username() local_deleted = False remote_deleted = False if remote: logger.info(f"Deleting remote Dataset {owner}/{dataset_name}") # Extract valid Bearer token access_token = flask.g.get('access_token', None) id_token = flask.g.get('id_token', None) if not access_token or not id_token: raise ValueError( "Deleting a remote Dataset requires a valid session.") try: ds = InventoryManager().load_dataset( logged_in_user, owner, dataset_name, author=get_logged_in_author()) except InventoryException: raise ValueError( "A dataset must exist locally to delete it in the remote.") # Delete the dataset's files if supported if ds.is_managed(): ds.backend.set_default_configuration(logged_in_user, access_token, id_token) ds.backend.delete_contents(ds) # Get remote server configuration config = Configuration() remote_config = config.get_remote_configuration() # Delete the repository mgr = GitLabManager(remote_config['git_remote'], remote_config['admin_service'], access_token=access_token) mgr.remove_repository(owner, dataset_name) logger.info(f"Deleted {owner}/{dataset_name} repository from the" f" remote repository {remote_config['git_remote']}") # Call Index service to remove project from cloud index and search # Don't raise an exception if the index delete fails, since this can be handled relatively gracefully repo_id = mgr.get_repository_id(owner, dataset_name) response = requests.delete( f"https://{remote_config['index_service']}/index/{repo_id}", headers={ "Authorization": f"Bearer {access_token}", "Identity": id_token }, timeout=30) if response.status_code != 204: # Soft failure, still continue logger.error( f"Failed to remove {owner}/{dataset_name} from cloud index. " f"Status Code: {response.status_code}") logger.error(response.json()) else: logger.info( f"Deleted remote repository {owner}/{dataset_name} from cloud index" ) # Remove locally any references to that cloud repo that's just been deleted. try: ds.remove_remote() except GigantumException as e: logger.warning(e) remote_deleted = True if local: logger.info(f"Deleting local Dataset {owner}/{dataset_name}") # Delete the dataset dataset_delete_job = InventoryManager().delete_dataset( logged_in_user, owner, dataset_name) local_deleted = True # Schedule Job to clear file cache if dataset is no longer in use job_metadata = {'method': 'clean_dataset_file_cache'} job_kwargs = { 'logged_in_username': logged_in_user, 'dataset_owner': dataset_delete_job.namespace, 'dataset_name': dataset_delete_job.name, 'cache_location': dataset_delete_job.cache_root } dispatcher = Dispatcher() job_key = dispatcher.dispatch_task(jobs.clean_dataset_file_cache, metadata=job_metadata, kwargs=job_kwargs) logger.info( f"Dispatched clean_dataset_file_cache({owner}/{dataset_name}) to Job {job_key}" ) return DeleteDataset(local_deleted=local_deleted, remote_deleted=remote_deleted)
def mutate_and_get_payload(cls, root, info, owner, labbook_name, confirm, client_mutation_id=None): if confirm is True: # Load config data configuration = Configuration().config # Extract valid Bearer token token = None if hasattr(info.context.headers, 'environ'): if "HTTP_AUTHORIZATION" in info.context.headers.environ: token = parse_token( info.context.headers.environ["HTTP_AUTHORIZATION"]) if not token: raise ValueError( "Authorization header not provided. Cannot perform remote delete operation." ) # Get remote server configuration default_remote = configuration['git']['default_remote'] admin_service = None for remote in configuration['git']['remotes']: if default_remote == remote: admin_service = configuration['git']['remotes'][remote][ 'admin_service'] index_service = configuration['git']['remotes'][remote][ 'index_service'] break if not admin_service: raise ValueError('admin_service could not be found') # Perform delete operation mgr = GitLabManager(default_remote, admin_service, access_token=token) mgr.remove_repository(owner, labbook_name) logger.info( f"Deleted {owner}/{labbook_name} from the remote repository {default_remote}" ) # Call Index service to remove project from cloud index and search # Don't raise an exception if the index delete fails, since this can be handled relatively gracefully # for now, but do return success=false success = True access_token = flask.g.get('access_token', None) id_token = flask.g.get('id_token', None) repo_id = mgr.get_repository_id(owner, labbook_name) response = requests.delete( f"https://{index_service}/index/{repo_id}", headers={ "Authorization": f"Bearer {access_token}", "Identity": id_token }, timeout=10) if response.status_code != 204: logger.error(f"Failed to remove project from cloud index. " f"Status Code: {response.status_code}") logger.error(response.json()) else: logger.info( f"Deleted remote repository {owner}/{labbook_name} from cloud index" ) # Remove locally any references to that cloud repo that's just been deleted. try: username = get_logged_in_username() lb = InventoryManager().load_labbook( username, owner, labbook_name, author=get_logged_in_author()) lb.remove_remote() lb.remove_lfs_remotes() except GigantumException as e: logger.warning(e) return DeleteLabbook(success=True) else: logger.info( f"Dry run deleting {labbook_name} from remote repository -- not deleted." ) return DeleteLabbook(success=False)
def mutate_and_get_payload(cls, root, info, owner, dataset_name, pull_only=False, override_method="abort", client_mutation_id=None): # Load Dataset username = get_logged_in_username() ds = InventoryManager().load_dataset(username, owner, dataset_name, author=get_logged_in_author()) # Extract valid Bearer token token = None if hasattr(info.context.headers, 'environ'): if "HTTP_AUTHORIZATION" in info.context.headers.environ: token = parse_token( info.context.headers.environ["HTTP_AUTHORIZATION"]) if not token: raise ValueError( "Authorization header not provided. Must have a valid session to query for collaborators" ) default_remote = ds.client_config.config['git']['default_remote'] admin_service = None for remote in ds.client_config.config['git']['remotes']: if default_remote == remote: admin_service = ds.client_config.config['git']['remotes'][ remote]['admin_service'] break if not admin_service: raise ValueError('admin_service could not be found') # Configure git creds mgr = GitLabManager(default_remote, admin_service, access_token=token) mgr.configure_git_credentials(default_remote, username) override = MergeOverride(override_method) job_metadata = {'method': 'sync_dataset', 'dataset': ds.key} job_kwargs = { 'repository': ds, 'username': username, 'pull_only': pull_only, 'override': override, 'access_token': token, 'id_token': flask.g.id_token } dispatcher = Dispatcher() job_key = dispatcher.dispatch_task(jobs.sync_repository, kwargs=job_kwargs, metadata=job_metadata) logger.info( f"Syncing Dataset {ds.root_dir} in background job with key {job_key.key_str}" ) return SyncDataset(job_key=job_key.key_str)
def mutate_and_get_payload(cls, root, info, owner, dataset_name, username, permissions, client_mutation_id=None): # Here "username" refers to the intended recipient username. # Todo: it should probably be renamed here and in the frontend to "collaboratorUsername" logged_in_username = get_logged_in_username() lb = InventoryManager().load_dataset(logged_in_username, owner, dataset_name, author=get_logged_in_author()) # TODO: Future work will look up remote in LabBook data, allowing user to select remote. default_remote = lb.client_config.config['git']['default_remote'] admin_service = None for remote in lb.client_config.config['git']['remotes']: if default_remote == remote: admin_service = lb.client_config.config['git']['remotes'][ remote]['admin_service'] break # Extract valid Bearer token if "HTTP_AUTHORIZATION" in info.context.headers.environ: token = parse_token( info.context.headers.environ["HTTP_AUTHORIZATION"]) else: raise ValueError( "Authorization header not provided. " "Must have a valid session to query for collaborators") if permissions == 'readonly': perm = ProjectPermissions.READ_ONLY elif permissions == 'readwrite': perm = ProjectPermissions.READ_WRITE elif permissions == 'owner': perm = ProjectPermissions.OWNER else: raise ValueError(f"Unknown permission set: {permissions}") # Add collaborator to remote service mgr = GitLabManager(default_remote, admin_service, token) existing_collabs = mgr.get_collaborators(owner, dataset_name) if username not in [n[1] for n in existing_collabs]: logger.info(f"Adding user {username} to {owner}/{dataset_name}" f"with permission {perm}") mgr.add_collaborator(owner, dataset_name, username, perm) else: logger.warning(f"Changing permission of {username} on" f"{owner}/{dataset_name} to {perm}") mgr.delete_collaborator(owner, dataset_name, username) mgr.add_collaborator(owner, dataset_name, username, perm) create_data = {"owner": owner, "name": dataset_name} return AddDatasetCollaborator(updated_dataset=DatasetObject( **create_data))
def mutate_and_get_payload(cls, root, info, dataset_owner, dataset_name, parameters=None, confirm=None, client_mutation_id=None): logged_in_username = get_logged_in_username() im = InventoryManager() ds = im.load_dataset(logged_in_username, dataset_owner, dataset_name, get_logged_in_author()) ds.backend.set_default_configuration(logged_in_username, bearer_token=flask.g.access_token, id_token=flask.g.id_token) should_confirm = False error_message = None confirm_message = None background_job_key = None is_configured = None if confirm is None: if parameters: # Update the configuration current_config = ds.backend_config for param in parameters: current_config[param.parameter] = param.value ds.backend_config = current_config # Validate the configuration try: confirm_message = ds.backend.confirm_configuration(ds) if confirm_message is not None: should_confirm = True except ValueError as err: error_message = f"{err}" is_configured = False else: if confirm is False: # Clear configuration current_config = ds.backend_config for param in parameters: current_config[param.parameter] = None ds.backend_config = current_config else: if ds.backend.can_update_from_remote: d = Dispatcher() kwargs = { 'logged_in_username': logged_in_username, 'access_token': flask.g.access_token, 'id_token': flask.g.id_token, 'dataset_owner': dataset_owner, 'dataset_name': dataset_name, } # Gen unique keys for tracking jobs metadata = { 'dataset': f"{logged_in_username}|{dataset_owner}|{dataset_name}", 'method': 'update_unmanaged_dataset_from_remote' } job_response = d.dispatch_task( jobs.update_unmanaged_dataset_from_remote, kwargs=kwargs, metadata=metadata) background_job_key = job_response.key_str if is_configured is None: is_configured = ds.backend.is_configured return ConfigureDataset( dataset=Dataset(id="{}&{}".format(dataset_owner, dataset_name), name=dataset_name, owner=dataset_owner), is_configured=is_configured, should_confirm=should_confirm, confirm_message=confirm_message, error_message=error_message, has_background_job=ds.backend.can_update_from_remote, background_job_key=background_job_key)
def mutate_and_get_payload(cls, root, info, name, description, repository, component_id, revision, is_untracked=False, client_mutation_id=None): username = get_logged_in_username() # Create a new empty LabBook lb = LabBook(author=get_logged_in_author()) # TODO: Set owner/namespace properly once supported fully lb.new(owner={"username": username}, username=username, name=name, description=description, bypass_lfs=is_untracked) if is_untracked: FileOperations.set_untracked(lb, 'input') FileOperations.set_untracked(lb, 'output') input_set = FileOperations.is_set_untracked(lb, 'input') output_set = FileOperations.is_set_untracked(lb, 'output') if not (input_set and output_set): raise ValueError( f'{str(lb)} untracking for input/output in malformed state' ) if not lb.is_repo_clean: raise ValueError( f'{str(lb)} should have clean Git state after setting for untracked' ) # Create a Activity Store instance store = ActivityStore(lb) # Create detail record adr = ActivityDetailRecord(ActivityDetailType.LABBOOK, show=False, importance=0) adr.add_value('text/plain', f"Created new LabBook: {username}/{name}") # Create activity record ar = ActivityRecord(ActivityType.LABBOOK, message=f"Created new LabBook: {username}/{name}", show=True, importance=255, linked_commit=lb.git.commit_hash) ar.add_detail_object(adr) # Store store.create_activity_record(ar) # Add Base component cm = ComponentManager(lb) cm.add_component("base", repository, component_id, revision) # Prime dataloader with labbook you just created dataloader = LabBookLoader() dataloader.prime(f"{username}&{username}&{lb.name}", lb) # Get a graphene instance of the newly created LabBook return CreateLabbook(labbook=Labbook(owner=username, name=lb.name))