def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
section,
key,
updated_index=None,
updated_description=None,
client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username,
owner,
labbook_name,
author=get_logged_in_author())
with lb.lock():
new_favorite = lb.update_favorite(
section,
key,
new_description=updated_description,
new_index=updated_index)
# Create data to populate edge
create_data = {
"id": f"{owner}&{labbook_name}&{section}&{key}",
"owner": owner,
"section": section,
"key": key,
"_favorite_data": new_favorite
}
# Create dummy cursor
cursor = base64.b64encode(
f"{str(new_favorite['index'])}".encode('utf-8'))
return UpdateLabbookFavorite(
updated_favorite_edge=LabbookFavoriteConnection.Edge(
node=LabbookFavorite(**create_data), cursor=cursor))
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
section,
directory,
client_mutation_id=None):
username = get_logged_in_username()
working_directory = Configuration().config['git']['working_directory']
inferred_lb_directory = os.path.join(working_directory, username,
owner, 'labbooks', labbook_name)
lb = LabBook(author=get_logged_in_author())
lb.from_directory(inferred_lb_directory)
lb.makedir(os.path.join(section, directory),
create_activity_record=True)
logger.info(f"Made new directory in `{directory}`")
# Prime dataloader with labbook you already loaded
dataloader = LabBookLoader()
dataloader.prime(f"{owner}&{labbook_name}&{lb.name}", lb)
# Create data to populate edge
file_info = lb.get_file_info(section, directory)
create_data = {
'owner': owner,
'name': labbook_name,
'section': section,
'key': file_info['key'],
'_file_info': file_info
}
# TODO: Fix cursor implementation, this currently doesn't make sense
cursor = base64.b64encode(f"{0}".encode('utf-8'))
return MakeLabbookDirectory(
new_labbook_file_edge=LabbookFileConnection.Edge(
node=LabbookFile(**create_data), cursor=cursor))
def mutate_and_get_payload(cls, root, info, owner, labbook_name, visibility,
client_mutation_id=None):
# Load LabBook
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username, owner, labbook_name,
author=get_logged_in_author())
# Extract valid Bearer token
token = None
if hasattr(info.context.headers, 'environ'):
if "HTTP_AUTHORIZATION" in info.context.headers.environ:
token = parse_token(info.context.headers.environ["HTTP_AUTHORIZATION"])
if not token:
raise ValueError("Authorization header not provided. Must have a valid session to query for collaborators")
default_remote = lb.client_config.config['git']['default_remote']
admin_service = None
for remote in lb.client_config.config['git']['remotes']:
if default_remote == remote:
admin_service = lb.client_config.config['git']['remotes'][remote]['admin_service']
break
if not admin_service:
raise ValueError('admin_service could not be found')
# Configure git creds
mgr = GitLabManager(default_remote, admin_service, access_token=token)
mgr.configure_git_credentials(default_remote, username)
if visibility not in ['public', 'private']:
raise ValueError(f'Visibility must be either "public" or "private";'
f'("{visibility}" invalid)')
with lb.lock():
mgr.set_visibility(namespace=owner, repository_name=labbook_name, visibility=visibility)
cursor = base64.b64encode(f"{0}".encode('utf-8'))
lbedge = LabbookConnection.Edge(node=LabbookObject(owner=owner, name=labbook_name),
cursor=cursor)
return SetVisibility(new_labbook_edge=lbedge)
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
section,
key,
client_mutation_id=None):
username = get_logged_in_username()
lb = LabBook(author=get_logged_in_author())
lb.from_name(username, owner, labbook_name)
# Manually generate the Node ID for now. This simplifies the connection between the file browser and favorites
# widgets in the UI
favorite_node_id = f"LabbookFavorite:{owner}&{labbook_name}&{section}&{key}"
favorite_node_id = base64.b64encode(favorite_node_id.encode()).decode()
# Remove Favorite
lb.remove_favorite(section, key)
return RemoveLabbookFavorite(success=True,
removed_node_id=favorite_node_id)
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
username,
client_mutation_id=None):
logged_in_username = get_logged_in_username()
lb = InventoryManager().load_labbook(logged_in_username,
owner,
labbook_name,
author=get_logged_in_author())
# TODO: Future work will look up remote in LabBook data, allowing user to select remote.
default_remote = lb.client_config.config['git']['default_remote']
admin_service = None
for remote in lb.client_config.config['git']['remotes']:
if default_remote == remote:
admin_service = lb.client_config.config['git']['remotes'][
remote]['admin_service']
break
# Extract valid Bearer token
if "HTTP_AUTHORIZATION" in info.context.headers.environ:
token = parse_token(
info.context.headers.environ["HTTP_AUTHORIZATION"])
else:
raise ValueError(
"Authorization header not provided. Must have a valid session to query for collaborators"
)
# Add collaborator to remote service
mgr = GitLabManager(default_remote, admin_service, token)
mgr.delete_collaborator(owner, labbook_name, username)
create_data = {"owner": owner, "name": labbook_name}
return DeleteLabbookCollaborator(updated_labbook=Labbook(
**create_data))
def _start_jupyter_tool(cls, labbook: LabBook, router: ProxyRouter, username: str,
container_override_id: str = None):
tool_port = 8888
labbook_ip = ContainerOperations.get_labbook_ip(labbook, username)
labbook_endpoint = f'http://{labbook_ip}:{tool_port}'
matched_routes = router.get_matching_routes(labbook_endpoint, 'jupyter')
run_start_jupyter = True
suffix = None
if len(matched_routes) == 1:
logger.info(f'Found existing Jupyter instance in route table for {str(labbook)}.')
suffix = matched_routes[0]
# wait for jupyter to be up
try:
check_jupyter_reachable(labbook_ip, tool_port, suffix)
run_start_jupyter = False
except GigantumException:
logger.warning(f'Detected stale route. Attempting to restart Jupyter and clean up route table.')
router.remove(suffix[1:])
elif len(matched_routes) > 1:
raise ValueError(f"Multiple Jupyter instances found in route table for {str(labbook)}! Restart container.")
if run_start_jupyter:
rt_prefix = unique_id()
rt_prefix, _ = router.add(labbook_endpoint, f'jupyter/{rt_prefix}')
# Start jupyterlab
suffix = start_jupyter(labbook, username, tag=container_override_id, proxy_prefix=rt_prefix)
# Ensure we start monitor IFF jupyter isn't already running.
start_labbook_monitor(labbook, username, 'jupyterlab',
url=f'{labbook_endpoint}/{rt_prefix}',
author=get_logged_in_author())
return suffix
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
set_public=False,
client_mutation_id=None):
# Load LabBook
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username,
owner,
labbook_name,
author=get_logged_in_author())
# Extract valid Bearer token
if "HTTP_AUTHORIZATION" in info.context.headers.environ:
token = parse_token(
info.context.headers.environ["HTTP_AUTHORIZATION"])
else:
raise ValueError(
"Authorization header not provided. Must have a valid session to query for collaborators"
)
job_metadata = {'method': 'publish_labbook', 'labbook': lb.key}
job_kwargs = {
'repository': lb,
'username': username,
'access_token': token,
'public': set_public
}
dispatcher = Dispatcher()
job_key = dispatcher.dispatch_task(jobs.publish_repository,
kwargs=job_kwargs,
metadata=job_metadata)
logger.info(
f"Publishing LabBook {lb.root_dir} in background job with key {job_key.key_str}"
)
return PublishLabbook(job_key=job_key.key_str)
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
section,
directory,
client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username,
owner,
labbook_name,
author=get_logged_in_author())
with lb.lock():
FileOperations.makedir(lb,
os.path.join(section, directory),
create_activity_record=True)
# Prime dataloader with labbook you already loaded
dataloader = LabBookLoader()
dataloader.prime(f"{owner}&{labbook_name}&{lb.name}", lb)
# Create data to populate edge
file_info = FileOperations.get_file_info(lb, section, directory)
create_data = {
'owner': owner,
'name': labbook_name,
'section': section,
'key': file_info['key'],
'_file_info': file_info
}
# TODO: Fix cursor implementation, this currently doesn't make sense
cursor = base64.b64encode(f"{0}".encode('utf-8'))
return MakeLabbookDirectory(
new_labbook_file_edge=LabbookFileConnection.Edge(
node=LabbookFile(**create_data), cursor=cursor))
def mutate_and_get_payload(cls,
root,
info,
name,
description,
repository,
base_id,
revision,
is_untracked=False,
client_mutation_id=None):
username = get_logged_in_username()
inv_manager = InventoryManager()
lb = inv_manager.create_labbook(username=username,
owner=username,
labbook_name=name,
description=description,
author=get_logged_in_author())
adr = ActivityDetailRecord(ActivityDetailType.LABBOOK,
show=False,
importance=0)
adr.add_value('text/plain', f"Created new LabBook: {username}/{name}")
# Create activity record
ar = ActivityRecord(ActivityType.LABBOOK,
message=f"Created new LabBook: {username}/{name}",
show=True,
importance=255,
linked_commit=lb.git.commit_hash)
ar.add_detail_object(adr)
store = ActivityStore(lb)
store.create_activity_record(ar)
cm = ComponentManager(lb)
cm.add_base(repository, base_id, revision)
return CreateLabbook(labbook=Labbook(owner=username, name=lb.name))
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
title,
body=None,
tags=None,
client_mutation_id=None):
username = get_logged_in_username()
# Load LabBook instance
lb = LabBook(author=get_logged_in_author())
lb.from_name(username, owner, labbook_name)
# Create a Activity Store instance
store = ActivityStore(lb)
# Create detail record
adr = ActivityDetailRecord(ActivityDetailType.NOTE,
show=True,
importance=255)
if body:
adr.add_value('text/markdown', body)
# Create activity record
ar = ActivityRecord(ActivityType.NOTE,
message=title,
linked_commit="no-linked-commit",
importance=255,
tags=tags)
ar.add_detail_object(adr)
ar = store.create_activity_record(ar)
return CreateUserNote(new_activity_record_edge=ActivityConnection.Edge(
node=ActivityRecordObject(
owner=owner, name=labbook_name, commit=ar.commit),
cursor=ar.commit))
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
confirm,
client_mutation_id=None):
username = get_logged_in_username()
working_directory = Configuration().config['git']['working_directory']
inferred_lb_directory = os.path.join(working_directory, username,
owner, 'labbooks', labbook_name)
lb = LabBook(author=get_logged_in_author())
lb.from_directory(inferred_lb_directory)
if confirm:
logger.warning(f"Deleting {str(lb)}...")
try:
lb, stopped = ContainerOperations.stop_container(
labbook=lb, username=username)
except OSError:
pass
lb, docker_removed = ContainerOperations.delete_image(
labbook=lb, username=username)
if not docker_removed:
raise ValueError(
f'Cannot delete docker image for {str(lb)} - unable to delete LB from disk'
)
shutil.rmtree(lb.root_dir, ignore_errors=True)
if os.path.exists(lb.root_dir):
logger.error(
f'Deleted {str(lb)} but root directory {lb.root_dir} still exists!'
)
return DeleteLabbook(success=False)
else:
return DeleteLabbook(success=True)
else:
logger.info(f"Dry run in deleting {str(lb)} -- not deleted.")
return DeleteLabbook(success=False)
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
branch_name,
client_mutation_id=None):
"""Method to perform mutation
Args:
input:
context:
info:
Returns:
"""
username = get_logged_in_username()
# Load an existing LabBook
labbook_obj = LabBook(author=get_logged_in_author())
labbook_obj.from_name(username, owner, labbook_name)
# Checkout
labbook_obj.checkout_branch(branch_name)
return CheckoutBranch(labbook=Labbook(owner=owner, name=labbook_name))
def mutate_and_get_payload(cls, root, info, owner, labbook_name, other_branch_name,
override_method="abort", client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username, owner, labbook_name,
author=get_logged_in_author())
with lb.lock():
override = MergeOverride(override_method)
bm = BranchManager(lb, username=username)
if override == MergeOverride.ABORT:
bm.merge_from(other_branch=other_branch_name)
elif override == MergeOverride.OURS:
bm.merge_use_ours(other_branch=other_branch_name)
elif override == MergeOverride.THEIRS:
bm.merge_use_theirs(other_branch=other_branch_name)
else:
raise ValueError(f"Unknown override method {override}")
# Run update_linked_datasets() to initialize and cleanup dataset submodules. You don't need to schedule
# auto-import jobs because the user will have switched to the branch to pull it before merging.
InventoryManager().update_linked_datasets(lb, username)
return MergeFromBranch(Labbook(id="{}&{}".format(owner, labbook_name),
name=labbook_name, owner=owner))
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
branch_name,
client_mutation_id=None):
"""Method to perform mutation
Args:
input:
context:
info:
Returns:
"""
username = get_logged_in_username()
# Load an existing LabBook
labbook_obj = LabBook(author=get_logged_in_author())
labbook_obj.from_name(username, owner, labbook_name)
# Create Branch
labbook_obj.checkout_branch(branch_name, new=True)
if labbook_obj.active_branch != branch_name:
raise ValueError(
f"Create branch failed, could not switch to new branch {branch_name}"
)
# Create a LabbookRef to the branch
create_data = {
"owner": owner,
"name": labbook_name,
"prefix": None,
"branch": branch_name,
}
return CreateBranch(branch=LabbookRef(**create_data))
def mutate_and_get_payload(cls,
root,
info,
dataset_owner,
dataset_name,
key,
client_mutation_id=None):
logged_in_username = get_logged_in_username()
ds = InventoryManager().load_dataset(logged_in_username,
dataset_owner,
dataset_name,
author=get_logged_in_author())
ds.namespace = dataset_owner
m = Manifest(ds, logged_in_username)
if key[-1] != '/':
raise ValueError(
"Provided relative path must end in `/` to indicate it is a directory"
)
with ds.lock():
file_info = m.create_directory(key)
create_data = {
'owner': dataset_owner,
'name': dataset_name,
'key': file_info['key'],
'_file_info': file_info
}
# TODO: Fix cursor implementation, this currently doesn't make sense
cursor = base64.b64encode(f"{0}".encode('utf-8'))
return MakeDatasetDirectory(
new_dataset_file_edge=DatasetFileConnection.Edge(
node=DatasetFile(**create_data), cursor=cursor))
def mutate_and_get_payload(cls,
root,
info,
owner,
dataset_name,
client_mutation_id=None):
username = get_logged_in_username()
working_directory = Configuration().config['git']['working_directory']
ds = InventoryManager().load_dataset(username,
owner,
dataset_name,
author=get_logged_in_author())
job_metadata = {'method': 'export_dataset_as_zip', 'dataset': ds.key}
job_kwargs = {
'dataset_path': ds.root_dir,
'ds_export_directory': os.path.join(working_directory, 'export')
}
dispatcher = Dispatcher()
job_key = dispatcher.dispatch_task(jobs.export_dataset_as_zip,
kwargs=job_kwargs,
metadata=job_metadata)
return ExportDataset(job_key=job_key.key_str)
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
branch_name,
delete_local=False,
delete_remote=False,
client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username,
owner,
labbook_name,
author=get_logged_in_author())
with lb.lock():
bm = BranchManager(lb, username=username)
if delete_local:
bm.remove_branch(target_branch=branch_name)
if delete_remote:
bm.remove_remote_branch(target_branch=branch_name)
return DeleteExperimentalBranch(
Labbook(id="{}&{}".format(owner, labbook_name),
name=labbook_name,
owner=owner))
def helper_resolve_linked_datasets(labbook, info):
submodules = labbook.git.list_submodules()
datasets = list()
for submodule in submodules:
try:
namespace, dataset_name = submodule['name'].split("&")
submodule_dir = os.path.join(labbook.root_dir, '.gigantum', 'datasets', namespace, dataset_name)
ds = InventoryManager().load_dataset_from_directory(submodule_dir, author=get_logged_in_author())
ds.namespace = namespace
info.context.dataset_loader.prime(f"{get_logged_in_username()}&{namespace}&{dataset_name}", ds)
datasets.append(Dataset(owner=namespace, name=dataset_name))
except InventoryException:
continue
return datasets
def mutate_and_get_payload(cls,
root,
info,
dataset_owner,
dataset_name,
from_local=False,
from_remote=False,
client_mutation_id=None):
logged_in_username = get_logged_in_username()
im = InventoryManager()
ds = im.load_dataset(logged_in_username, dataset_owner, dataset_name,
get_logged_in_author())
ds.backend.set_default_configuration(logged_in_username,
bearer_token=flask.g.access_token,
id_token=flask.g.id_token)
if not ds.backend.is_configured:
raise ValueError("Dataset is not fully configured. Cannot update.")
d = Dispatcher()
kwargs = {
'logged_in_username': logged_in_username,
'access_token': flask.g.access_token,
'id_token': flask.g.id_token,
'dataset_owner': dataset_owner,
'dataset_name': dataset_name,
}
background_job_key = None
if from_remote is True:
if ds.backend.can_update_from_remote:
# Gen unique keys for tracking jobs
metadata = {
'dataset':
f"{logged_in_username}|{dataset_owner}|{dataset_name}",
'method': 'update_unmanaged_dataset_from_remote'
}
job_response = d.dispatch_task(
jobs.update_unmanaged_dataset_from_remote,
kwargs=kwargs,
metadata=metadata)
background_job_key = job_response.key_str
else:
raise ValueError(
"This dataset type does not support automatic update via querying its remote"
)
elif from_local is True:
# Gen unique keys for tracking jobs
metadata = {
'dataset':
f"{logged_in_username}|{dataset_owner}|{dataset_name}",
'method': 'update_unmanaged_dataset_from_local'
}
job_response = d.dispatch_task(
jobs.update_unmanaged_dataset_from_local,
kwargs=kwargs,
metadata=metadata)
background_job_key = job_response.key_str
else:
ValueError("Either `fromRemote` or `fromLocal` must be True.")
return UpdateUnmanagedDataset(dataset=Dataset(id="{}&{}".format(
dataset_owner, dataset_name),
name=dataset_name,
owner=dataset_owner),
background_job_key=background_job_key)
def mutate_and_get_payload(cls,
root,
info,
labbook_owner,
labbook_name,
dataset_owner,
dataset_name,
action,
dataset_url=None,
client_mutation_id=None):
logged_in_username = get_logged_in_username()
im = InventoryManager()
lb = im.load_labbook(logged_in_username,
labbook_owner,
labbook_name,
author=get_logged_in_author())
with lb.lock():
if action == 'link':
if dataset_url:
remote_domain = cls._get_remote_domain(
dataset_url, dataset_owner, dataset_name)
if remote_domain:
# Make sure git creds are configured for the remote
admin_service = None
for remote in lb.client_config.config['git'][
'remotes']:
if remote_domain == remote:
admin_service = lb.client_config.config['git'][
'remotes'][remote]['admin_service']
break
if "HTTP_AUTHORIZATION" in info.context.headers.environ:
token = parse_token(info.context.headers.
environ["HTTP_AUTHORIZATION"])
else:
raise ValueError(
"Authorization header not provided."
" Must have a valid session to query for collaborators"
)
mgr = GitLabManager(remote_domain, admin_service,
token)
mgr.configure_git_credentials(remote_domain,
logged_in_username)
else:
# Link to local dataset
ds = im.load_dataset(logged_in_username, dataset_owner,
dataset_name)
dataset_url = f"{ds.root_dir}/.git"
# Link the dataset to the labbook
ds = im.link_dataset_to_labbook(dataset_url, dataset_owner,
dataset_name, lb)
ds.namespace = dataset_owner
# Preload the dataloader
info.context.dataset_loader.prime(
f"{get_logged_in_username()}&{dataset_owner}&{dataset_name}",
ds)
# Relink the revision
m = Manifest(ds, logged_in_username)
m.link_revision()
elif action == 'unlink':
im.unlink_dataset_from_labbook(dataset_owner, dataset_name, lb)
elif action == 'update':
ds = im.update_linked_dataset_reference(
dataset_owner, dataset_name, lb)
m = Manifest(ds, logged_in_username)
m.force_reload()
info.context.dataset_loader.prime(
f"{get_logged_in_username()}&{dataset_owner}&{dataset_name}",
ds)
else:
raise ValueError(
"Unsupported action. Use `link`, `unlink`, or `update`")
info.context.labbook_loader.prime(
f"{get_logged_in_username()}&{labbook_owner}&{labbook_name}",
lb)
edge = LabbookConnection.Edge(node=Labbook(owner=labbook_owner,
name=labbook_name),
cursor=base64.b64encode(
f"{0}".encode('utf-8')))
return ModifyDatasetLink(new_labbook_edge=edge)
def mutate_and_get_payload(cls,
root,
info,
owner,
dataset_name,
local=False,
remote=False,
client_mutation_id=None):
logged_in_user = get_logged_in_username()
local_deleted = False
remote_deleted = False
if remote:
logger.info(f"Deleting remote Dataset {owner}/{dataset_name}")
# Extract valid Bearer token
access_token = flask.g.get('access_token', None)
id_token = flask.g.get('id_token', None)
if not access_token or not id_token:
raise ValueError(
"Deleting a remote Dataset requires a valid session.")
try:
ds = InventoryManager().load_dataset(
logged_in_user,
owner,
dataset_name,
author=get_logged_in_author())
except InventoryException:
raise ValueError(
"A dataset must exist locally to delete it in the remote.")
# Delete the dataset's files if supported
if ds.is_managed():
ds.backend.set_default_configuration(logged_in_user,
access_token, id_token)
ds.backend.delete_contents(ds)
# Get remote server configuration
config = Configuration()
remote_config = config.get_remote_configuration()
# Delete the repository
mgr = GitLabManager(remote_config['git_remote'],
remote_config['admin_service'],
access_token=access_token)
mgr.remove_repository(owner, dataset_name)
logger.info(f"Deleted {owner}/{dataset_name} repository from the"
f" remote repository {remote_config['git_remote']}")
# Call Index service to remove project from cloud index and search
# Don't raise an exception if the index delete fails, since this can be handled relatively gracefully
repo_id = mgr.get_repository_id(owner, dataset_name)
response = requests.delete(
f"https://{remote_config['index_service']}/index/{repo_id}",
headers={
"Authorization": f"Bearer {access_token}",
"Identity": id_token
},
timeout=30)
if response.status_code != 204:
# Soft failure, still continue
logger.error(
f"Failed to remove {owner}/{dataset_name} from cloud index. "
f"Status Code: {response.status_code}")
logger.error(response.json())
else:
logger.info(
f"Deleted remote repository {owner}/{dataset_name} from cloud index"
)
# Remove locally any references to that cloud repo that's just been deleted.
try:
ds.remove_remote()
except GigantumException as e:
logger.warning(e)
remote_deleted = True
if local:
logger.info(f"Deleting local Dataset {owner}/{dataset_name}")
# Delete the dataset
dataset_delete_job = InventoryManager().delete_dataset(
logged_in_user, owner, dataset_name)
local_deleted = True
# Schedule Job to clear file cache if dataset is no longer in use
job_metadata = {'method': 'clean_dataset_file_cache'}
job_kwargs = {
'logged_in_username': logged_in_user,
'dataset_owner': dataset_delete_job.namespace,
'dataset_name': dataset_delete_job.name,
'cache_location': dataset_delete_job.cache_root
}
dispatcher = Dispatcher()
job_key = dispatcher.dispatch_task(jobs.clean_dataset_file_cache,
metadata=job_metadata,
kwargs=job_kwargs)
logger.info(
f"Dispatched clean_dataset_file_cache({owner}/{dataset_name}) to Job {job_key}"
)
return DeleteDataset(local_deleted=local_deleted,
remote_deleted=remote_deleted)
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
confirm,
client_mutation_id=None):
if confirm is True:
# Load config data
configuration = Configuration().config
# Extract valid Bearer token
token = None
if hasattr(info.context.headers, 'environ'):
if "HTTP_AUTHORIZATION" in info.context.headers.environ:
token = parse_token(
info.context.headers.environ["HTTP_AUTHORIZATION"])
if not token:
raise ValueError(
"Authorization header not provided. Cannot perform remote delete operation."
)
# Get remote server configuration
default_remote = configuration['git']['default_remote']
admin_service = None
for remote in configuration['git']['remotes']:
if default_remote == remote:
admin_service = configuration['git']['remotes'][remote][
'admin_service']
index_service = configuration['git']['remotes'][remote][
'index_service']
break
if not admin_service:
raise ValueError('admin_service could not be found')
# Perform delete operation
mgr = GitLabManager(default_remote,
admin_service,
access_token=token)
mgr.remove_repository(owner, labbook_name)
logger.info(
f"Deleted {owner}/{labbook_name} from the remote repository {default_remote}"
)
# Call Index service to remove project from cloud index and search
# Don't raise an exception if the index delete fails, since this can be handled relatively gracefully
# for now, but do return success=false
success = True
access_token = flask.g.get('access_token', None)
id_token = flask.g.get('id_token', None)
repo_id = mgr.get_repository_id(owner, labbook_name)
response = requests.delete(
f"https://{index_service}/index/{repo_id}",
headers={
"Authorization": f"Bearer {access_token}",
"Identity": id_token
},
timeout=10)
if response.status_code != 204:
logger.error(f"Failed to remove project from cloud index. "
f"Status Code: {response.status_code}")
logger.error(response.json())
else:
logger.info(
f"Deleted remote repository {owner}/{labbook_name} from cloud index"
)
# Remove locally any references to that cloud repo that's just been deleted.
try:
username = get_logged_in_username()
lb = InventoryManager().load_labbook(
username,
owner,
labbook_name,
author=get_logged_in_author())
lb.remove_remote()
lb.remove_lfs_remotes()
except GigantumException as e:
logger.warning(e)
return DeleteLabbook(success=True)
else:
logger.info(
f"Dry run deleting {labbook_name} from remote repository -- not deleted."
)
return DeleteLabbook(success=False)
def mutate_and_get_payload(cls,
root,
info,
owner,
dataset_name,
pull_only=False,
override_method="abort",
client_mutation_id=None):
# Load Dataset
username = get_logged_in_username()
ds = InventoryManager().load_dataset(username,
owner,
dataset_name,
author=get_logged_in_author())
# Extract valid Bearer token
token = None
if hasattr(info.context.headers, 'environ'):
if "HTTP_AUTHORIZATION" in info.context.headers.environ:
token = parse_token(
info.context.headers.environ["HTTP_AUTHORIZATION"])
if not token:
raise ValueError(
"Authorization header not provided. Must have a valid session to query for collaborators"
)
default_remote = ds.client_config.config['git']['default_remote']
admin_service = None
for remote in ds.client_config.config['git']['remotes']:
if default_remote == remote:
admin_service = ds.client_config.config['git']['remotes'][
remote]['admin_service']
break
if not admin_service:
raise ValueError('admin_service could not be found')
# Configure git creds
mgr = GitLabManager(default_remote, admin_service, access_token=token)
mgr.configure_git_credentials(default_remote, username)
override = MergeOverride(override_method)
job_metadata = {'method': 'sync_dataset', 'dataset': ds.key}
job_kwargs = {
'repository': ds,
'username': username,
'pull_only': pull_only,
'override': override,
'access_token': token,
'id_token': flask.g.id_token
}
dispatcher = Dispatcher()
job_key = dispatcher.dispatch_task(jobs.sync_repository,
kwargs=job_kwargs,
metadata=job_metadata)
logger.info(
f"Syncing Dataset {ds.root_dir} in background job with key {job_key.key_str}"
)
return SyncDataset(job_key=job_key.key_str)
def mutate_and_get_payload(cls,
root,
info,
owner,
dataset_name,
username,
permissions,
client_mutation_id=None):
# Here "username" refers to the intended recipient username.
# Todo: it should probably be renamed here and in the frontend to "collaboratorUsername"
logged_in_username = get_logged_in_username()
lb = InventoryManager().load_dataset(logged_in_username,
owner,
dataset_name,
author=get_logged_in_author())
# TODO: Future work will look up remote in LabBook data, allowing user to select remote.
default_remote = lb.client_config.config['git']['default_remote']
admin_service = None
for remote in lb.client_config.config['git']['remotes']:
if default_remote == remote:
admin_service = lb.client_config.config['git']['remotes'][
remote]['admin_service']
break
# Extract valid Bearer token
if "HTTP_AUTHORIZATION" in info.context.headers.environ:
token = parse_token(
info.context.headers.environ["HTTP_AUTHORIZATION"])
else:
raise ValueError(
"Authorization header not provided. "
"Must have a valid session to query for collaborators")
if permissions == 'readonly':
perm = ProjectPermissions.READ_ONLY
elif permissions == 'readwrite':
perm = ProjectPermissions.READ_WRITE
elif permissions == 'owner':
perm = ProjectPermissions.OWNER
else:
raise ValueError(f"Unknown permission set: {permissions}")
# Add collaborator to remote service
mgr = GitLabManager(default_remote, admin_service, token)
existing_collabs = mgr.get_collaborators(owner, dataset_name)
if username not in [n[1] for n in existing_collabs]:
logger.info(f"Adding user {username} to {owner}/{dataset_name}"
f"with permission {perm}")
mgr.add_collaborator(owner, dataset_name, username, perm)
else:
logger.warning(f"Changing permission of {username} on"
f"{owner}/{dataset_name} to {perm}")
mgr.delete_collaborator(owner, dataset_name, username)
mgr.add_collaborator(owner, dataset_name, username, perm)
create_data = {"owner": owner, "name": dataset_name}
return AddDatasetCollaborator(updated_dataset=DatasetObject(
**create_data))
def mutate_and_get_payload(cls,
root,
info,
dataset_owner,
dataset_name,
parameters=None,
confirm=None,
client_mutation_id=None):
logged_in_username = get_logged_in_username()
im = InventoryManager()
ds = im.load_dataset(logged_in_username, dataset_owner, dataset_name,
get_logged_in_author())
ds.backend.set_default_configuration(logged_in_username,
bearer_token=flask.g.access_token,
id_token=flask.g.id_token)
should_confirm = False
error_message = None
confirm_message = None
background_job_key = None
is_configured = None
if confirm is None:
if parameters:
# Update the configuration
current_config = ds.backend_config
for param in parameters:
current_config[param.parameter] = param.value
ds.backend_config = current_config
# Validate the configuration
try:
confirm_message = ds.backend.confirm_configuration(ds)
if confirm_message is not None:
should_confirm = True
except ValueError as err:
error_message = f"{err}"
is_configured = False
else:
if confirm is False:
# Clear configuration
current_config = ds.backend_config
for param in parameters:
current_config[param.parameter] = None
ds.backend_config = current_config
else:
if ds.backend.can_update_from_remote:
d = Dispatcher()
kwargs = {
'logged_in_username': logged_in_username,
'access_token': flask.g.access_token,
'id_token': flask.g.id_token,
'dataset_owner': dataset_owner,
'dataset_name': dataset_name,
}
# Gen unique keys for tracking jobs
metadata = {
'dataset':
f"{logged_in_username}|{dataset_owner}|{dataset_name}",
'method': 'update_unmanaged_dataset_from_remote'
}
job_response = d.dispatch_task(
jobs.update_unmanaged_dataset_from_remote,
kwargs=kwargs,
metadata=metadata)
background_job_key = job_response.key_str
if is_configured is None:
is_configured = ds.backend.is_configured
return ConfigureDataset(
dataset=Dataset(id="{}&{}".format(dataset_owner, dataset_name),
name=dataset_name,
owner=dataset_owner),
is_configured=is_configured,
should_confirm=should_confirm,
confirm_message=confirm_message,
error_message=error_message,
has_background_job=ds.backend.can_update_from_remote,
background_job_key=background_job_key)
def mutate_and_get_payload(cls,
root,
info,
name,
description,
repository,
component_id,
revision,
is_untracked=False,
client_mutation_id=None):
username = get_logged_in_username()
# Create a new empty LabBook
lb = LabBook(author=get_logged_in_author())
# TODO: Set owner/namespace properly once supported fully
lb.new(owner={"username": username},
username=username,
name=name,
description=description,
bypass_lfs=is_untracked)
if is_untracked:
FileOperations.set_untracked(lb, 'input')
FileOperations.set_untracked(lb, 'output')
input_set = FileOperations.is_set_untracked(lb, 'input')
output_set = FileOperations.is_set_untracked(lb, 'output')
if not (input_set and output_set):
raise ValueError(
f'{str(lb)} untracking for input/output in malformed state'
)
if not lb.is_repo_clean:
raise ValueError(
f'{str(lb)} should have clean Git state after setting for untracked'
)
# Create a Activity Store instance
store = ActivityStore(lb)
# Create detail record
adr = ActivityDetailRecord(ActivityDetailType.LABBOOK,
show=False,
importance=0)
adr.add_value('text/plain', f"Created new LabBook: {username}/{name}")
# Create activity record
ar = ActivityRecord(ActivityType.LABBOOK,
message=f"Created new LabBook: {username}/{name}",
show=True,
importance=255,
linked_commit=lb.git.commit_hash)
ar.add_detail_object(adr)
# Store
store.create_activity_record(ar)
# Add Base component
cm = ComponentManager(lb)
cm.add_component("base", repository, component_id, revision)
# Prime dataloader with labbook you just created
dataloader = LabBookLoader()
dataloader.prime(f"{username}&{username}&{lb.name}", lb)
# Get a graphene instance of the newly created LabBook
return CreateLabbook(labbook=Labbook(owner=username, name=lb.name))
