def mutate_and_get_payload(cls,
root,
info,
owner,
dataset_name,
transaction_id,
cancel=False,
rollback=False,
client_mutation_id=None):
username = get_logged_in_username()
ds = InventoryManager().load_dataset(username,
owner,
dataset_name,
author=get_logged_in_author())
with ds.lock():
if cancel and rollback:
logger.warning(
f"Cancelled tx {transaction_id}, doing git reset")
# TODO: Add ability to reset
else:
logger.info(
f"Done batch upload {transaction_id}, cancelled={cancel}")
if cancel:
logger.warning("Sweeping aborted batch upload.")
m = "Cancelled upload `{transaction_id}`. " if cancel else ''
# Sweep up and process all files added during upload
manifest = Manifest(ds, username)
manifest.sweep_all_changes(upload=True, extra_msg=m)
return CompleteDatasetUploadTransaction(success=True)
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
other_branch_name,
override_method="abort",
client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username,
owner,
labbook_name,
author=get_logged_in_author())
with lb.lock():
override = MergeOverride(override_method)
bm = BranchManager(lb, username=username)
if override == MergeOverride.ABORT:
bm.merge_from(other_branch=other_branch_name)
elif override == MergeOverride.OURS:
bm.merge_use_ours(other_branch=other_branch_name)
elif override == MergeOverride.THEIRS:
bm.merge_use_theirs(other_branch=other_branch_name)
else:
raise ValueError(f"Unknown override method {override}")
return MergeFromBranch(
Labbook(id="{}&{}".format(owner, labbook_name),
name=labbook_name,
owner=owner))
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
branch_name,
revision=None,
description=None,
client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username,
owner,
labbook_name,
author=get_logged_in_author())
with lb.lock():
bm = BranchManager(lb, username=username)
full_branch_title = bm.create_branch(title=branch_name,
revision=revision)
logger.info(
f"In {str(lb)} created new experimental feature branch: "
f"{full_branch_title}")
if description:
cls._update_branch_description(lb, description)
return CreateExperimentalBranch(
Labbook(id="{}&{}".format(owner, labbook_name),
name=labbook_name,
owner=owner))
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
filename,
mount_path,
client_mutation_id=None):
if len(mount_path) > 0 and mount_path[0] == '~':
mount_path = mount_path.replace('~', '/home/giguser')
if not cls._is_target_valid(mount_path):
raise ValueError(f"Mount path {mount_path} is not a valid path")
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username,
owner,
labbook_name,
author=get_logged_in_author())
with lb.lock():
secstore = SecretStore(lb, username)
secstore[filename] = mount_path
cls._record_insert_activity(secstore, filename, lb, mount_path)
env = Environment(owner=owner, name=lb.name)
return InsertSecretsEntry(environment=env)
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username,
owner,
labbook_name,
author=get_logged_in_author())
migrated = False
with lb.lock():
t0 = time.time()
workflow = LabbookWorkflow(lb)
migrated = workflow.migrate()
tf = time.time()
if migrated:
logger.info(f"Migrated {str(lb)} in {tf-t0:.2}sec")
else:
logger.info(f"No migration needed for {str(lb)}")
return MigrateLabbookSchema(
Labbook(id=f"{owner}&{labbook_name}",
name=labbook_name,
owner=owner))
def savehook(username, owner, labbook_name):
try:
changed_file = request.args.get('file')
jupyter_token = request.args.get('jupyter_token')
logger.debug(f"Received save hook for {changed_file} in {username}/{owner}/{labbook_name}")
redis_conn = redis.Redis(db=1)
lb_jupyter_token_key = '-'.join(['gmlb', username, owner, labbook_name, 'jupyter-token'])
lb_active_key = f"{'|'.join([username, owner, labbook_name])}&is-busy*"
r = redis_conn.get(lb_jupyter_token_key.encode())
if r is None:
logger.error(f"Could not find jupyter token for {username}/{owner}/{labbook_name}")
abort(400)
if r.decode() != jupyter_token:
raise ValueError("Incoming jupyter token must be valid")
if len(redis_conn.keys(lb_active_key.encode())) > 0:
# A kernel in this project is still active. Don't save auto-commit because it can blow up the
# repository size depending on what the user is doing
logger.info(f"Skipping jupyter savehook for {username}/{owner}/{labbook_name} due to active kernel")
return 'success'
lb = InventoryManager().load_labbook(username, owner, labbook_name,
author=get_logged_in_author())
with lb.lock():
lb.sweep_uncommitted_changes()
logger.info(f"Jupyter save hook saved {changed_file} from {str(lb)}")
return 'success'
except Exception as err:
logger.error(err)
return abort(400)
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
description_content,
client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username,
owner,
labbook_name,
author=get_logged_in_author())
lb.description = description_content
with lb.lock():
lb.git.add(os.path.join(lb.config_path))
commit = lb.git.commit('Updating description')
adr = ActivityDetailRecord(ActivityDetailType.LABBOOK, show=False)
adr.add_value('text/plain', "Updated description of Project")
ar = ActivityRecord(ActivityType.LABBOOK,
message="Updated description of Project",
linked_commit=commit.hexsha,
tags=["labbook"],
show=False)
ar.add_detail_object(adr)
ars = ActivityStore(lb)
ars.create_activity_record(ar)
return SetLabbookDescription(success=True)
def mutate_and_get_payload(cls, root, info, owner, labbook_name, section, directory,
client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username, owner, labbook_name,
author=get_logged_in_author())
with lb.lock():
FileOperations.makedir(lb, os.path.join(section, directory), create_activity_record=True)
# Prime dataloader with labbook you already loaded
dataloader = LabBookLoader()
dataloader.prime(f"{owner}&{labbook_name}&{lb.name}", lb)
# Create data to populate edge
file_info = FileOperations.get_file_info(lb, section, directory)
create_data = {'owner': owner,
'name': labbook_name,
'section': section,
'key': file_info['key'],
'_file_info': file_info}
# TODO: Fix cursor implementation, this currently doesn't make sense
cursor = base64.b64encode(f"{0}".encode('utf-8'))
return MakeLabbookDirectory(
new_labbook_file_edge=LabbookFileConnection.Edge(
node=LabbookFile(**create_data),
cursor=cursor))
def mutate_and_get_payload(cls, root, info, owner, labbook_name, section, key, description=None, is_dir=False,
client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username, owner, labbook_name,
author=get_logged_in_author())
# Add Favorite
if is_dir:
is_dir = is_dir
# Make sure trailing slashes are always present when favoriting a dir
if key[-1] != "/":
key = f"{key}/"
with lb.lock():
new_favorite = lb.create_favorite(section, key, description=description, is_dir=is_dir)
# Create data to populate edge
create_data = {"id": f"{owner}&{labbook_name}&{section}&{key}",
"owner": owner,
"section": section,
"name": labbook_name,
"key": key,
"index": new_favorite['index'],
"_favorite_data": new_favorite}
# Create cursor
cursor = base64.b64encode(f"{str(new_favorite['index'])}".encode('utf-8'))
return AddLabbookFavorite(new_favorite_edge=LabbookFavoriteConnection.Edge(node=LabbookFavorite(**create_data),
cursor=cursor))
def mutate_and_get_payload(cls, root, info, owner, labbook_name, section, src_path, dst_path,
client_mutation_id=None, **kwargs):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username, owner, labbook_name,
author=get_logged_in_author())
with lb.lock():
mv_results = FileOperations.move_file(lb, section, src_path, dst_path)
file_edges = list()
for file_dict in mv_results:
file_edges.append(LabbookFile(owner=owner,
name=labbook_name,
section=section,
key=file_dict['key'],
is_dir=file_dict['is_dir'],
is_favorite=file_dict['is_favorite'],
modified_at=file_dict['modified_at'],
size=str(file_dict['size'])))
cursors = [base64.b64encode("{}".format(cnt).encode("UTF-8")).decode("UTF-8")
for cnt, x in enumerate(file_edges)]
edge_objs = [LabbookFileConnection.Edge(node=e, cursor=c) for e, c in zip(file_edges, cursors)]
return MoveLabbookFile(updated_edges=edge_objs)
def mutate_and_get_payload(cls,
root,
info,
owner,
dataset_name,
description,
client_mutation_id=None):
username = get_logged_in_username()
ds = InventoryManager().load_dataset(username,
owner,
dataset_name,
author=get_logged_in_author())
ds.description = description
with ds.lock():
ds.git.add(os.path.join(ds.root_dir, '.gigantum/gigantum.yaml'))
commit = ds.git.commit('Updating description')
adr = ActivityDetailRecord(ActivityDetailType.LABBOOK, show=False)
adr.add_value('text/plain',
f"Updated Dataset description: {description}")
ar = ActivityRecord(ActivityType.LABBOOK,
message="Updated Dataset description",
linked_commit=commit.hexsha,
tags=["dataset"],
show=False)
ar.add_detail_object(adr)
ars = ActivityStore(ds)
ars.create_activity_record(ar)
return SetDatasetDescription(
updated_dataset=Dataset(owner=owner, name=dataset_name))
def mutate_and_process_upload(cls, info, upload_file_path, upload_filename,
**kwargs):
if not upload_file_path:
logger.error('No file uploaded')
raise ValueError('No file uploaded')
username = get_logged_in_username()
owner = kwargs.get('owner')
dataset_name = kwargs.get('dataset_name')
file_path = kwargs.get('file_path')
try:
ds = InventoryManager().load_dataset(username,
owner,
dataset_name,
author=get_logged_in_author())
with ds.lock():
if not os.path.abspath(upload_file_path):
raise ValueError(
f"Source file `{upload_file_path}` not an absolute path"
)
if not os.path.isfile(upload_file_path):
raise ValueError(
f"Source file does not exist at `{upload_file_path}`")
manifest = Manifest(ds, username)
full_dst = manifest.get_abs_path(file_path)
# If file (hard link) already exists, remove it first so you don't write to all files with same content
if os.path.isfile(full_dst):
os.remove(full_dst)
full_dst_base = os.path.dirname(full_dst)
if not os.path.isdir(full_dst_base):
pathlib.Path(full_dst_base).mkdir(parents=True,
exist_ok=True)
shutil.move(upload_file_path, full_dst)
file_info = manifest.gen_file_info(file_path)
finally:
try:
logger.debug(f"Removing temp file {upload_file_path}")
os.remove(upload_file_path)
except FileNotFoundError:
pass
# Create data to populate edge
create_data = {
'owner': owner,
'name': dataset_name,
'key': file_info['key'],
'_file_info': file_info
}
# TODO: Fix cursor implementation. this currently doesn't make sense when adding edges
cursor = base64.b64encode(f"{0}".encode('utf-8'))
return AddDatasetFile(new_dataset_file_edge=DatasetFileConnection.Edge(
node=DatasetFile(**create_data), cursor=cursor))
def mutate_and_get_payload(cls, root, info, owner, labbook_name, client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username, owner, labbook_name,
author=get_logged_in_author())
with lb.lock():
cls._stop_container(lb, username)
return StopContainer(environment=Environment(owner=owner, name=labbook_name))
def mutate_and_get_payload(cls, root, info, owner, labbook_name, branch_name, client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username, owner, labbook_name,
author=get_logged_in_author())
with lb.lock():
wf = LabbookWorkflow(lb)
wf.checkout(username, branch_name)
return WorkonBranch(Labbook(id="{}&{}".format(owner, labbook_name),
name=labbook_name, owner=owner))
def mutate_and_get_payload(cls, root, info, owner, labbook_name, section, file_paths, client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username, owner, labbook_name,
author=get_logged_in_author())
with lb.lock():
FileOperations.delete_files(lb, section, file_paths)
return DeleteLabbookFiles(success=True)
def mutate_and_get_payload(cls, root, info, owner, labbook_name, content, client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username, owner, labbook_name,
author=get_logged_in_author())
# Write data
with lb.lock():
lb.write_readme(content)
return WriteLabbookReadme(updated_labbook=Labbook(owner=owner, name=labbook_name))
def mutate_and_get_payload(cls, root, info, owner, labbook_name,
transaction_id, cancel=False, rollback=False,
client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username, owner, labbook_name,
author=get_logged_in_author())
with lb.lock():
FileOperations.complete_batch(lb, transaction_id, cancel=cancel,
rollback=rollback)
return CompleteBatchUploadTransaction(success=True)
def mutate_and_get_payload(cls, root: str, info: str, owner: str, labbook_name: str, dev_tool: str,
container_override_id: str = None, client_mutation_id: str = None):
username = get_logged_in_username()
labbook = InventoryManager().load_labbook(username, owner, labbook_name,
author=get_logged_in_author())
with labbook.lock(failfast=True):
path = cls._start_dev_tool(labbook, username, dev_tool.lower(), container_override_id)
return StartDevTool(path=path)
def mutate_and_get_payload(cls,
root,
info,
owner,
dataset_name,
visibility,
client_mutation_id=None):
# Load Dataset
username = get_logged_in_username()
ds = InventoryManager().load_dataset(username,
owner,
dataset_name,
author=get_logged_in_author())
# Extract valid Bearer token
token = None
if hasattr(info.context.headers, 'environ'):
if "HTTP_AUTHORIZATION" in info.context.headers.environ:
token = parse_token(
info.context.headers.environ["HTTP_AUTHORIZATION"])
if not token:
raise ValueError(
"Authorization header not provided. Must have a valid session to query for collaborators"
)
default_remote = ds.client_config.config['git']['default_remote']
admin_service = None
for remote in ds.client_config.config['git']['remotes']:
if default_remote == remote:
admin_service = ds.client_config.config['git']['remotes'][
remote]['admin_service']
break
if not admin_service:
raise ValueError('admin_service could not be found')
# Configure git creds
mgr = GitLabManager(default_remote, admin_service, access_token=token)
mgr.configure_git_credentials(default_remote, username)
if visibility not in ['public', 'private']:
raise ValueError(
f'Visibility must be either "public" or "private";'
f'("{visibility}" invalid)')
with ds.lock():
mgr.set_visibility(namespace=owner,
repository_name=dataset_name,
visibility=visibility)
cursor = base64.b64encode(f"{0}".encode('utf-8'))
dsedge = DatasetConnection.Edge(node=DatasetObject(owner=owner,
name=dataset_name),
cursor=cursor)
return SetDatasetVisibility(new_dataset_edge=dsedge)
def mutate_and_get_payload(cls, root, info, owner, labbook_name, client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username, owner, labbook_name,
author=get_logged_in_author())
with lb.lock():
lb, container_id = ContainerOperations.start_container(
labbook=lb, username=username)
logger.info(f'Started new {lb} container ({container_id})')
return StartContainer(environment=Environment(owner=owner, name=labbook_name))
def mutate_and_get_payload(cls, root, info, owner, labbook_name, branch_name, delete_local=False,
delete_remote=False, client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username, owner, labbook_name,
author=get_logged_in_author())
with lb.lock():
bm = BranchManager(lb, username=username)
if delete_local:
bm.remove_branch(target_branch=branch_name)
if delete_remote:
bm.remove_remote_branch(target_branch=branch_name)
return DeleteExperimentalBranch(Labbook(id="{}&{}".format(owner, labbook_name),
name=labbook_name, owner=owner))
def mutate_and_get_payload(cls, root, info, owner, labbook_name, section, key, client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username, owner, labbook_name,
author=get_logged_in_author())
# Manually generate the Node ID for now. This simplifies the connection between the file browser and favorites
# widgets in the UI
favorite_node_id = f"LabbookFavorite:{owner}&{labbook_name}&{section}&{key}"
favorite_node_id = base64.b64encode(favorite_node_id.encode()).decode()
with lb.lock():
lb.remove_favorite(section, key)
return RemoveLabbookFavorite(success=True, removed_node_id=favorite_node_id)
def export_dataset_as_zip(dataset_path: str, ds_export_directory: str) -> str:
"""Return path to archive file of exported dataset. """
p = os.getpid()
logger = LMLogger.get_logger()
logger.info(f"(Job {p}) Starting export_dataset_as_zip({dataset_path})")
try:
ds = InventoryManager().load_dataset_from_directory(dataset_path)
with ds.lock():
path = ZipExporter.export_dataset(ds.root_dir, ds_export_directory)
return path
except Exception as e:
logger.exception(f"(Job {p}) Error on export_dataset_as_zip: {e}")
raise
def export_labbook_as_zip(labbook_path: str, lb_export_directory: str) -> str:
"""Return path to archive file of exported labbook. """
p = os.getpid()
logger = LMLogger.get_logger()
logger.info(f"(Job {p}) Starting export_labbook_as_zip({labbook_path})")
try:
lb = InventoryManager().load_labbook_from_directory(labbook_path)
with lb.lock():
path = ZipExporter.export_labbook(lb.root_dir, lb_export_directory)
return path
except Exception as e:
logger.exception(f"(Job {p}) Error on export_labbook_as_zip: {e}")
raise
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
remote_name,
remote_url,
client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username,
owner,
labbook_name,
author=get_logged_in_author())
with lb.lock():
lb.add_remote(remote_name, remote_url)
return AddLabbookRemote(success=True)
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username,
owner,
labbook_name,
author=get_logged_in_author())
# TODO - Should we cehck if a custom docker component already exists?
with lb.lock():
cm = ComponentManager(lb)
cm.remove_docker_snippet(cm.DEFAULT_CUSTOM_DOCKER_NAME)
return RemoveCustomDocker(
updated_environment=Environment(owner=owner, name=labbook_name))
def mutate_and_process_upload(cls, info, upload_file_path, upload_filename,
**kwargs):
if not upload_file_path:
logger.error('No file uploaded')
raise ValueError('No file uploaded')
owner = kwargs.get('owner')
labbook_name = kwargs.get('labbook_name')
section = kwargs.get('section')
transaction_id = kwargs.get('transaction_id')
file_path = kwargs.get('file_path')
try:
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username,
owner,
labbook_name,
author=get_logged_in_author())
dst_path = os.path.join(os.path.dirname(file_path),
upload_filename)
with lb.lock():
fops = FileOperations.put_file(labbook=lb,
section=section,
src_file=upload_file_path,
dst_path=dst_path,
txid=transaction_id)
finally:
try:
logger.debug(f"Removing temp file {upload_file_path}")
os.remove(upload_file_path)
except FileNotFoundError:
pass
# Create data to populate edge
create_data = {
'owner': owner,
'name': labbook_name,
'section': section,
'key': fops['key'],
'_file_info': fops
}
# TODO: Fix cursor implementation..this currently doesn't make sense when adding edges without a refresh
cursor = base64.b64encode(f"{0}".encode('utf-8'))
return AddLabbookFile(new_labbook_file_edge=LabbookFileConnection.Edge(
node=LabbookFile(**create_data), cursor=cursor))
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
packages,
client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username,
owner,
labbook_name,
author=get_logged_in_author())
with lb.lock():
new_edges = cls._add_package_components(lb, packages)
return AddPackageComponents(new_package_component_edges=new_edges)
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
filename,
client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username,
owner,
labbook_name,
author=get_logged_in_author())
with lb.lock():
secstore = SecretStore(lb, username)
secstore.delete_file(filename)
env = Environment(owner=owner, name=lb.name)
return DeleteSecretsFile(environment=env)
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
app_name,
client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username,
owner,
labbook_name,
author=get_logged_in_author())
bam = BundledAppManager(lb)
with lb.lock():
bam.remove_bundled_app(app_name)
return SetBundledApp(
environment=Environment(name=labbook_name, owner=owner))
