def resolve_visibility(self, info):
app_config = Configuration().config
default_remote = app_config['git']['default_remote']
admin_service = None
for remote in Configuration().config['git']['remotes']:
if default_remote == remote:
admin_service = app_config['git']['remotes'][remote]['admin_service']
break
# Extract valid Bearer token
if "HTTP_AUTHORIZATION" in info.context.headers.environ:
token = parse_token(info.context.headers.environ["HTTP_AUTHORIZATION"])
else:
raise ValueError("Authorization header not provided. Must have a valid session to query for collaborators")
# Get collaborators from remote service
mgr = GitLabManager(default_remote, admin_service, token)
try:
d = mgr.repo_details(namespace=self.owner, repository_name=self.name)
return d.get('visibility')
except ValueError:
return "unknown"
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
username,
client_mutation_id=None):
logged_in_username = get_logged_in_username()
lb = InventoryManager().load_labbook(logged_in_username,
owner,
labbook_name,
author=get_logged_in_author())
# TODO: Future work will look up remote in LabBook data, allowing user to select remote.
default_remote = lb.client_config.config['git']['default_remote']
admin_service = None
for remote in lb.client_config.config['git']['remotes']:
if default_remote == remote:
admin_service = lb.client_config.config['git']['remotes'][
remote]['admin_service']
break
# Extract valid Bearer token
if "HTTP_AUTHORIZATION" in info.context.headers.environ:
token = parse_token(
info.context.headers.environ["HTTP_AUTHORIZATION"])
else:
raise ValueError(
"Authorization header not provided. Must have a valid session to query for collaborators"
)
# Add collaborator to remote service
mgr = GitLabManager(default_remote, admin_service, token)
mgr.delete_collaborator(owner, labbook_name, username)
create_data = {"owner": owner, "name": labbook_name}
return DeleteLabbookCollaborator(updated_labbook=LabbookObject(
**create_data))
def _fetch_collaborators(self, labbook, info):
"""Helper method to fetch this labbook's collaborators
Args:
info: The graphene info object for this requests
"""
# TODO: Future work will look up remote in LabBook data, allowing user to select remote.
default_remote = labbook.client_config.config['git']['default_remote']
admin_service = None
for remote in labbook.client_config.config['git']['remotes']:
if default_remote == remote:
admin_service = labbook.client_config.config['git']['remotes'][
remote]['admin_service']
break
# Extract valid Bearer token
if "HTTP_AUTHORIZATION" in info.context.headers.environ:
token = parse_token(
info.context.headers.environ["HTTP_AUTHORIZATION"])
else:
raise ValueError(
"Authorization header not provided. Must have a valid session to query for collaborators"
)
# Get collaborators from remote service
mgr = GitLabManager(default_remote, admin_service, token)
try:
self._collaborators = [
Collaborator(owner=self.owner,
name=self.name,
collaborator_username=c[1],
permission=ProjectPermissions(c[2]).name)
for c in mgr.get_collaborators(self.owner, self.name)
]
except ValueError:
# If ValueError Raised, assume repo doesn't exist yet
self._collaborators = []
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
set_public=False,
client_mutation_id=None):
# Load LabBook
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username,
owner,
labbook_name,
author=get_logged_in_author())
# Extract valid Bearer token
if "HTTP_AUTHORIZATION" in info.context.headers.environ:
token = parse_token(
info.context.headers.environ["HTTP_AUTHORIZATION"])
else:
raise ValueError(
"Authorization header not provided. Must have a valid session to query for collaborators"
)
job_metadata = {'method': 'publish_labbook', 'labbook': lb.key}
job_kwargs = {
'repository': lb,
'username': username,
'access_token': token,
'public': set_public
}
dispatcher = Dispatcher()
job_key = dispatcher.dispatch_task(jobs.publish_repository,
kwargs=job_kwargs,
metadata=job_metadata)
logger.info(
f"Publishing LabBook {lb.root_dir} in background job with key {job_key.key_str}"
)
return PublishLabbook(job_key=job_key.key_str)
def helper_resolve_visibility(dataset, info):
# TODO: Future work will look up remote in Dataset data, allowing user to select remote.
default_remote = dataset.client_config.config['git']['default_remote']
admin_service = None
for remote in dataset.client_config.config['git']['remotes']:
if default_remote == remote:
admin_service = dataset.client_config.config['git']['remotes'][remote]['admin_service']
break
# Extract valid Bearer token
if "HTTP_AUTHORIZATION" in info.context.headers.environ:
token = parse_token(info.context.headers.environ["HTTP_AUTHORIZATION"])
else:
raise ValueError("Authorization header not provided. Must have a valid session to query for collaborators")
# Get collaborators from remote service
mgr = GitLabManager(default_remote, admin_service, token)
try:
owner = InventoryManager().query_owner(dataset)
d = mgr.repo_details(namespace=owner, repository_name=dataset.name)
return d.get('visibility')
except ValueError:
return "local"
def resolve(self, next, root, info, **args):
if not self.identity_mgr:
self.identity_mgr = get_identity_manager_instance()
# On first field processed in request, authenticate
if not hasattr(info.context, "auth_middleware_complete"):
# Pull the token out of the header if available
token = None
if "Authorization" in info.context.headers:
token = parse_token(info.context.headers["Authorization"])
# Save token to the request context for future use (e.g. look up a user's profile information if needed)
flask.g.access_token = token
# Check if you are authenticated
try:
self.identity_mgr.is_authenticated(token)
except AuthenticationError:
raise AuthenticationError("User not authenticated", 401)
info.context.auth_middleware_complete = True
return next(root, info, **args)
def mutate_and_get_payload(cls,
root,
info,
owner,
dataset_name,
pull_only=False,
override_method="abort",
client_mutation_id=None):
# Load Dataset
username = get_logged_in_username()
ds = InventoryManager().load_dataset(username,
owner,
dataset_name,
author=get_logged_in_author())
# Extract valid Bearer token
token = None
if hasattr(info.context.headers, 'environ'):
if "HTTP_AUTHORIZATION" in info.context.headers.environ:
token = parse_token(
info.context.headers.environ["HTTP_AUTHORIZATION"])
if not token:
raise ValueError(
"Authorization header not provided. Must have a valid session to query for collaborators"
)
default_remote = ds.client_config.config['git']['default_remote']
admin_service = None
for remote in ds.client_config.config['git']['remotes']:
if default_remote == remote:
admin_service = ds.client_config.config['git']['remotes'][
remote]['admin_service']
break
if not admin_service:
raise ValueError('admin_service could not be found')
# Configure git creds
mgr = GitLabManager(default_remote, admin_service, access_token=token)
mgr.configure_git_credentials(default_remote, username)
override = MergeOverride(override_method)
job_metadata = {'method': 'sync_dataset', 'dataset': ds.key}
job_kwargs = {
'repository': ds,
'username': username,
'pull_only': pull_only,
'override': override,
'access_token': token,
'id_token': flask.g.id_token
}
dispatcher = Dispatcher()
job_key = dispatcher.dispatch_task(jobs.sync_repository,
kwargs=job_kwargs,
metadata=job_metadata)
logger.info(
f"Syncing Dataset {ds.root_dir} in background job with key {job_key.key_str}"
)
return SyncDataset(job_key=job_key.key_str)
def mutate_and_get_payload(cls,
root,
info,
owner,
dataset_name,
username,
permissions,
client_mutation_id=None):
# Here "username" refers to the intended recipient username.
# Todo: it should probably be renamed here and in the frontend to "collaboratorUsername"
logged_in_username = get_logged_in_username()
lb = InventoryManager().load_dataset(logged_in_username,
owner,
dataset_name,
author=get_logged_in_author())
# TODO: Future work will look up remote in LabBook data, allowing user to select remote.
default_remote = lb.client_config.config['git']['default_remote']
admin_service = None
for remote in lb.client_config.config['git']['remotes']:
if default_remote == remote:
admin_service = lb.client_config.config['git']['remotes'][
remote]['admin_service']
break
# Extract valid Bearer token
if "HTTP_AUTHORIZATION" in info.context.headers.environ:
token = parse_token(
info.context.headers.environ["HTTP_AUTHORIZATION"])
else:
raise ValueError(
"Authorization header not provided. "
"Must have a valid session to query for collaborators")
if permissions == 'readonly':
perm = ProjectPermissions.READ_ONLY
elif permissions == 'readwrite':
perm = ProjectPermissions.READ_WRITE
elif permissions == 'owner':
perm = ProjectPermissions.OWNER
else:
raise ValueError(f"Unknown permission set: {permissions}")
# Add collaborator to remote service
mgr = GitLabManager(default_remote, admin_service, token)
existing_collabs = mgr.get_collaborators(owner, dataset_name)
if username not in [n[1] for n in existing_collabs]:
logger.info(f"Adding user {username} to {owner}/{dataset_name}"
f"with permission {perm}")
mgr.add_collaborator(owner, dataset_name, username, perm)
else:
logger.warning(f"Changing permission of {username} on"
f"{owner}/{dataset_name} to {perm}")
mgr.delete_collaborator(owner, dataset_name, username)
mgr.add_collaborator(owner, dataset_name, username, perm)
create_data = {"owner": owner, "name": dataset_name}
return AddDatasetCollaborator(updated_dataset=DatasetObject(
**create_data))
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
confirm,
client_mutation_id=None):
if confirm is True:
# Load config data
configuration = Configuration().config
# Extract valid Bearer token
token = None
if hasattr(info.context.headers, 'environ'):
if "HTTP_AUTHORIZATION" in info.context.headers.environ:
token = parse_token(
info.context.headers.environ["HTTP_AUTHORIZATION"])
if not token:
raise ValueError(
"Authorization header not provided. Cannot perform remote delete operation."
)
# Get remote server configuration
default_remote = configuration['git']['default_remote']
admin_service = None
for remote in configuration['git']['remotes']:
if default_remote == remote:
admin_service = configuration['git']['remotes'][remote][
'admin_service']
index_service = configuration['git']['remotes'][remote][
'index_service']
break
if not admin_service:
raise ValueError('admin_service could not be found')
# Perform delete operation
mgr = GitLabManager(default_remote,
admin_service,
access_token=token)
mgr.remove_repository(owner, labbook_name)
logger.info(
f"Deleted {owner}/{labbook_name} from the remote repository {default_remote}"
)
# Call Index service to remove project from cloud index and search
# Don't raise an exception if the index delete fails, since this can be handled relatively gracefully
# for now, but do return success=false
success = True
access_token = flask.g.get('access_token', None)
id_token = flask.g.get('id_token', None)
repo_id = mgr.get_repository_id(owner, labbook_name)
response = requests.delete(
f"https://{index_service}/index/{repo_id}",
headers={
"Authorization": f"Bearer {access_token}",
"Identity": id_token
},
timeout=10)
if response.status_code != 204:
logger.error(f"Failed to remove project from cloud index. "
f"Status Code: {response.status_code}")
logger.error(response.json())
else:
logger.info(
f"Deleted remote repository {owner}/{labbook_name} from cloud index"
)
# Remove locally any references to that cloud repo that's just been deleted.
try:
username = get_logged_in_username()
lb = InventoryManager().load_labbook(
username,
owner,
labbook_name,
author=get_logged_in_author())
lb.remove_remote()
lb.remove_lfs_remotes()
except GigantumException as e:
logger.warning(e)
return DeleteLabbook(success=True)
else:
logger.info(
f"Dry run deleting {labbook_name} from remote repository -- not deleted."
)
return DeleteLabbook(success=False)
def mutate_and_get_payload(cls,
root,
info,
labbook_owner,
labbook_name,
dataset_owner,
dataset_name,
action,
dataset_url=None,
client_mutation_id=None):
logged_in_username = get_logged_in_username()
im = InventoryManager()
lb = im.load_labbook(logged_in_username,
labbook_owner,
labbook_name,
author=get_logged_in_author())
with lb.lock():
if action == 'link':
if dataset_url:
remote_domain = cls._get_remote_domain(
dataset_url, dataset_owner, dataset_name)
if remote_domain:
# Make sure git creds are configured for the remote
admin_service = None
for remote in lb.client_config.config['git'][
'remotes']:
if remote_domain == remote:
admin_service = lb.client_config.config['git'][
'remotes'][remote]['admin_service']
break
if "HTTP_AUTHORIZATION" in info.context.headers.environ:
token = parse_token(info.context.headers.
environ["HTTP_AUTHORIZATION"])
else:
raise ValueError(
"Authorization header not provided."
" Must have a valid session to query for collaborators"
)
mgr = GitLabManager(remote_domain, admin_service,
token)
mgr.configure_git_credentials(remote_domain,
logged_in_username)
else:
# Link to local dataset
ds = im.load_dataset(logged_in_username, dataset_owner,
dataset_name)
dataset_url = f"{ds.root_dir}/.git"
# Link the dataset to the labbook
ds = im.link_dataset_to_labbook(dataset_url, dataset_owner,
dataset_name, lb)
ds.namespace = dataset_owner
# Preload the dataloader
info.context.dataset_loader.prime(
f"{get_logged_in_username()}&{dataset_owner}&{dataset_name}",
ds)
# Relink the revision
m = Manifest(ds, logged_in_username)
m.link_revision()
elif action == 'unlink':
im.unlink_dataset_from_labbook(dataset_owner, dataset_name, lb)
elif action == 'update':
ds = im.update_linked_dataset_reference(
dataset_owner, dataset_name, lb)
m = Manifest(ds, logged_in_username)
m.force_reload()
info.context.dataset_loader.prime(
f"{get_logged_in_username()}&{dataset_owner}&{dataset_name}",
ds)
else:
raise ValueError(
"Unsupported action. Use `link`, `unlink`, or `update`")
info.context.labbook_loader.prime(
f"{get_logged_in_username()}&{labbook_owner}&{labbook_name}",
lb)
edge = LabbookConnection.Edge(node=Labbook(owner=labbook_owner,
name=labbook_name),
cursor=base64.b64encode(
f"{0}".encode('utf-8')))
return ModifyDatasetLink(new_labbook_edge=edge)
def resolve_remote_labbooks(self, info, sort: str, reverse: bool,
**kwargs):
"""Method to return a all RemoteLabbook instances for the logged in user
This is a remote call, so should be fetched on its own and only when needed. The user must have a valid
session for data to be returned.
It is recommended to use large page size (e.g. 50-100). This is due to how the remote server returns all the
available data at once, so it is more efficient to load a lot of records at a time.
Args:
sort(sort_mode): String specifying how labbooks should be sorted
reverse(bool): Reverse sorting if True
Supported sorting modes:
- az: naturally sort
- created_on: sort by creation date, newest first
- modified_on: sort by modification date, newest first
Returns:
list(Labbook)
"""
# Load config data
configuration = Configuration().config
# Extract valid Bearer token
token = None
if hasattr(info.context.headers, 'environ'):
if "HTTP_AUTHORIZATION" in info.context.headers.environ:
token = parse_token(
info.context.headers.environ["HTTP_AUTHORIZATION"])
if not token:
raise ValueError(
"Authorization header not provided. Cannot list remote LabBooks."
)
# Get remote server configuration
default_remote = configuration['git']['default_remote']
admin_service = None
for remote in configuration['git']['remotes']:
if default_remote == remote:
admin_service = configuration['git']['remotes'][remote][
'admin_service']
break
if not admin_service:
raise ValueError('admin_service could not be found')
# Query backend for data
mgr = GitLabManager(default_remote, admin_service, access_token=token)
edges = mgr.list_labbooks(sort_mode=sort, reverse=reverse)
cursors = [
base64.b64encode("{}".format(cnt).encode("UTF-8")).decode("UTF-8")
for cnt, x in enumerate(edges)
]
# Process slicing and cursor args
lbc = ListBasedConnection(edges, cursors, kwargs)
lbc.apply()
# Get Labbook instances
edge_objs = []
for edge, cursor in zip(lbc.edges, lbc.cursors):
create_data = {
"id": "{}&{}".format(edge["namespace"], edge["labbook_name"]),
"name": edge["labbook_name"],
"owner": edge["namespace"],
"description": edge["description"],
"creation_date_utc": edge["created_on"],
"modified_date_utc": edge["modified_on"]
}
edge_objs.append(
RemoteLabbookConnection.Edge(node=RemoteLabbook(**create_data),
cursor=cursor))
return RemoteLabbookConnection(edges=edge_objs,
page_info=lbc.page_info)
def resolve_remote_labbooks(self, info, order_by: str, sort: str, **kwargs):
"""Method to return a all RemoteLabbook instances for the logged in user
This is a remote call, so should be fetched on its own and only when needed. The user must have a valid
session for data to be returned.
Args:
order_by(str): String specifying how labbooks should be sorted
sort(str): 'desc' for descending (default) 'asc' for ascending
Supported order_by modes:
- name: naturally sort on the name
- created_on: sort by creation date
- modified_on: sort by modification date
Returns:
list(Labbook)
"""
# Load config data
configuration = Configuration().config
# Extract valid Bearer token
token = None
if hasattr(info.context.headers, 'environ'):
if "HTTP_AUTHORIZATION" in info.context.headers.environ:
token = parse_token(info.context.headers.environ["HTTP_AUTHORIZATION"])
if not token:
raise ValueError("Authorization header not provided. Cannot list remote LabBooks.")
# Get remote server configuration
default_remote = configuration['git']['default_remote']
index_service = None
for remote in configuration['git']['remotes']:
if default_remote == remote:
index_service = configuration['git']['remotes'][remote]['index_service']
break
if not index_service:
raise ValueError('index_service could not be found')
# Prep arguments
if "first" in kwargs:
per_page = int(kwargs['first'])
elif "last" in kwargs:
raise ValueError("Cannot page in reverse direction, must provide first/after parameters instead")
else:
per_page = 20
# use version=2 to page through both projects and datasets returned by gitlab but only return projects.
# omitting the version parameter will return projects and datasets together, but avoids breaking functionality
# in any gigantum instances not using the latest client code
url = f"https://{index_service}/projects?version=2&first={per_page}"
# Add optional arguments
if kwargs.get("after"):
url = f"{url}&after={kwargs.get('after')}"
elif kwargs.get("before"):
raise ValueError("Cannot page in reverse direction, must provide first/after parameters instead")
if order_by is not None:
if order_by not in ['name', 'created_on', 'modified_on']:
raise ValueError(f"Unsupported order_by: {order_by}. Use `name`, `created_on`, `modified_on`")
url = f"{url}&order_by={order_by}"
if sort is not None:
if sort not in ['desc', 'asc']:
raise ValueError(f"Unsupported sort: {sort}. Use `desc`, `asc`")
url = f"{url}&sort={sort}"
# Query SaaS index service for data
access_token = flask.g.get('access_token', None)
id_token = flask.g.get('id_token', None)
response = requests.get(url, headers={"Authorization": f"Bearer {access_token}",
"Identity": id_token})
if response.status_code != 200:
raise IOError("Failed to retrieve Project listing from remote server")
edges = response.json()
# Get Labbook instances
edge_objs = []
for edge in edges:
create_data = {"id": "{}&{}".format(edge["namespace"], edge["project"]),
"name": edge["project"],
"owner": edge["namespace"],
"description": edge["description"],
"creation_date_utc": edge["created_at"],
"modified_date_utc": edge["modified_at"],
"visibility": "public" if edge.get("visibility") == "public_project" else "private"}
edge_objs.append(RemoteLabbookConnection.Edge(node=RemoteLabbook(**create_data),
cursor=edge['cursor']))
# Create Page Info instance
has_previous_page = True if kwargs.get("after") else False
has_next_page = len(edges) == per_page
page_info = graphene.relay.PageInfo(has_next_page=has_next_page, has_previous_page=has_previous_page,
start_cursor=edges[0]['cursor'] if edges else None,
end_cursor=edges[-1]['cursor'] if edges else None)
return RemoteLabbookConnection(edges=edge_objs, page_info=page_info)
