def create_remote_gitlab_repo(repository: Repository,
username: str,
visibility: str,
access_token: Optional[str] = None) -> None:
"""Create a new repository in GitLab,
Note: It may make more sense to factor this out later on. """
default_remote = repository.client_config.config['git']['default_remote']
admin_service = None
for remote in repository.client_config.config['git']['remotes']:
if default_remote == remote:
admin_service = repository.client_config.config['git']['remotes'][
remote]['admin_service']
break
if not admin_service:
raise ValueError('admin_service could not be found')
try:
# Add collaborator to remote service
mgr = GitLabManager(default_remote,
admin_service,
access_token=access_token or 'invalid')
mgr.configure_git_credentials(default_remote, username)
mgr.create_labbook(
namespace=InventoryManager().query_owner(repository),
labbook_name=repository.name,
visibility=visibility)
repository.add_remote(
"origin",
f"https://{default_remote}/{username}/{repository.name}.git")
except Exception as e:
raise GitLabRemoteError(e)
def _configure_git(cls, lb, info) -> GitLabManager:
# Extract valid Bearer token
# TODO - This code is duplicated all over the place, must be refactored.
token = None
if hasattr(info.context.headers, 'environ'):
if "HTTP_AUTHORIZATION" in info.context.headers.environ:
token = parse_token(info.context.headers.environ["HTTP_AUTHORIZATION"])
if not token:
raise ValueError("Authorization header not provided. "
"Must have a valid session to query for collaborators")
default_remote = lb.client_config.config['git']['default_remote']
admin_service = None
for remote in lb.client_config.config['git']['remotes']:
if default_remote == remote:
admin_service = lb.client_config.config['git']['remotes'][remote]['admin_service']
break
if not admin_service:
raise ValueError('admin_service could not be found')
# Configure git creds
mgr = GitLabManager(default_remote, admin_service, access_token=token)
mgr.configure_git_credentials(default_remote, get_logged_in_username())
return mgr
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
pull_only=False,
override_method="abort",
client_mutation_id=None):
# Load LabBook
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username,
owner,
labbook_name,
author=get_logged_in_author())
# Extract valid Bearer token
token = None
if hasattr(info.context.headers, 'environ'):
if "HTTP_AUTHORIZATION" in info.context.headers.environ:
token = parse_token(
info.context.headers.environ["HTTP_AUTHORIZATION"])
if not token:
raise ValueError(
"Authorization header not provided. "
"Must have a valid session to query for collaborators")
default_remote = lb.client_config.config['git']['default_remote']
admin_service = None
for remote in lb.client_config.config['git']['remotes']:
if default_remote == remote:
admin_service = lb.client_config.config['git']['remotes'][
remote]['admin_service']
break
if not admin_service:
raise ValueError('admin_service could not be found')
# Configure git creds
mgr = GitLabManager(default_remote, admin_service, access_token=token)
mgr.configure_git_credentials(default_remote, username)
override = MergeOverride(override_method)
job_metadata = {'method': 'sync_labbook', 'labbook': lb.key}
job_kwargs = {
'repository': lb,
'pull_only': pull_only,
'username': username,
'override': override
}
dispatcher = Dispatcher()
job_key = dispatcher.dispatch_task(jobs.sync_repository,
kwargs=job_kwargs,
metadata=job_metadata)
logger.info(
f"Syncing LabBook {lb.root_dir} in background job with key {job_key.key_str}"
)
return SyncLabbook(job_key=job_key.key_str)
def mutate_and_get_payload(cls,
root,
info,
owner,
dataset_name,
visibility,
client_mutation_id=None):
# Load Dataset
username = get_logged_in_username()
ds = InventoryManager().load_dataset(username,
owner,
dataset_name,
author=get_logged_in_author())
# Extract valid Bearer token
token = None
if hasattr(info.context.headers, 'environ'):
if "HTTP_AUTHORIZATION" in info.context.headers.environ:
token = parse_token(
info.context.headers.environ["HTTP_AUTHORIZATION"])
if not token:
raise ValueError(
"Authorization header not provided. Must have a valid session to query for collaborators"
)
default_remote = ds.client_config.config['git']['default_remote']
admin_service = None
for remote in ds.client_config.config['git']['remotes']:
if default_remote == remote:
admin_service = ds.client_config.config['git']['remotes'][
remote]['admin_service']
break
if not admin_service:
raise ValueError('admin_service could not be found')
# Configure git creds
mgr = GitLabManager(default_remote, admin_service, access_token=token)
mgr.configure_git_credentials(default_remote, username)
if visibility not in ['public', 'private']:
raise ValueError(
f'Visibility must be either "public" or "private";'
f'("{visibility}" invalid)')
with ds.lock():
mgr.set_visibility(namespace=owner,
repository_name=dataset_name,
visibility=visibility)
cursor = base64.b64encode(f"{0}".encode('utf-8'))
dsedge = DatasetConnection.Edge(node=DatasetObject(owner=owner,
name=dataset_name),
cursor=cursor)
return SetDatasetVisibility(new_dataset_edge=dsedge)
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
remote_url,
client_mutation_id=None):
username = get_logged_in_username()
logger.info(f"Importing remote labbook from {remote_url}")
lb = LabBook(author=get_logged_in_author())
default_remote = lb.client_config.config['git']['default_remote']
admin_service = None
for remote in lb.client_config.config['git']['remotes']:
if default_remote == remote:
admin_service = lb.client_config.config['git']['remotes'][
remote]['admin_service']
break
# Extract valid Bearer token
if hasattr(info.context, 'headers'
) and "HTTP_AUTHORIZATION" in info.context.headers.environ:
token = parse_token(
info.context.headers.environ["HTTP_AUTHORIZATION"])
else:
raise ValueError(
"Authorization header not provided. Must have a valid session to query for collaborators"
)
gl_mgr = GitLabManager(default_remote,
admin_service=admin_service,
access_token=token)
gl_mgr.configure_git_credentials(default_remote, username)
job_metadata = {'method': 'import_labbook_from_remote'}
job_kwargs = {'remote_url': remote_url, 'username': username}
dispatcher = Dispatcher()
job_key = dispatcher.dispatch_task(jobs.import_labbook_from_remote,
metadata=job_metadata,
kwargs=job_kwargs)
logger.info(
f"Dispatched import_labbook_from_remote({remote_url}) to Job {job_key}"
)
return ImportRemoteLabbook(job_key=job_key.key_str)
def mutate_and_get_payload(cls,
root,
info,
owner,
dataset_name,
remote_url,
client_mutation_id=None):
username = get_logged_in_username()
logger.info(f"Importing remote dataset from {remote_url}")
ds = Dataset(author=get_logged_in_author())
default_remote = ds.client_config.config['git']['default_remote']
admin_service = None
for remote in ds.client_config.config['git']['remotes']:
if default_remote == remote:
admin_service = ds.client_config.config['git']['remotes'][
remote]['admin_service']
break
# Extract valid Bearer token
if hasattr(info.context, 'headers'
) and "HTTP_AUTHORIZATION" in info.context.headers.environ:
token = parse_token(
info.context.headers.environ["HTTP_AUTHORIZATION"])
else:
raise ValueError(
"Authorization header not provided. Must have a valid session to query for collaborators"
)
gl_mgr = GitLabManager(default_remote,
admin_service=admin_service,
access_token=token)
gl_mgr.configure_git_credentials(default_remote, username)
wf = DatasetWorkflow.import_from_remote(remote_url, username=username)
ds = wf.dataset
import_owner = InventoryManager().query_owner(ds)
# TODO: Fix cursor implementation, this currently doesn't make sense
cursor = base64.b64encode(f"{0}".encode('utf-8'))
dsedge = DatasetConnection.Edge(node=DatasetObject(owner=import_owner,
name=ds.name),
cursor=cursor)
return ImportRemoteDataset(new_dataset_edge=dsedge)
def check_and_import_dataset(logged_in_username: str,
dataset_owner: str,
dataset_name: str,
remote_url: str,
access_token: Optional[str] = None,
config_file: Optional[str] = None) -> None:
"""Job to check if a dataset exists in the user's working directory, and if not import it. This is primarily used
when importing, syncing, or switching branches on a project with linked datasets
Args:
logged_in_username: username for the currently logged in user
dataset_owner: Owner of the labbook if this dataset is linked
dataset_name: Name of the labbook if this dataset is linked
remote_url: URL of the dataset to import if needed
access_token: The current user's access token, needed to initialize git credentials in certain situations
config_file: config file (used for test mocking)
Returns:
None
"""
logger = LMLogger.get_logger()
p = os.getpid()
try:
logger.info(
f"(Job {p}) Starting check_and_import_dataset(logged_in_username={logged_in_username},"
f"dataset_owner={dataset_owner}, dataset_name={dataset_name}")
im = InventoryManager(config_file=config_file)
try:
# Check for dataset already existing in the user's working directory
im.load_dataset(logged_in_username, dataset_owner, dataset_name)
logger.info(
f"{logged_in_username}/{dataset_owner}/{dataset_name} exists. Skipping auto-import."
)
return
except InventoryException:
# Dataset not found, import it
logger.info(
f"{logged_in_username}/{dataset_owner}/{dataset_name} not found. "
f"Auto-importing remote dataset from {remote_url}")
config_obj = Configuration(config_file=config_file)
if access_token:
# If the access token is set, git creds should be configured
remote_parts = urlsplit(remote_url)
if remote_parts.netloc:
remote_target = f"{remote_parts.scheme}://{remote_parts.netloc}/"
else:
remote_target = remote_parts.path
admin_service = None
for remote in config_obj.config['git']['remotes']:
if remote == remote_target:
admin_service = config_obj.config['git']['remotes'][
remote]['admin_service']
break
if not admin_service:
raise ValueError(
f"Failed to configure admin service URL based on target remote: {remote_target}"
)
gl_mgr = GitLabManager(remote_target,
admin_service=admin_service,
access_token=access_token)
gl_mgr.configure_git_credentials(remote_target,
logged_in_username)
gitworkflows_utils.clone_repo(
remote_url=remote_url,
username=logged_in_username,
owner=dataset_owner,
load_repository=im.load_dataset_from_directory,
put_repository=im.put_dataset)
logger.info(
f"{logged_in_username}/{dataset_owner}/{dataset_name} auto-imported successfully"
)
except Exception as err:
logger.error(f"(Job {p}) Error in check_and_import_dataset job")
logger.exception(err)
raise
def mutate_and_get_payload(cls,
root,
info,
labbook_owner,
labbook_name,
dataset_owner,
dataset_name,
action,
dataset_url=None,
client_mutation_id=None):
logged_in_username = get_logged_in_username()
im = InventoryManager()
lb = im.load_labbook(logged_in_username,
labbook_owner,
labbook_name,
author=get_logged_in_author())
with lb.lock():
if action == 'link':
if dataset_url:
remote_domain = cls._get_remote_domain(
dataset_url, dataset_owner, dataset_name)
if remote_domain:
# Make sure git creds are configured for the remote
admin_service = None
for remote in lb.client_config.config['git'][
'remotes']:
if remote_domain == remote:
admin_service = lb.client_config.config['git'][
'remotes'][remote]['admin_service']
break
if "HTTP_AUTHORIZATION" in info.context.headers.environ:
token = parse_token(info.context.headers.
environ["HTTP_AUTHORIZATION"])
else:
raise ValueError(
"Authorization header not provided."
" Must have a valid session to query for collaborators"
)
mgr = GitLabManager(remote_domain, admin_service,
token)
mgr.configure_git_credentials(remote_domain,
logged_in_username)
else:
# Link to local dataset
ds = im.load_dataset(logged_in_username, dataset_owner,
dataset_name)
dataset_url = f"{ds.root_dir}/.git"
# Link the dataset to the labbook
ds = im.link_dataset_to_labbook(dataset_url, dataset_owner,
dataset_name, lb)
ds.namespace = dataset_owner
# Preload the dataloader
info.context.dataset_loader.prime(
f"{get_logged_in_username()}&{dataset_owner}&{dataset_name}",
ds)
# Relink the revision
m = Manifest(ds, logged_in_username)
m.link_revision()
elif action == 'unlink':
im.unlink_dataset_from_labbook(dataset_owner, dataset_name, lb)
elif action == 'update':
ds = im.update_linked_dataset_reference(
dataset_owner, dataset_name, lb)
m = Manifest(ds, logged_in_username)
m.force_reload()
info.context.dataset_loader.prime(
f"{get_logged_in_username()}&{dataset_owner}&{dataset_name}",
ds)
else:
raise ValueError(
"Unsupported action. Use `link`, `unlink`, or `update`")
info.context.labbook_loader.prime(
f"{get_logged_in_username()}&{labbook_owner}&{labbook_name}",
lb)
edge = LabbookConnection.Edge(node=Labbook(owner=labbook_owner,
name=labbook_name),
cursor=base64.b64encode(
f"{0}".encode('utf-8')))
return ModifyDatasetLink(new_labbook_edge=edge)