def _purge_quota(self, quota, params):
"""
This method should only be called for a Quota that has previously been deleted.
Purging a deleted Quota deletes all of the following from the database:
- UserQuotaAssociations where quota_id == Quota.id
- GroupQuotaAssociations where quota_id == Quota.id
"""
quotas = util.listify(quota)
names = []
for q in quotas:
if not q.deleted:
names.append(q.name)
if len(names) == 1:
raise ActionInputError(
"Quota '%s' has not been deleted, so it cannot be purged." %
(names[0]))
elif len(names) > 1:
raise ActionInputError(
"Quotas have not been deleted so they cannot be undeleted: " +
', '.join(names))
message = "Purged %d quotas: " % len(quotas)
for q in quotas:
# Delete UserQuotaAssociations
for uqa in q.users:
self.sa_session.delete(uqa)
# Delete GroupQuotaAssociations
for gqa in q.groups:
self.sa_session.delete(gqa)
names.append(q.name)
self.sa_session.flush()
message += ', '.join(names)
return message
def _manage_users_and_groups_for_quota(self,
quota,
params,
decode_id=None):
if quota.default:
raise ActionInputError(
'Default quotas cannot be associated with specific users and groups.'
)
else:
in_users = [
self.sa_session.query(
self.app.model.User).get(decode_id(x) if decode_id else x)
for x in util.listify(params.in_users)
]
if None in in_users:
raise ActionInputError(
"One or more invalid user id has been provided.")
in_groups = [
self.sa_session.query(self.app.model.Group).get(
decode_id(x) if decode_id else x)
for x in util.listify(params.in_groups)
]
if None in in_groups:
raise ActionInputError(
"One or more invalid group id has been provided.")
self.app.quota_agent.set_entity_quota_associations(
quotas=[quota], users=in_users, groups=in_groups)
self.sa_session.refresh(quota)
message = "Quota '%s' has been updated with %d associated users and %d associated groups." % (
quota.name, len(in_users), len(in_groups))
return message
def _rename_quota( self, quota, params ):
if not params.name:
raise ActionInputError( 'Enter a valid name' )
elif params.name != quota.name and self.sa_session.query( self.app.model.Quota ).filter( self.app.model.Quota.table.c.name==params.name ).first():
raise ActionInputError( 'A quota with that name already exists' )
else:
old_name = quota.name
quota.name = params.name
quota.description = params.description
self.sa_session.add( quota )
self.sa_session.flush()
message = "Quota '%s' has been renamed to '%s'" % ( old_name, params.name )
return message
def _create_quota(self, params, decode_id=None):
if params.amount.lower() in ('unlimited', 'none', 'no limit'):
create_amount = None
else:
try:
create_amount = util.size_to_bytes(params.amount)
except AssertionError:
create_amount = False
if not params.name or not params.description:
raise ActionInputError("Enter a valid name and a description.")
elif self.sa_session.query(self.app.model.Quota).filter(self.app.model.Quota.table.c.name == params.name).first():
raise ActionInputError("Quota names must be unique and a quota with that name already exists, so choose another name.")
elif not params.get('amount', None):
raise ActionInputError("Enter a valid quota amount.")
elif create_amount is False:
raise ActionInputError("Unable to parse the provided amount.")
elif params.operation not in self.app.model.Quota.valid_operations:
raise ActionInputError("Enter a valid operation.")
elif params.default != 'no' and params.default not in self.app.model.DefaultQuotaAssociation.types.__members__.values():
raise ActionInputError("Enter a valid default type.")
elif params.default != 'no' and params.operation != '=':
raise ActionInputError("Operation for a default quota must be '='.")
elif create_amount is None and params.operation != '=':
raise ActionInputError("Operation for an unlimited quota must be '='.")
else:
# Create the quota
quota = self.app.model.Quota(name=params.name, description=params.description, amount=create_amount, operation=params.operation)
self.sa_session.add(quota)
# If this is a default quota, create the DefaultQuotaAssociation
if params.default != 'no':
self.app.quota_agent.set_default_quota(params.default, quota)
message = "Default quota '%s' has been created." % quota.name
else:
# Create the UserQuotaAssociations
in_users = [self.sa_session.query(self.app.model.User).get(decode_id(x) if decode_id else x) for x in util.listify(params.in_users)]
in_groups = [self.sa_session.query(self.app.model.Group).get(decode_id(x) if decode_id else x) for x in util.listify(params.in_groups)]
if None in in_users:
raise ActionInputError("One or more invalid user id has been provided.")
for user in in_users:
uqa = self.app.model.UserQuotaAssociation(user, quota)
self.sa_session.add(uqa)
# Create the GroupQuotaAssociations
if None in in_groups:
raise ActionInputError("One or more invalid group id has been provided.")
for group in in_groups:
gqa = self.app.model.GroupQuotaAssociation(group, quota)
self.sa_session.add(gqa)
message = "Quota '%s' has been created with %d associated users and %d associated groups." % (quota.name, len(in_users), len(in_groups))
self.sa_session.flush()
return quota, message
def rename_quota(self, quota, params) -> str:
if not params.name:
raise ActionInputError('Enter a valid name.')
elif params.name != quota.name and self.sa_session.query(model.Quota).filter(model.Quota.name == params.name).first():
raise ActionInputError('A quota with that name already exists.')
else:
old_name = quota.name
quota.name = params.name
if params.description:
quota.description = params.description
self.sa_session.add(quota)
self.sa_session.flush()
message = f"Quota '{old_name}' has been renamed to '{params.name}'."
return message
def unset_quota_default(self, quota, params=None) -> str:
if not quota.default:
raise ActionInputError(f"Quota '{quota.name}' is not a default.")
else:
message = f"Quota '{quota.name}' is no longer the default for {quota.default[0].type} users."
for dqa in quota.default:
self.sa_session.delete(dqa)
self.sa_session.flush()
return message
def undelete_quota(self, quota, params=None) -> str:
quotas = util.listify(quota)
names = []
for q in quotas:
if not q.deleted:
names.append(q.name)
if len(names) == 1:
raise ActionInputError(f"Quota '{names[0]}' has not been deleted, so it cannot be undeleted.")
elif len(names) > 1:
raise ActionInputError(f"Quotas have not been deleted so they cannot be undeleted: {', '.join(names)}")
message = f"Undeleted {len(quotas)} quotas: "
for q in quotas:
q.deleted = False
self.sa_session.add(q)
names.append(q.name)
self.sa_session.flush()
message += ', '.join(names)
return message
def _undelete_quota( self, quota, params ):
quotas = util.listify( quota )
names = []
for q in quotas:
if not q.deleted:
names.append( q.name )
if len( names ) == 1:
raise ActionInputError( "Quota '%s' has not been deleted, so it cannot be undeleted." % ( names[0] ) )
elif len( names ) > 1:
raise ActionInputError( "Quotas have not been deleted so they cannot be undeleted: " + ', '.join( names ) )
message = "Undeleted %d quotas: " % len( quotas )
for q in quotas:
q.deleted = False
self.sa_session.add( q )
names.append( q.name )
self.sa_session.flush()
message += ', '.join( names )
return message
def _mark_quota_deleted( self, quota, params ):
quotas = util.listify( quota )
names = []
for q in quotas:
if q.default:
names.append( q.name )
if len( names ) == 1:
raise ActionInputError( "Quota '%s' is a default, please unset it as a default before deleting it" % ( names[0] ) )
elif len( names ) > 1:
raise ActionInputError( "Quotas are defaults, please unset them as defaults before deleting them: " + ', '.join( names ) )
message = "Deleted %d quotas: " % len( quotas )
for q in quotas:
q.deleted = True
self.sa_session.add( q )
names.append( q.name )
self.sa_session.flush()
message += ', '.join( names )
return message
def delete_quota(self, quota, params=None) -> str:
quotas = util.listify(quota)
names = []
for q in quotas:
if q.default:
names.append(q.name)
if len(names) == 1:
raise ActionInputError(f"Quota '{names[0]}' is a default, please unset it as a default before deleting it.")
elif len(names) > 1:
raise ActionInputError(f"Quotas are defaults, please unset them as defaults before deleting them: {', '.join(names)}")
message = f"Deleted {len(quotas)} quotas: "
for q in quotas:
q.deleted = True
self.sa_session.add(q)
names.append(q.name)
self.sa_session.flush()
message += ', '.join(names)
return message
def _unset_quota_default( self, quota, params ):
if not quota.default:
raise ActionInputError( "Quota '%s' is not a default." % quota.name )
else:
message = "Quota '%s' is no longer the default for %s users." % ( quota.name, quota.default[0].type )
for dqa in quota.default:
self.sa_session.delete( dqa )
self.sa_session.flush()
return message
def update( self, trans, id, **kwd ):
"""
PATCH /api/repositories/{encoded_repository_id}
Updates information about a repository in the Tool Shed.
:param id: the encoded id of the Repository object
:param payload: dictionary structure containing::
'name': repo's name (optional)
'synopsis': repo's synopsis (optional)
'description': repo's description (optional)
'remote_repository_url': repo's remote repo (optional)
'homepage_url': repo's homepage url (optional)
'category_ids': list of existing encoded TS category ids
the updated repo should be associated with (optional)
:type payload: dict
:returns: detailed repository information
:rtype: dict
:raises: RequestParameterInvalidException, InsufficientPermissionsException
"""
payload = kwd.get( 'payload', None )
if not payload:
raise RequestParameterMissingException( "You did not specify any payload." )
name = payload.get( 'name', None )
synopsis = payload.get( 'synopsis', None )
description = payload.get( 'description', None )
remote_repository_url = payload.get( 'remote_repository_url', None )
homepage_url = payload.get( 'homepage_url', None )
category_ids = payload.get( 'category_ids', None )
if category_ids is not None:
# We need to know if it was actually passed, and listify turns None into []
category_ids = util.listify( category_ids )
update_kwds = dict(
name=name,
description=synopsis,
long_description=description,
remote_repository_url=remote_repository_url,
homepage_url=homepage_url,
category_ids=category_ids,
)
repo, message = repository_util.update_repository( app=trans.app, trans=trans, id=id, **update_kwds )
if repo is None:
if "You are not the owner" in message:
raise InsufficientPermissionsException( message )
else:
raise ActionInputError( message )
repository_dict = repo.to_dict( view='element', value_mapper=self.__get_value_mapper( trans ) )
repository_dict[ 'category_ids' ] = \
[ trans.security.encode_id( x.category.id ) for x in repo.categories ]
return repository_dict
def _manage_users_and_groups_for_quota( self, quota, params ):
if quota.default:
raise ActionInputError( 'Default quotas cannot be associated with specific users and groups' )
else:
in_users = [ self.sa_session.query( self.app.model.User ).get( x ) for x in util.listify( params.in_users ) ]
in_groups = [ self.sa_session.query( self.app.model.Group ).get( x ) for x in util.listify( params.in_groups ) ]
self.app.quota_agent.set_entity_quota_associations( quotas=[ quota ], users=in_users, groups=in_groups )
self.sa_session.refresh( quota )
message = "Quota '%s' has been updated with %d associated users and %d associated groups" % ( quota.name, len( in_users ), len( in_groups ) )
return message
def create_quota(self,
payload: dict,
decode_id=None) -> Tuple[model.Quota, str]:
params = CreateQuotaParams.parse_obj(payload)
create_amount = self._parse_amount(params.amount)
if self.sa_session.query(
model.Quota).filter(model.Quota.name == params.name).first():
raise ActionInputError(
"Quota names must be unique and a quota with that name already exists, please choose another name."
)
elif create_amount is False:
raise ActionInputError("Unable to parse the provided amount.")
elif params.operation not in model.Quota.valid_operations:
raise ActionInputError("Enter a valid operation.")
elif params.default != DefaultQuotaValues.NO and params.operation != QuotaOperation.EXACT:
raise ActionInputError(
"Operation for a default quota must be '='.")
elif create_amount is None and params.operation != QuotaOperation.EXACT:
raise ActionInputError(
"Operation for an unlimited quota must be '='.")
# Create the quota
quota = model.Quota(name=params.name,
description=params.description,
amount=create_amount,
operation=params.operation)
self.sa_session.add(quota)
# If this is a default quota, create the DefaultQuotaAssociation
if params.default != DefaultQuotaValues.NO:
self.quota_agent.set_default_quota(params.default, quota)
message = f"Default quota '{quota.name}' has been created."
else:
# Create the UserQuotaAssociations
in_users = [
self.sa_session.query(
model.User).get(decode_id(x) if decode_id else x)
for x in util.listify(params.in_users)
]
in_groups = [
self.sa_session.query(
model.Group).get(decode_id(x) if decode_id else x)
for x in util.listify(params.in_groups)
]
if None in in_users:
raise ActionInputError(
"One or more invalid user id has been provided.")
for user in in_users:
uqa = model.UserQuotaAssociation(user, quota)
self.sa_session.add(uqa)
# Create the GroupQuotaAssociations
if None in in_groups:
raise ActionInputError(
"One or more invalid group id has been provided.")
for group in in_groups:
gqa = model.GroupQuotaAssociation(group, quota)
self.sa_session.add(gqa)
message = f"Quota '{quota.name}' has been created with {len(in_users)} associated users and {len(in_groups)} associated groups."
self.sa_session.flush()
return quota, message
def edit_quota(self, quota, params) -> str:
if params.amount.lower() in ('unlimited', 'none', 'no limit'):
new_amount = None
else:
try:
new_amount = util.size_to_bytes(params.amount)
except (AssertionError, ValueError):
new_amount = False
if not params.amount:
raise ActionInputError('Enter a valid amount.')
elif new_amount is False:
raise ActionInputError('Unable to parse the provided amount.')
elif params.operation not in model.Quota.valid_operations:
raise ActionInputError('Enter a valid operation.')
else:
quota.amount = new_amount
quota.operation = params.operation
self.sa_session.add(quota)
self.sa_session.flush()
message = f"Quota '{quota.name}' is now '{quota.operation}{quota.display_amount}'."
return message
def _edit_quota( self, quota, params ):
if params.amount.lower() in ( 'unlimited', 'none', 'no limit' ):
new_amount = None
else:
try:
new_amount = util.size_to_bytes( params.amount )
except AssertionError:
new_amount = False
if not params.amount:
raise ActionInputError( 'Enter a valid amount' )
elif new_amount is False:
raise ActionInputError( 'Unable to parse the provided amount' )
elif params.operation not in self.app.model.Quota.valid_operations:
raise ActionInputError( 'Enter a valid operation' )
else:
quota.amount = new_amount
quota.operation = params.operation
self.sa_session.add( quota )
self.sa_session.flush()
message = "Quota '%s' is now '%s'" % ( quota.name, quota.operation + quota.display_amount )
return message
def _set_quota_default( self, quota, params ):
if params.default != 'no' and params.default not in self.app.model.DefaultQuotaAssociation.types.__dict__.values():
raise ActionInputError( 'Enter a valid default type.' )
else:
if params.default != 'no':
self.app.quota_agent.set_default_quota( params.default, quota )
message = "Quota '%s' is now the default for %s users" % ( quota.name, params.default )
else:
if quota.default:
message = "Quota '%s' is no longer the default for %s users." % ( quota.name, quota.default[0].type )
for dqa in quota.default:
self.sa_session.delete( dqa )
self.sa_session.flush()
else:
message = "Quota '%s' is not a default." % quota.name
return message
def set_quota_default(self, quota, params) -> str:
if params.default != 'no' and params.default not in model.DefaultQuotaAssociation.types.__members__.values():
raise ActionInputError('Enter a valid default type.')
else:
if params.default != 'no':
self.quota_agent.set_default_quota(params.default, quota)
message = f"Quota '{quota.name}' is now the default for {params.default} users."
else:
if quota.default:
message = f"Quota '{quota.name}' is no longer the default for {quota.default[0].type} users."
for dqa in quota.default:
self.sa_session.delete(dqa)
self.sa_session.flush()
else:
message = f"Quota '{quota.name}' is not a default."
return message
def get(self, trans, payload, **kwargs):
"""
POST /api/cloud/storage/get
gets given objects from a given cloud-based bucket to a Galaxy history.
:type trans: galaxy.webapps.base.webapp.GalaxyWebTransaction
:param trans: Galaxy web transaction
:type payload: dict
:param payload: A dictionary structure containing the following keys:
* history_id: the (encoded) id of history to which the object should be received to.
* bucket: the name of a bucket from which data should be fetched from (e.g., a bucket name on AWS S3).
* objects: a list of the names of objects to be fetched.
* authz_id: the encoded ID of CloudAuthz to be used for authorizing access to the resource
provider. You may get a list of the defined authorizations via
`/api/cloud/authz`. Also, you can use `/api/cloud/authz/create` to define a
new authorization.
* input_args [Optional; default value is an empty dict] a dictionary containing the following keys:
** `dbkey`: [Optional; default value: is `?`]
Sets the genome (e.g., `hg19`) of the objects being
fetched to Galaxy.
** `file_type`: [Optional; default value is `auto`]
Sets the Galaxy datatype (e.g., `bam`) for the
objects being fetched to Galaxy. See the following
link for a complete list of Galaxy data types:
https://galaxyproject.org/learn/datatypes/
** `space_to_tab`: [Optional; default value is `False`]
A boolean value ("true" or "false") that sets if spaces
should be converted to tab in the objects being
fetched to Galaxy. Applicable only if `to_posix_lines`
is True
** `to_posix_lines`: [Optional; default value is `Yes`]
A boolean value ("true" or "false"); if "Yes", converts
universal line endings to POSIX line endings. Set to
"False" if you upload a gzip, bz2 or zip archive
containing a binary file.
:param kwargs:
:rtype: dictionary
:return: a dictionary containing a `summary` view of the datasets copied from the given cloud-based storage.
"""
if not isinstance(payload, dict):
raise ActionInputError(
'Invalid payload data type. The payload is expected to be a dictionary, '
'but received data of type `{}`.'.format(str(type(payload))))
missing_arguments = []
encoded_history_id = payload.get("history_id", None)
if encoded_history_id is None:
missing_arguments.append("history_id")
bucket = payload.get("bucket", None)
if bucket is None:
missing_arguments.append("bucket")
objects = payload.get("objects", None)
if objects is None:
missing_arguments.append("objects")
encoded_authz_id = payload.get("authz_id", None)
if encoded_authz_id is None:
missing_arguments.append("authz_id")
if len(missing_arguments) > 0:
raise ActionInputError(
f"The following required arguments are missing in the payload: {missing_arguments}"
)
try:
history_id = self.decode_id(encoded_history_id)
except exceptions.MalformedId as e:
raise ActionInputError(f'Invalid history ID. {e}')
try:
authz_id = self.decode_id(encoded_authz_id)
except exceptions.MalformedId as e:
raise ActionInputError(f'Invalid authz ID. {e}')
if not isinstance(objects, list):
raise ActionInputError(
'The `objects` should be a list, but received an object of type {} instead.'
.format(type(objects)))
datasets = self.cloud_manager.get(trans=trans,
history_id=history_id,
bucket_name=bucket,
objects=objects,
authz_id=authz_id,
input_args=payload.get(
"input_args", None))
rtv = []
for dataset in datasets:
rtv.append(
self.datasets_serializer.serialize_to_view(dataset,
view='summary'))
return rtv
def send(self, trans, payload, **kwargs):
"""
POST /api/cloud/storage/send
Sends given dataset(s) in a given history to a given cloud-based bucket. Each dataset is named
using the label assigned to the dataset in the given history (see `HistoryDatasetAssociation.name`).
If no dataset ID is given, this API sends all the datasets belonging to a given history to a given
cloud-based bucket.
:type trans: galaxy.webapps.base.webapp.GalaxyWebTransaction
:param trans: Galaxy web transaction
:type payload: dictionary
:param payload: A dictionary structure containing the following keys:
* history_id the (encoded) id of history from which the object should be downloaed.
* bucket: the name of a bucket to which data should be sent (e.g., a bucket name on AWS S3).
* authz_id: the encoded ID of CloudAuthz to be used for authorizing access to the resource
provider. You may get a list of the defined authorizations via
`/api/cloud/authz`. Also, you can use `/api/cloud/authz/create` to define a
new authorization.
* dataset_ids: [Optional; default: None]
A list of encoded dataset IDs belonging to the specified history
that should be sent to the given bucket. If not provided, Galaxy sends
all the datasets belonging the specified history.
* overwrite_existing: [Optional; default: False]
A boolean value. If set to "True", and an object with same name of the dataset
to be sent already exist in the bucket, Galaxy replaces the existing object
with the dataset to be sent. If set to "False", Galaxy appends datetime
to the dataset name to prevent overwriting an existing object.
:rtype: dictionary
:return: Information about the (un)successfully submitted dataset send jobs,
containing the following keys:
* `bucket_name`: The name of bucket to which the listed datasets are queued
to be sent.
* `sent_dataset_labels`: A list of JSON objects with the following key-value pair:
** `object`: The name of object is queued to be created.
** `job_id`: The id of the queued send job.
* `failed_dataset_labels`: A list of JSON objects with the following key-value pair
representing the datasets Galaxy failed to create
(and queue) send job for:
** `object`: The name of object is queued to be created.
** `error`: A descriptive error message.
"""
missing_arguments = []
encoded_history_id = payload.get("history_id", None)
if encoded_history_id is None:
missing_arguments.append("history_id")
bucket = payload.get("bucket", None)
if bucket is None:
missing_arguments.append("bucket")
encoded_authz_id = payload.get("authz_id", None)
if encoded_authz_id is None:
missing_arguments.append("authz_id")
if len(missing_arguments) > 0:
raise ActionInputError(
f"The following required arguments are missing in the payload: {missing_arguments}"
)
try:
history_id = self.decode_id(encoded_history_id)
except exceptions.MalformedId as e:
raise ActionInputError(f'Invalid history ID. {e}')
try:
authz_id = self.decode_id(encoded_authz_id)
except exceptions.MalformedId as e:
raise ActionInputError(f'Invalid authz ID. {e}')
encoded_dataset_ids = payload.get("dataset_ids", None)
if encoded_dataset_ids is None:
dataset_ids = None
else:
dataset_ids = set()
invalid_dataset_ids = []
for encoded_id in encoded_dataset_ids:
try:
dataset_ids.add(self.decode_id(encoded_id))
except exceptions.MalformedId:
invalid_dataset_ids.append(encoded_id)
if len(invalid_dataset_ids) > 0:
raise ActionInputError(
"The following provided dataset IDs are invalid, please correct them and retry. "
"{}".format(invalid_dataset_ids))
log.info(
msg=
"Received api/send request for `{}` datasets using authnz with id `{}`, and history `{}`."
"".format(
"all the dataset in the given history" if not dataset_ids else
len(dataset_ids), authz_id, history_id))
sent, failed = self.cloud_manager.send(trans=trans,
history_id=history_id,
bucket_name=bucket,
authz_id=authz_id,
dataset_ids=dataset_ids,
overwrite_existing=payload.get(
"overwrite_existing",
False))
return {
'sent_dataset_labels': sent,
'failed_dataset_labels': failed,
'bucket_name': bucket
}
def download(self, trans, payload, **kwargs):
"""
* POST /api/cloud/storage/download
Downloads a given dataset in a given history to a given cloud-based bucket. Each dataset is named
using the label assigned to the dataset in the given history (see `HistoryDatasetAssociation.name`).
If no dataset ID is given, this API copies all the datasets belonging to a given history to a given
cloud-based bucket.
:type trans: galaxy.web.framework.webapp.GalaxyWebTransaction
:param trans: Galaxy web transaction
:type payload: dictionary
:param payload: A dictionary structure containing the following keys:
* history_id the (encoded) id of history from which the object should be downloaed.
* provider: the name of a cloud-based resource provider (e.g., `aws`, `azure`, or `openstack`).
* bucket: the name of a bucket to which data should be downloaded (e.g., a bucket name on AWS S3).
* credentials: a dictionary containing all the credentials required to authenticated to the
specified provider (e.g., {"secret_key": YOUR_AWS_SECRET_TOKEN,
"access_key": YOUR_AWS_ACCESS_TOKEN}).
* dataset_ids: [Optional; default: None]
A list of encoded dataset IDs belonging to the specified history
that should be downloaded to the given bucket. If not provided, Galaxy downloads
all the datasets belonging the specified history.
* overwrite_existing: [Optional; default: False]
A boolean value. If set to "True", and an object with same name of the dataset
to be downloaded already exist in the bucket, Galaxy replaces the existing object
with the dataset to be downloaded. If set to "False", Galaxy appends datetime
to the dataset name to prevent overwriting an existing object.
:param kwargs:
:rtype: dictionary
:return: Information about the downloaded datasets, including downloaded_dataset_labels
and destination bucket name.
"""
missing_arguments = []
encoded_history_id = payload.get("history_id", None)
if encoded_history_id is None:
missing_arguments.append("history_id")
provider = payload.get("provider", None)
if provider is None:
missing_arguments.append("provider")
bucket = payload.get("bucket", None)
if bucket is None:
missing_arguments.append("bucket")
credentials = payload.get("credentials", None)
if credentials is None:
missing_arguments.append("credentials")
if len(missing_arguments) > 0:
raise ActionInputError(
"The following required arguments are missing in the payload: {}"
.format(missing_arguments))
try:
history_id = self.decode_id(encoded_history_id)
except exceptions.MalformedId as e:
raise ActionInputError('Invalid history ID. {}'.format(e))
encoded_dataset_ids = payload.get("dataset_ids", None)
if encoded_dataset_ids is None:
dataset_ids = None
else:
dataset_ids = set()
invalid_dataset_ids = []
for encoded_id in encoded_dataset_ids:
try:
dataset_ids.add(self.decode_id(encoded_id))
except exceptions.MalformedId:
invalid_dataset_ids.append(encoded_id)
if len(invalid_dataset_ids) > 0:
raise ActionInputError(
"The following provided dataset IDs are invalid, please correct them and retry. "
"{}".format(invalid_dataset_ids))
uploaded = self.cloud_manager.download(trans=trans,
history_id=history_id,
provider=provider,
bucket_name=bucket,
credentials=credentials,
dataset_ids=dataset_ids,
overwrite_existing=payload.get(
"overwrite_existing",
False))
return {'downloaded_dataset_labels': uploaded, 'bucket_name': bucket}
def create(self, trans, payload, **kwargs):
"""
* POST /api/cloud/authz
Request to store the payload as a cloudauthz (cloud authorization) configuration for a user.
:type trans: galaxy.webapps.base.webapp.GalaxyWebTransaction
:param trans: Galaxy web transaction
:type payload: dict
:param payload: A dictionary structure containing the following keys:
* provider: the cloud-based resource provider to which this configuration belongs to.
* config: a dictionary containing all the configuration required to request temporary credentials
from the provider. See the following page for details:
https://galaxyproject.org/cloud/authnz/
* authn_id: the (encoded) ID of a third-party authentication of a user. To have this ID, user must
have logged-in to this Galaxy server using third-party identity (e.g., Google), or has
associated his/her Galaxy account with a third-party OIDC-based identity. See this page:
https://galaxyproject.org/admin/authentication/
* description: [Optional] a brief description for this configuration.
:param kwargs: empty dict
:rtype: dict
:return: a dictionary with the following kvp:
* status: HTTP response code
* message: A message complementary to the response code.
"""
msg_template = "Rejected user `" + str(
trans.user.id
) + "`'s request to create cloudauthz config because of {}."
if not isinstance(payload, dict):
raise ActionInputError(
'Invalid payload data type. The payload is expected to be a dictionary, but '
'received data of type `{}`.'.format(str(type(payload))))
missing_arguments = []
provider = payload.get('provider', None)
if provider is None:
missing_arguments.append('provider')
config = payload.get('config', None)
if config is None:
missing_arguments.append('config')
authn_id = payload.get('authn_id', None)
if authn_id is None and provider.lower() not in ["azure", "gcp"]:
missing_arguments.append('authn_id')
if len(missing_arguments) > 0:
log.debug(
msg_template.format(
"missing required config {}".format(missing_arguments)))
raise RequestParameterMissingException(
'The following required arguments are missing in the payload: '
'{}'.format(missing_arguments))
description = payload.get("description", "")
if not isinstance(config, dict):
log.debug(
msg_template.format(
"invalid config type `{}`, expect `dict`".format(
type(config))))
raise RequestParameterInvalidException(
'Invalid type for the required `config` variable; expect `dict` '
'but received `{}`.'.format(type(config)))
if authn_id:
try:
authn_id = self.decode_id(authn_id)
except Exception:
log.debug(
msg_template.format("cannot decode authn_id `" +
str(authn_id) + "`"))
raise MalformedId('Invalid `authn_id`!')
try:
trans.app.authnz_manager.can_user_assume_authn(trans, authn_id)
except Exception as e:
raise e
# No two authorization configuration with
# exact same key/value should exist.
for ca in trans.user.cloudauthzs:
if ca.equals(trans.user.id, provider, authn_id, config):
log.debug(
"Rejected user `{}`'s request to create cloud authorization because a similar config "
"already exists.".format(trans.user.id))
raise ActionInputError(
"A similar cloud authorization configuration is already defined."
)
try:
new_cloudauthz = self.cloudauthz_manager.create(
user_id=trans.user.id,
provider=provider,
config=config,
authn_id=authn_id,
description=description)
view = self.cloudauthz_serializer.serialize_to_view(
new_cloudauthz,
trans=trans,
**self._parse_serialization_params(kwargs, 'summary'))
log.debug(
'Created a new cloudauthz record for the user id `{}` '.format(
str(trans.user.id)))
return view
except Exception as e:
log.exception(
msg_template.format(
"exception while creating the new cloudauthz record"))
raise InternalServerError(
'An unexpected error has occurred while responding to the create request of the '
'cloudauthz API.' + unicodify(e))
def upload(self, trans, payload, **kwargs):
"""
* POST /api/cloud/storage/upload
Uploads given objects from a given cloud-based bucket to a Galaxy history.
:type trans: galaxy.web.framework.webapp.GalaxyWebTransaction
:param trans: Galaxy web transaction
:type payload: dict
:param payload: A dictionary structure containing the following keys:
* history_id: the (encoded) id of history to which the object should be uploaded to.
* provider: the name of a cloud-based resource provided (e.g., `aws`, `azure`, or `openstack`).
* bucket: the name of a bucket from which data should be uploaded from (e.g., a bucket name on AWS S3).
* objects: a list of the names of objects to be uploaded.
* credentials: a dictionary containing all the credentials required to authenticated to the
specified provider (e.g., {"secret_key": YOUR_AWS_SECRET_TOKEN, "access_key": YOUR_AWS_ACCESS_TOKEN}).
* input_args [Optional; default value is an empty dict] a dictionary containing the following keys:
** `dbkey`: [Optional; default value: is `?`]
Sets the genome (e.g., `hg19`) of the objects being
uploaded to Galaxy.
** `file_type`: [Optional; default value is `auto`]
Sets the Galaxy datatype (e.g., `bam`) for the
objects being uploaded to Galaxy. See the following
link for a complete list of Galaxy data types:
https://galaxyproject.org/learn/datatypes/
** `space_to_tab`: [Optional; default value is `False`]
A boolean value ("true" or "false") that sets if spaces
should be converted to tab in the objects being
uploaded to Galaxy. Applicable only if `to_posix_lines`
is True
** `to_posix_lines`: [Optional; default value is `Yes`]
A boolean value ("true" or "false"); if "Yes", converts
universal line endings to POSIX line endings. Set to
"False" if you upload a gzip, bz2 or zip archive
containing a binary file.
:param kwargs:
:rtype: dictionary
:return: a dictionary containing a `summary` view of the datasets copied from the given cloud-based storage.
"""
if not isinstance(payload, dict):
raise ActionInputError(
'Invalid payload data type. The payload is expected to be a dictionary, '
'but received data of type `{}`.'.format(str(type(payload))))
missing_arguments = []
encoded_history_id = payload.get("history_id", None)
if encoded_history_id is None:
missing_arguments.append("history_id")
provider = payload.get("provider", None)
if provider is None:
missing_arguments.append("provider")
bucket = payload.get("bucket", None)
if bucket is None:
missing_arguments.append("bucket")
objects = payload.get("objects", None)
if objects is None:
missing_arguments.append("objects")
credentials = payload.get("credentials", None)
if credentials is None:
missing_arguments.append("credentials")
if len(missing_arguments) > 0:
raise ActionInputError(
"The following required arguments are missing in the payload: {}"
.format(missing_arguments))
try:
history_id = self.decode_id(encoded_history_id)
except exceptions.MalformedId as e:
raise ActionInputError('Invalid history ID. {}'.format(e))
if not isinstance(objects, list):
raise ActionInputError(
'The `objects` should be a list, but received an object of type {} instead.'
.format(type(objects)))
datasets = self.cloud_manager.upload(trans=trans,
history_id=history_id,
provider=provider,
bucket_name=bucket,
objects=objects,
credentials=credentials,
input_args=payload.get(
"input_args", None))
rtv = []
for dataset in datasets:
rtv.append(
self.datasets_serializer.serialize_to_view(dataset,
view='summary'))
return rtv
