def create(self, trans, payload, **kwd):
"""
POST /api/uploads/
"""
session_id = payload.get("session_id")
session_start = payload.get("session_start")
session_chunk = payload.get("session_chunk")
if re.match('^[\w-]+$', session_id) is None:
raise MessageException("Requires a session id.")
if session_start is None:
raise MessageException("Requires a session start.")
if not hasattr(session_chunk, "file"):
raise MessageException("Requires a session chunk.")
target_file = os.path.join(trans.app.config.new_file_path, session_id)
target_size = 0
if os.path.exists(target_file):
target_size = os.path.getsize(target_file)
if session_start != target_size:
raise MessageException("Incorrect session start.")
chunk_size = os.fstat(session_chunk.file.fileno()).st_size
if chunk_size > trans.app.config.chunk_upload_size:
raise MessageException("Invalid chunk size.")
with open(target_file, "a") as f:
while True:
read_chunk = session_chunk.file.read(self.READ_CHUNK_SIZE)
if not read_chunk:
break
f.write(read_chunk)
session_chunk.file.close()
return {"message": "Successful."}
def request(self, trans, **params):
"""
GET /api/tool_shed/request
"""
tool_shed_url = params.pop("tool_shed_url")
controller = params.pop("controller")
if controller is None:
raise MessageException(
"Please provide a toolshed controller name.")
tool_shed_registry = trans.app.tool_shed_registry
if tool_shed_registry is None:
raise MessageException("Toolshed registry not available.")
if tool_shed_url in tool_shed_registry.tool_sheds.values():
pathspec = ["api", controller]
if "id" in params:
pathspec.append(params.pop("id"))
if "action" in params:
pathspec.append(params.pop("action"))
try:
return json.loads(
util.url_get(tool_shed_url,
params=dict(params),
pathspec=pathspec))
except Exception as e:
raise MessageException("Invalid server response. %s." % str(e))
else:
raise MessageException("Invalid toolshed url.")
def _get_user(self, trans, id):
user = self.get_user(trans, id)
if not user:
raise MessageException('Invalid user (%s).' % id)
if user != trans.user and not trans.user_is_admin():
raise MessageException('Access denied.')
return user
def set_password(self, trans, id, payload={}, **kwd):
"""
Allows to change a user password.
"""
password = payload.get('password')
confirm = payload.get('confirm')
current = payload.get('current')
token = payload.get('token')
token_result = None
if token:
# If a token was supplied, validate and set user
token_result = trans.sa_session.query(
trans.app.model.PasswordResetToken).get(token)
if not token_result or not token_result.expiration_time > datetime.utcnow(
):
raise MessageException(
'Invalid or expired password reset token, please request a new one.'
)
user = token_result.user
else:
# The user is changing their own password, validate their current password
user = self._get_user(trans, id)
(ok, message) = trans.app.auth_manager.check_change_password(
user, current)
if not ok:
raise MessageException(message)
if user:
# Validate the new password
message = validate_password(trans, password, confirm)
if message:
raise MessageException(message)
else:
# Save new password
user.set_password_cleartext(password)
# if we used a token, invalidate it and log the user in.
if token_result:
trans.handle_user_login(token_result.user)
token_result.expiration_time = datetime.utcnow()
trans.sa_session.add(token_result)
# Invalidate all other sessions
for other_galaxy_session in trans.sa_session.query(trans.app.model.GalaxySession) \
.filter(and_(trans.app.model.GalaxySession.table.c.user_id == user.id,
trans.app.model.GalaxySession.table.c.is_valid == true(),
trans.app.model.GalaxySession.table.c.id != trans.galaxy_session.id)):
other_galaxy_session.is_valid = False
trans.sa_session.add(other_galaxy_session)
trans.sa_session.add(user)
trans.sa_session.flush()
trans.log_event('User change password')
return {'message': 'Password has been saved.'}
raise MessageException('Failed to determine user, access denied.')
def __load_element( self, trans, element_identifier ):
#if not isinstance( element_identifier, dict ):
# # Is allowing this to just be the id of an hda too clever? Somewhat
# # consistent with other API methods though.
# element_identifier = dict( src='hda', id=str( element_identifier ) )
# Previously created collection already found in request, just pass
# through as is.
if "__object__" in element_identifier:
return element_identifier[ "__object__" ]
# dateset_identifier is dict {src=hda|ldda|hdca|new_collection, id=<encoded_id>}
try:
src_type = element_identifier.get( 'src', 'hda' )
except AttributeError:
raise MessageException( "Dataset collection element definition (%s) not dictionary-like." % element_identifier )
encoded_id = element_identifier.get( 'id', None )
if not src_type or not encoded_id:
raise RequestParameterInvalidException( "Problem decoding element identifier %s" % element_identifier )
if src_type == 'hda':
decoded_id = int( trans.app.security.decode_id( encoded_id ) )
element = self.hda_manager.get( trans, decoded_id, check_ownership=False )
elif src_type == 'ldda':
element = self.ldda_manager.get( trans, encoded_id )
elif src_type == 'hdca':
# TODO: Option to copy? Force copy? Copy or allow if not owned?
element = self.__get_history_collection_instance( trans, encoded_id ).collection
# TODO: ldca.
else:
raise RequestParameterInvalidException( "Unknown src_type parameter supplied '%s'." % src_type )
return element
def _create_instance_for_collection(self, trans, parent, name, dataset_collection, implicit_output_name=None, implicit_inputs=None, tags=None, flush=True):
if isinstance(parent, model.History):
dataset_collection_instance = self.model.HistoryDatasetCollectionAssociation(
collection=dataset_collection,
name=name,
)
if implicit_inputs:
for input_name, input_collection in implicit_inputs:
dataset_collection_instance.add_implicit_input_collection(input_name, input_collection)
if implicit_output_name:
dataset_collection_instance.implicit_output_name = implicit_output_name
log.debug("Created collection with %d elements" % (len(dataset_collection_instance.collection.elements)))
# Handle setting hid
parent.add_dataset_collection(dataset_collection_instance)
elif isinstance(parent, model.LibraryFolder):
dataset_collection_instance = self.model.LibraryDatasetCollectionAssociation(
collection=dataset_collection,
folder=parent,
name=name,
)
else:
message = "Internal logic error - create called with unknown parent type %s" % type(parent)
log.exception(message)
raise MessageException(message)
tags = self._append_tags(dataset_collection_instance, implicit_inputs, tags)
return self.__persist(dataset_collection_instance, flush=flush)
def __init__(self, step_proxy, cwl_input):
self._cwl_input = cwl_input
self.step_proxy = step_proxy
self.workflow_proxy = step_proxy._workflow_proxy
cwl_input_id = cwl_input["id"]
cwl_source_id = cwl_input.get("source", None)
if cwl_source_id is None:
if "valueFrom" not in cwl_input and "default" not in cwl_input:
msg = "Workflow step input must define a source, a valueFrom, or a default value. Obtained [%s]." % cwl_input
raise MessageException(msg)
assert cwl_input_id
step_name, input_name = split_step_references(
cwl_input_id,
multiple=False,
workflow_id=step_proxy.cwl_workflow_id)
self.step_name = step_name
self.input_name = input_name
self.cwl_input_id = cwl_input_id
self.cwl_source_id = cwl_source_id
scatter_inputs = [
split_step_references(i,
multiple=False,
workflow_id=step_proxy.cwl_workflow_id)[1]
for i in listify(step_proxy._step.tool.get("scatter", []))
]
scatter = self.input_name in scatter_inputs
self.scatter = scatter
def create(
self,
trans,
parent, # PRECONDITION: security checks on ability to add to parent occurred during load.
name,
collection_type,
element_identifiers=None,
elements=None,
implicit_collection_info=None,
):
"""
"""
if element_identifiers:
validate_input_element_identifiers(element_identifiers)
dataset_collection = self.__create_dataset_collection(
trans=trans,
collection_type=collection_type,
element_identifiers=element_identifiers,
elements=elements,
)
if isinstance(parent, model.History):
dataset_collection_instance = self.model.HistoryDatasetCollectionAssociation(
collection=dataset_collection,
name=name,
)
if implicit_collection_info:
for input_name, input_collection in implicit_collection_info[
"implicit_inputs"]:
dataset_collection_instance.add_implicit_input_collection(
input_name, input_collection)
for output_dataset in implicit_collection_info.get(
"outputs_datasets"):
output_dataset.hidden_beneath_collection_instance = dataset_collection_instance
trans.sa_session.add(output_dataset)
dataset_collection_instance.implicit_output_name = implicit_collection_info[
"implicit_output_name"]
log.debug("Created collection with %d elements" %
(len(dataset_collection_instance.collection.elements)))
# Handle setting hid
parent.add_dataset_collection(dataset_collection_instance)
elif isinstance(parent, model.LibraryFolder):
dataset_collection_instance = self.model.LibraryDatasetCollectionAssociation(
collection=dataset_collection,
folder=parent,
name=name,
)
else:
message = "Internal logic error - create called with unknown parent type %s" % type(
parent)
log.exception(message)
raise MessageException(message)
return self.__persist(dataset_collection_instance)
def update(self, trans, payload, **kwd):
"""
PUT /api/tags/
Apply a new set of tags to an item; previous tags are deleted.
"""
item_id = payload.get("item_id")
item_class = payload.get("item_class")
item_tags = payload.get("item_tags")
if item_id is None:
raise MessageException("Please provide the item id (item_id).")
if item_class is None:
raise MessageException("Please provide the item class (item_class).")
if item_tags and len(item_tags) > 0:
item_tags = ",".join(item_tags)
item = self._get_item(trans, item_class, trans.security.decode_id(item_id))
user = trans.user
self.get_tag_handler(trans).delete_item_tags(user, item)
self.get_tag_handler(trans).apply_item_tags(user, item, item_tags)
trans.sa_session.flush()
def _create_instance_for_collection(self,
trans,
parent,
name,
dataset_collection,
implicit_output_name=None,
implicit_inputs=None,
tags=None,
set_hid=True,
flush=True):
if isinstance(parent, model.History):
dataset_collection_instance = model.HistoryDatasetCollectionAssociation(
collection=dataset_collection,
name=name,
)
if implicit_inputs:
for input_name, input_collection in implicit_inputs:
dataset_collection_instance.add_implicit_input_collection(
input_name, input_collection)
if implicit_output_name:
dataset_collection_instance.implicit_output_name = implicit_output_name
log.debug("Created collection with %d elements" %
(len(dataset_collection_instance.collection.elements)))
if set_hid:
parent.add_dataset_collection(dataset_collection_instance)
elif isinstance(parent, model.LibraryFolder):
dataset_collection_instance = model.LibraryDatasetCollectionAssociation(
collection=dataset_collection,
folder=parent,
name=name,
)
else:
message = "Internal logic error - create called with unknown parent type %s" % type(
parent)
log.exception(message)
raise MessageException(message)
# Tags may be coming in as a dictionary of tag model objects if copying them from other
# existing Galaxy objects or as a list of strings if the tags are coming from user supplied
# values.
if isinstance(tags, list):
assert implicit_inputs is None, implicit_inputs
tags = self.tag_handler.add_tags_from_list(
trans.user, dataset_collection_instance, tags)
else:
tags = self._append_tags(dataset_collection_instance,
implicit_inputs, tags)
return self.__persist(dataset_collection_instance, flush=flush)
def __load_element(self, trans, element_identifier, hide_source_items, copy_elements, history=None):
# if not isinstance( element_identifier, dict ):
# # Is allowing this to just be the id of an hda too clever? Somewhat
# # consistent with other API methods though.
# element_identifier = dict( src='hda', id=str( element_identifier ) )
# Previously created collection already found in request, just pass
# through as is.
if "__object__" in element_identifier:
the_object = element_identifier["__object__"]
if the_object is not None and the_object.id:
context = self.model.context
if the_object not in context:
the_object = context.query(type(the_object)).get(the_object.id)
return the_object
# dataset_identifier is dict {src=hda|ldda|hdca|new_collection, id=<encoded_id>}
try:
src_type = element_identifier.get('src', 'hda')
except AttributeError:
raise MessageException("Dataset collection element definition (%s) not dictionary-like." % element_identifier)
encoded_id = element_identifier.get('id', None)
if not src_type or not encoded_id:
message_template = "Problem decoding element identifier %s - must contain a 'src' and a 'id'."
message = message_template % element_identifier
raise RequestParameterInvalidException(message)
tags = element_identifier.pop('tags', None)
tag_str = ''
if tags:
tag_str = ",".join(str(_) for _ in tags)
if src_type == 'hda':
decoded_id = int(trans.app.security.decode_id(encoded_id))
hda = self.hda_manager.get_accessible(decoded_id, trans.user)
if copy_elements:
element = self.hda_manager.copy(hda, history=history or trans.history, hide_copy=True)
else:
element = hda
if hide_source_items and self.hda_manager.get_owned(hda.id, user=trans.user, current_history=history or trans.history):
hda.visible = False
self.tag_handler.apply_item_tags(user=trans.user, item=element, tags_str=tag_str)
elif src_type == 'ldda':
element = self.ldda_manager.get(trans, encoded_id, check_accessible=True)
element = element.to_history_dataset_association(history or trans.history, add_to_history=True, visible=not hide_source_items)
self.tag_handler.apply_item_tags(user=trans.user, item=element, tags_str=tag_str)
elif src_type == 'hdca':
# TODO: Option to copy? Force copy? Copy or allow if not owned?
element = self.__get_history_collection_instance(trans, encoded_id).collection
# TODO: ldca.
else:
raise RequestParameterInvalidException("Unknown src_type parameter supplied '%s'." % src_type)
return element
def _get(self, url, params=None):
response = requests.get(url, params=params)
if response.ok:
return response.json()
else:
code = response.status_code
message = response.text
try:
trs_error_dict = response.json()
code = int(trs_error_dict["code"])
message = trs_error_dict["message"]
except Exception:
pass
raise MessageException.from_code(code, message)
def delete_custom_builds(self, trans, id, key, payload={}, **kwd):
"""
DELETE /api/users/{id}/custom_builds/{key}
Delete a custom build.
:param id: the encoded id of the user
:type id: str
:param id: custom build key to be deleted
:type id: str
"""
user = self._get_user(trans, id)
dbkeys = json.loads(user.preferences['dbkeys']) if 'dbkeys' in user.preferences else {}
if key and key in dbkeys:
del dbkeys[key]
user.preferences['dbkeys'] = json.dumps(dbkeys)
trans.sa_session.flush()
return {'message': 'Deleted %s.' % key}
else:
raise MessageException('Could not find and delete build (%s).' % key)
def _add_entry(self,
entry,
allow_duplicates=True,
persist=False,
entry_source=None,
**kwd):
# accepts dict or list of columns
if isinstance(entry, dict):
fields = []
for column_name in self.get_column_name_list():
if column_name not in entry:
log.debug(
"Using default column value for column '%s' when adding data table entry (%s) to table '%s'.",
column_name, entry, self.name)
field_value = self.get_empty_field_by_name(column_name)
else:
field_value = entry[column_name]
fields.append(field_value)
else:
fields = entry
if self.largest_index < len(fields):
fields = self._replace_field_separators(fields)
if (allow_duplicates and self.allow_duplicate_entries
) or fields not in self.get_fields():
self.data.append(fields)
else:
raise MessageException(
f"Attempted to add fields ({fields}) to data table '{self.name}', but this entry already exists and allow_duplicates is False."
)
else:
raise MessageException(
f"Attempted to add fields ({fields}) to data table '{self.name}', but there were not enough fields specified ( {len(fields)} < {self.largest_index + 1} )."
)
filename = None
if persist:
filename = self.get_filename_for_source(entry_source)
if filename is None:
# If we reach this point, there is no data table with a corresponding .loc file.
raise MessageException(
f"Unable to determine filename for persisting data table '{self.name}' values: '{self.fields}'."
)
else:
log.debug("Persisting changes to file: %s", filename)
with FileLock(filename):
try:
if os.path.exists(filename):
data_table_fh = open(filename, 'r+b')
if os.stat(filename).st_size > 0:
# ensure last existing line ends with new line
data_table_fh.seek(-1, 2) # last char in file
last_char = data_table_fh.read(1)
if last_char not in [b'\n', b'\r']:
data_table_fh.write(b'\n')
else:
data_table_fh = open(filename, 'wb')
except OSError as e:
log.exception('Error opening data table file (%s): %s',
filename, e)
raise
fields = f"{self.separator.join(fields)}\n"
data_table_fh.write(fields.encode('utf-8'))
def _check_can_convert_to_pdf_or_raise():
"""Checks if the HTML to PDF converter is available."""
if not weasyprint:
raise MessageException("PDF conversion service not available.")
def add_custom_builds(self, trans, id, key, payload={}, **kwd):
"""
PUT /api/users/{id}/custom_builds/{key}
Add new custom build.
:param id: the encoded id of the user
:type id: str
:param id: custom build key
:type id: str
:param payload: data with new build details
:type payload: dict
"""
user = self._get_user(trans, id)
dbkeys = json.loads(user.preferences['dbkeys']
) if 'dbkeys' in user.preferences else {}
name = payload.get('name')
len_type = payload.get('len|type')
len_value = payload.get('len|value')
if len_type not in ['file', 'fasta', 'text'] or not len_value:
raise MessageException('Please specify a valid data source type.')
if not name or not key:
raise MessageException(
'You must specify values for all the fields.')
elif key in dbkeys:
raise MessageException(
'There is already a custom build with that key. Delete it first if you want to replace it.'
)
else:
# Have everything needed; create new build.
build_dict = {'name': name}
if len_type in ['text', 'file']:
# Create new len file
new_len = trans.app.model.HistoryDatasetAssociation(
extension='len',
create_dataset=True,
sa_session=trans.sa_session)
trans.sa_session.add(new_len)
new_len.name = name
new_len.visible = False
new_len.state = trans.app.model.Job.states.OK
new_len.info = 'custom build .len file'
try:
trans.app.object_store.create(new_len.dataset)
except ObjectInvalid:
raise MessageException(
'Unable to create output dataset: object store is full.'
)
trans.sa_session.flush()
counter = 0
lines_skipped = 0
f = open(new_len.file_name, 'w')
# LEN files have format:
# <chrom_name><tab><chrom_length>
for line in len_value.split('\n'):
# Splits at the last whitespace in the line
lst = line.strip().rsplit(None, 1)
if not lst or len(lst) < 2:
lines_skipped += 1
continue
chrom, length = lst[0], lst[1]
try:
length = int(length)
except ValueError:
lines_skipped += 1
continue
if chrom != escape(chrom):
build_dict[
'message'] = 'Invalid chromosome(s) with HTML detected and skipped.'
lines_skipped += 1
continue
counter += 1
f.write('%s\t%s\n' % (chrom, length))
f.close()
build_dict['len'] = new_len.id
build_dict['count'] = counter
else:
build_dict['fasta'] = trans.security.decode_id(len_value)
dataset = trans.sa_session.query(
trans.app.model.HistoryDatasetAssociation).get(
build_dict['fasta'])
try:
new_len = dataset.get_converted_dataset(trans, 'len')
new_linecount = new_len.get_converted_dataset(
trans, 'linecount')
build_dict['len'] = new_len.id
build_dict['linecount'] = new_linecount.id
except:
raise MessageException('Failed to convert dataset.')
dbkeys[key] = build_dict
user.preferences['dbkeys'] = json.dumps(dbkeys)
trans.sa_session.flush()
return build_dict
def upload(self,
trans,
history_id,
provider,
bucket_name,
objects,
credentials,
input_args=None):
"""
Implements the logic of uploading a file from a cloud-based storage (e.g., Amazon S3)
and persisting it as a Galaxy dataset.
:type trans: galaxy.web.framework.webapp.GalaxyWebTransaction
:param trans: Galaxy web transaction
:type history_id: string
:param history_id: the (decoded) id of history to which the object should be uploaded to.
:type provider: string
:param provider: the name of cloud-based resource provided. A list of supported providers is given in
`SUPPORTED_PROVIDERS` variable.
:type bucket_name: string
:param bucket_name: the name of a bucket from which data should be uploaded (e.g., a bucket name on AWS S3).
:type objects: list of string
:param objects: the name of objects to be uploaded.
:type credentials: dict
:param credentials: a dictionary containing all the credentials required to authenticated to the
specified provider (e.g., {"secret_key": YOUR_AWS_SECRET_TOKEN,
"access_key": YOUR_AWS_ACCESS_TOKEN}).
:type input_args: dict
:param input_args: a [Optional] a dictionary of input parameters:
dbkey, file_type, space_to_tab, to_posix_lines (see galaxy/webapps/galaxy/api/cloud.py)
:rtype: list of galaxy.model.Dataset
:return: a list of datasets created for the uploaded files.
"""
if CloudProviderFactory is None:
raise Exception(NO_CLOUDBRIDGE_ERROR_MESSAGE)
if input_args is None:
input_args = {}
connection = self._configure_provider(provider, credentials)
try:
bucket = connection.storage.buckets.get(bucket_name)
if bucket is None:
raise RequestParameterInvalidException(
"The bucket `{}` not found.".format(bucket_name))
except Exception as e:
raise ItemAccessibilityException(
"Could not get the bucket `{}`: {}".format(
bucket_name, str(e)))
datasets = []
for obj in objects:
try:
key = bucket.objects.get(obj)
except Exception as e:
raise MessageException(
"The following error occurred while getting the object {}: {}"
.format(obj, str(e)))
if key is None:
raise ObjectNotFound(
"Could not get the object `{}`.".format(obj))
params = Params(self._get_inputs(obj, key, input_args),
sanitize=False)
incoming = params.__dict__
history = trans.sa_session.query(
trans.app.model.History).get(history_id)
if not history:
raise ObjectNotFound("History with ID `{}` not found.".format(
trans.app.security.encode_id(history_id)))
output = trans.app.toolbox.get_tool('upload1').handle_input(
trans, incoming, history=history)
job_errors = output.get('job_errors', [])
if job_errors:
raise ValueError(
'Following error occurred while uploading the given object(s) from {}: {}'
.format(provider, job_errors))
else:
for d in output['out_data']:
datasets.append(d[1].dataset)
return datasets
def set_information(self, trans, id, payload={}, **kwd):
"""
POST /api/users/{id}/information
Save a user's email, username, addresses etc.
:param id: the encoded id of the user
:type id: str
:param payload: data with new settings
:type payload: dict
"""
user = self._get_user(trans, id)
email = payload.get('email')
username = payload.get('username')
if email or username:
message = self._validate_email_publicname(
email, username) or validate_email(trans, email, user)
if not message and username:
message = validate_publicname(trans, username, user)
if message:
raise MessageException(message)
if user.email != email:
# Update user email and user's private role name which must match
private_role = trans.app.security_agent.get_private_user_role(
user)
private_role.name = email
private_role.description = 'Private role for ' + email
user.email = email
trans.sa_session.add(user)
trans.sa_session.add(private_role)
trans.sa_session.flush()
if trans.app.config.user_activation_on:
# Deactivate the user if email was changed and activation is on.
user.active = False
if self.send_verification_email(trans, user.email,
user.username):
message = 'The login information has been updated with the changes.<br>Verification email has been sent to your new email address. Please verify it by clicking the activation link in the email.<br>Please check your spam/trash folder in case you cannot find the message.'
else:
message = 'Unable to send activation email, please contact your local Galaxy administrator.'
if trans.app.config.error_email_to is not None:
message += ' Contact: %s' % trans.app.config.error_email_to
raise MessageException(message)
if user.username != username:
# Update public name
user.username = username
# Update user custom form
user_info_form_id = payload.get('info|form_id')
if user_info_form_id:
prefix = 'info|'
user_info_form = trans.sa_session.query(
trans.app.model.FormDefinition).get(
trans.security.decode_id(user_info_form_id))
user_info_values = {}
for item in payload:
if item.startswith(prefix):
user_info_values[item[len(prefix):]] = payload[item]
form_values = trans.model.FormValues(user_info_form,
user_info_values)
trans.sa_session.add(form_values)
user.values = form_values
# Update user addresses
address_dicts = {}
address_count = 0
for item in payload:
match = re.match(r'^address_(?P<index>\d+)\|(?P<attribute>\S+)',
item)
if match:
groups = match.groupdict()
index = int(groups['index'])
attribute = groups['attribute']
address_dicts[index] = address_dicts.get(index) or {}
address_dicts[index][attribute] = payload[item]
address_count = max(address_count, index + 1)
user.addresses = []
for index in range(0, address_count):
d = address_dicts[index]
if d.get('id'):
try:
user_address = trans.sa_session.query(
trans.app.model.UserAddress).get(
trans.security.decode_id(d['id']))
except Exception as e:
raise MessageException(
'Failed to access user address (%s). %s' %
(d['id'], e))
else:
user_address = trans.model.UserAddress()
trans.log_event('User address added')
for field in AddressField.fields():
if str(field[2]).lower() == 'required' and not d.get(field[0]):
raise MessageException('Address %s: %s (%s) required.' %
(index + 1, field[1], field[0]))
setattr(user_address, field[0], str(d.get(field[0], '')))
user_address.user = user
user.addresses.append(user_address)
trans.sa_session.add(user_address)
trans.sa_session.add(user)
trans.sa_session.flush()
trans.log_event('User information added')
return {'message': 'User information has been saved.'}
def create(self,
trans,
parent,
name,
collection_type,
element_identifiers=None,
elements=None,
implicit_collection_info=None,
trusted_identifiers=None,
hide_source_items=False,
tags=None):
"""
PRECONDITION: security checks on ability to add to parent
occurred during load.
"""
# Trust embedded, newly created objects created by tool subsystem.
if trusted_identifiers is None:
trusted_identifiers = implicit_collection_info is not None
if element_identifiers and not trusted_identifiers:
validate_input_element_identifiers(element_identifiers)
dataset_collection = self.create_dataset_collection(
trans=trans,
collection_type=collection_type,
element_identifiers=element_identifiers,
elements=elements,
hide_source_items=hide_source_items,
)
if isinstance(parent, model.History):
dataset_collection_instance = self.model.HistoryDatasetCollectionAssociation(
collection=dataset_collection,
name=name,
)
if implicit_collection_info:
for input_name, input_collection in implicit_collection_info[
"implicit_inputs"]:
dataset_collection_instance.add_implicit_input_collection(
input_name, input_collection)
for output_dataset in implicit_collection_info.get("outputs"):
if output_dataset not in trans.sa_session:
output_dataset = trans.sa_session.query(
type(output_dataset)).get(output_dataset.id)
if isinstance(output_dataset,
model.HistoryDatasetAssociation):
output_dataset.hidden_beneath_collection_instance = dataset_collection_instance
elif isinstance(output_dataset,
model.HistoryDatasetCollectionAssociation):
dataset_collection_instance.add_implicit_input_collection(
input_name, input_collection)
else:
# dataset collection, don't need to do anything...
pass
trans.sa_session.add(output_dataset)
dataset_collection_instance.implicit_output_name = implicit_collection_info[
"implicit_output_name"]
log.debug("Created collection with %d elements" %
(len(dataset_collection_instance.collection.elements)))
# Handle setting hid
parent.add_dataset_collection(dataset_collection_instance)
elif isinstance(parent, model.LibraryFolder):
dataset_collection_instance = self.model.LibraryDatasetCollectionAssociation(
collection=dataset_collection,
folder=parent,
name=name,
)
else:
message = "Internal logic error - create called with unknown parent type %s" % type(
parent)
log.exception(message)
raise MessageException(message)
tags = tags or {}
if implicit_collection_info:
for k, v in implicit_collection_info.get('implicit_inputs', []):
for tag in [t for t in v.tags if t.user_tname == 'name']:
tags[tag.value] = tag
for _, tag in tags.items():
dataset_collection_instance.tags.append(tag.copy())
return self.__persist(dataset_collection_instance)
def error(message):
raise MessageException(message, type='error')
def send(self,
trans,
history_id,
bucket_name,
authz_id,
dataset_ids=None,
overwrite_existing=False):
"""
Implements the logic of sending dataset(s) from a given history to a given cloud-based storage
(e.g., Amazon S3).
:type trans: galaxy.webapps.base.webapp.GalaxyWebTransaction
:param trans: Galaxy web transaction
:type history_id: string
:param history_id: the (encoded) id of history from which the object should be sent.
:type bucket_name: string
:param bucket_name: the name of a bucket to which data should be sent (e.g., a bucket
name on AWS S3).
:type authz_id: int
:param authz_id: the ID of CloudAuthz to be used for authorizing access to the resource provider.
You may get a list of the defined authorizations via `/api/cloud/authz`. Also,
you can use `/api/cloud/authz/create` to define a new authorization.
:type dataset_ids: set
:param dataset_ids: [Optional] The list of (decoded) dataset ID(s) belonging to the given
history which should be sent to the given provider. If not provided,
Galaxy sends all the datasets belonging to the given history.
:type overwrite_existing: boolean
:param overwrite_existing: [Optional] If set to "True", and an object with same name of the
dataset to be sent already exist in the bucket, Galaxy replaces
the existing object with the dataset to be sent. If set to
"False", Galaxy appends datetime to the dataset name to prevent
overwriting the existing object.
:rtype: tuple
:return: A tuple of two lists of labels of the objects that were successfully and
unsuccessfully sent to cloud.
"""
if CloudProviderFactory is None:
raise Exception(NO_CLOUDBRIDGE_ERROR_MESSAGE)
if not hasattr(trans.app, 'authnz_manager'):
err_msg = "The OpenID Connect protocol, a required feature for sending data to cloud, " \
"is not enabled on this Galaxy instance."
log.debug(err_msg)
raise MessageException(err_msg)
cloudauthz = trans.app.authnz_manager.try_get_authz_config(
trans.sa_session, trans.user.id, authz_id)
history = trans.sa_session.query(
trans.app.model.History).get(history_id)
if not history:
raise ObjectNotFound("History with ID `{}` not found.".format(
trans.app.security.encode_id(history_id)))
sent = []
failed = []
for hda in history.datasets:
if hda.deleted or hda.purged or hda.state != "ok" or hda.creating_job.tool_id == SEND_TOOL:
continue
if dataset_ids is None or hda.dataset.id in dataset_ids:
try:
object_label = hda.name.replace(" ", "_")
args = {
# We encode ID here because the tool wrapper expects
# an encoded ID and attempts decoding it.
"authz_id": trans.security.encode_id(cloudauthz.id),
"bucket": bucket_name,
"object_label": object_label,
"filename": hda,
"overwrite_existing": overwrite_existing
}
incoming = (util.Params(args, sanitize=False)).__dict__
d2c = trans.app.toolbox.get_tool(SEND_TOOL,
SEND_TOOL_VERSION)
if not d2c:
log.debug(
f"Failed to get the `send` tool per user `{trans.user.id}` request."
)
failed.append(
json.dumps({
"object":
object_label,
"error":
"Unable to get the `send` tool."
}))
continue
res = d2c.execute(trans, incoming, history=history)
job = res[0]
sent.append(
json.dumps({
"object":
object_label,
"job_id":
trans.app.security.encode_id(job.id)
}))
except Exception as e:
err_msg = "maybe invalid or unauthorized credentials. {}".format(
util.unicodify(e))
log.debug(
"Failed to send the dataset `{}` per user `{}` request to cloud, {}"
.format(object_label, trans.user.id, err_msg))
failed.append(
json.dumps({
"object": object_label,
"error": err_msg
}))
return sent, failed
def get(self,
trans,
history_id,
bucket_name,
objects,
authz_id,
input_args=None):
"""
Implements the logic of getting a file from a cloud-based storage (e.g., Amazon S3)
and persisting it as a Galaxy dataset.
This manager does NOT require use credentials, instead, it uses a more secure method,
which leverages CloudAuthz (https://github.com/galaxyproject/cloudauthz) and automatically
requests temporary credentials to access the defined resources.
:type trans: galaxy.webapps.base.webapp.GalaxyWebTransaction
:param trans: Galaxy web transaction
:type history_id: string
:param history_id: the (decoded) id of history to which the object should be received to.
:type bucket_name: string
:param bucket_name: the name of a bucket from which data should be fetched (e.g., a bucket name on AWS S3).
:type objects: list of string
:param objects: the name of objects to be fetched.
:type authz_id: int
:param authz_id: the ID of CloudAuthz to be used for authorizing access to the resource provider. You may
get a list of the defined authorizations sending GET to `/api/cloud/authz`. Also, you can
POST to `/api/cloud/authz` to define a new authorization.
:type input_args: dict
:param input_args: a [Optional] a dictionary of input parameters:
dbkey, file_type, space_to_tab, to_posix_lines (see galaxy/webapps/galaxy/api/cloud.py)
:rtype: list of galaxy.model.Dataset
:return: a list of datasets created for the fetched files.
"""
if CloudProviderFactory is None:
raise Exception(NO_CLOUDBRIDGE_ERROR_MESSAGE)
if input_args is None:
input_args = {}
if not hasattr(trans.app, 'authnz_manager'):
err_msg = "The OpenID Connect protocol, a required feature for getting data from cloud, " \
"is not enabled on this Galaxy instance."
log.debug(err_msg)
raise MessageException(err_msg)
cloudauthz = trans.app.authnz_manager.try_get_authz_config(
trans.sa_session, trans.user.id, authz_id)
credentials = trans.app.authnz_manager.get_cloud_access_credentials(
cloudauthz, trans.sa_session, trans.user.id, trans.request)
connection = self.configure_provider(cloudauthz.provider, credentials)
try:
bucket = connection.storage.buckets.get(bucket_name)
if bucket is None:
raise RequestParameterInvalidException(
f"The bucket `{bucket_name}` not found.")
except Exception as e:
raise ItemAccessibilityException(
"Could not get the bucket `{}`: {}".format(
bucket_name, util.unicodify(e)))
datasets = []
for obj in objects:
try:
key = bucket.objects.get(obj)
except Exception as e:
raise MessageException(
"The following error occurred while getting the object {}: {}"
.format(obj, util.unicodify(e)))
if key is None:
log.exception(
"Could not get object `{}` for user `{}`. Object may not exist, or the provided credentials are "
"invalid or not authorized to read the bucket/object.".
format(obj, trans.user.id))
raise ObjectNotFound(
"Could not get the object `{}`. Please check if the object exists, and credentials are valid and "
"authorized to read the bucket and object. ".format(obj))
params = Params(self._get_inputs(obj, key, input_args),
sanitize=False)
incoming = params.__dict__
history = trans.sa_session.query(
trans.app.model.History).get(history_id)
if not history:
raise ObjectNotFound("History with ID `{}` not found.".format(
trans.app.security.encode_id(history_id)))
output = trans.app.toolbox.get_tool('upload1').handle_input(
trans, incoming, history=history)
job_errors = output.get('job_errors', [])
if job_errors:
raise ValueError(
'Following error occurred while getting the given object(s) from {}: {}'
.format(cloudauthz.provider, job_errors))
else:
for d in output['out_data']:
datasets.append(d[1].dataset)
return datasets
def create(
self,
trans,
parent,
# PRECONDITION: security checks on ability to add to parent
# occurred during load.
name,
collection_type,
element_identifiers=None,
elements=None,
implicit_collection_info=None,
trusted_identifiers=None, # Trust preloaded element objects
):
"""
"""
# Trust embedded, newly created objects created by tool subsystem.
if trusted_identifiers is None:
trusted_identifiers = implicit_collection_info is not None
if element_identifiers and not trusted_identifiers:
validate_input_element_identifiers(element_identifiers)
dataset_collection = self.__create_dataset_collection(
trans=trans,
collection_type=collection_type,
element_identifiers=element_identifiers,
elements=elements,
)
if isinstance(parent, model.History):
dataset_collection_instance = self.model.HistoryDatasetCollectionAssociation(
collection=dataset_collection,
name=name,
)
if implicit_collection_info:
for input_name, input_collection in implicit_collection_info[
"implicit_inputs"]:
dataset_collection_instance.add_implicit_input_collection(
input_name, input_collection)
for output_dataset in implicit_collection_info.get("outputs"):
if isinstance(output_dataset,
model.HistoryDatasetAssociation):
output_dataset.hidden_beneath_collection_instance = dataset_collection_instance
elif isinstance(output_dataset,
model.HistoryDatasetCollectionAssociation):
dataset_collection_instance.add_implicit_input_collection(
input_name, input_collection)
else:
# dataset collection, don't need to do anything...
pass
trans.sa_session.add(output_dataset)
dataset_collection_instance.implicit_output_name = implicit_collection_info[
"implicit_output_name"]
log.debug("Created collection with %d elements" %
(len(dataset_collection_instance.collection.elements)))
# Handle setting hid
parent.add_dataset_collection(dataset_collection_instance)
elif isinstance(parent, model.LibraryFolder):
dataset_collection_instance = self.model.LibraryDatasetCollectionAssociation(
collection=dataset_collection,
folder=parent,
name=name,
)
else:
message = "Internal logic error - create called with unknown parent type %s" % type(
parent)
log.exception(message)
raise MessageException(message)
return self.__persist(dataset_collection_instance)
