def create(self, trans, payload, **kwd): """ POST /api/uploads/ """ session_id = payload.get("session_id") session_start = payload.get("session_start") session_chunk = payload.get("session_chunk") if re.match('^[\w-]+$', session_id) is None: raise MessageException("Requires a session id.") if session_start is None: raise MessageException("Requires a session start.") if not hasattr(session_chunk, "file"): raise MessageException("Requires a session chunk.") target_file = os.path.join(trans.app.config.new_file_path, session_id) target_size = 0 if os.path.exists(target_file): target_size = os.path.getsize(target_file) if session_start != target_size: raise MessageException("Incorrect session start.") chunk_size = os.fstat(session_chunk.file.fileno()).st_size if chunk_size > trans.app.config.chunk_upload_size: raise MessageException("Invalid chunk size.") with open(target_file, "a") as f: while True: read_chunk = session_chunk.file.read(self.READ_CHUNK_SIZE) if not read_chunk: break f.write(read_chunk) session_chunk.file.close() return {"message": "Successful."}
def request(self, trans, **params): """ GET /api/tool_shed/request """ tool_shed_url = params.pop("tool_shed_url") controller = params.pop("controller") if controller is None: raise MessageException( "Please provide a toolshed controller name.") tool_shed_registry = trans.app.tool_shed_registry if tool_shed_registry is None: raise MessageException("Toolshed registry not available.") if tool_shed_url in tool_shed_registry.tool_sheds.values(): pathspec = ["api", controller] if "id" in params: pathspec.append(params.pop("id")) if "action" in params: pathspec.append(params.pop("action")) try: return json.loads( util.url_get(tool_shed_url, params=dict(params), pathspec=pathspec)) except Exception as e: raise MessageException("Invalid server response. %s." % str(e)) else: raise MessageException("Invalid toolshed url.")
def _get_user(self, trans, id): user = self.get_user(trans, id) if not user: raise MessageException('Invalid user (%s).' % id) if user != trans.user and not trans.user_is_admin(): raise MessageException('Access denied.') return user
def set_password(self, trans, id, payload={}, **kwd): """ Allows to change a user password. """ password = payload.get('password') confirm = payload.get('confirm') current = payload.get('current') token = payload.get('token') token_result = None if token: # If a token was supplied, validate and set user token_result = trans.sa_session.query( trans.app.model.PasswordResetToken).get(token) if not token_result or not token_result.expiration_time > datetime.utcnow( ): raise MessageException( 'Invalid or expired password reset token, please request a new one.' ) user = token_result.user else: # The user is changing their own password, validate their current password user = self._get_user(trans, id) (ok, message) = trans.app.auth_manager.check_change_password( user, current) if not ok: raise MessageException(message) if user: # Validate the new password message = validate_password(trans, password, confirm) if message: raise MessageException(message) else: # Save new password user.set_password_cleartext(password) # if we used a token, invalidate it and log the user in. if token_result: trans.handle_user_login(token_result.user) token_result.expiration_time = datetime.utcnow() trans.sa_session.add(token_result) # Invalidate all other sessions for other_galaxy_session in trans.sa_session.query(trans.app.model.GalaxySession) \ .filter(and_(trans.app.model.GalaxySession.table.c.user_id == user.id, trans.app.model.GalaxySession.table.c.is_valid == true(), trans.app.model.GalaxySession.table.c.id != trans.galaxy_session.id)): other_galaxy_session.is_valid = False trans.sa_session.add(other_galaxy_session) trans.sa_session.add(user) trans.sa_session.flush() trans.log_event('User change password') return {'message': 'Password has been saved.'} raise MessageException('Failed to determine user, access denied.')
def __load_element( self, trans, element_identifier ): #if not isinstance( element_identifier, dict ): # # Is allowing this to just be the id of an hda too clever? Somewhat # # consistent with other API methods though. # element_identifier = dict( src='hda', id=str( element_identifier ) ) # Previously created collection already found in request, just pass # through as is. if "__object__" in element_identifier: return element_identifier[ "__object__" ] # dateset_identifier is dict {src=hda|ldda|hdca|new_collection, id=<encoded_id>} try: src_type = element_identifier.get( 'src', 'hda' ) except AttributeError: raise MessageException( "Dataset collection element definition (%s) not dictionary-like." % element_identifier ) encoded_id = element_identifier.get( 'id', None ) if not src_type or not encoded_id: raise RequestParameterInvalidException( "Problem decoding element identifier %s" % element_identifier ) if src_type == 'hda': decoded_id = int( trans.app.security.decode_id( encoded_id ) ) element = self.hda_manager.get( trans, decoded_id, check_ownership=False ) elif src_type == 'ldda': element = self.ldda_manager.get( trans, encoded_id ) elif src_type == 'hdca': # TODO: Option to copy? Force copy? Copy or allow if not owned? element = self.__get_history_collection_instance( trans, encoded_id ).collection # TODO: ldca. else: raise RequestParameterInvalidException( "Unknown src_type parameter supplied '%s'." % src_type ) return element
def _create_instance_for_collection(self, trans, parent, name, dataset_collection, implicit_output_name=None, implicit_inputs=None, tags=None, flush=True): if isinstance(parent, model.History): dataset_collection_instance = self.model.HistoryDatasetCollectionAssociation( collection=dataset_collection, name=name, ) if implicit_inputs: for input_name, input_collection in implicit_inputs: dataset_collection_instance.add_implicit_input_collection(input_name, input_collection) if implicit_output_name: dataset_collection_instance.implicit_output_name = implicit_output_name log.debug("Created collection with %d elements" % (len(dataset_collection_instance.collection.elements))) # Handle setting hid parent.add_dataset_collection(dataset_collection_instance) elif isinstance(parent, model.LibraryFolder): dataset_collection_instance = self.model.LibraryDatasetCollectionAssociation( collection=dataset_collection, folder=parent, name=name, ) else: message = "Internal logic error - create called with unknown parent type %s" % type(parent) log.exception(message) raise MessageException(message) tags = self._append_tags(dataset_collection_instance, implicit_inputs, tags) return self.__persist(dataset_collection_instance, flush=flush)
def __init__(self, step_proxy, cwl_input): self._cwl_input = cwl_input self.step_proxy = step_proxy self.workflow_proxy = step_proxy._workflow_proxy cwl_input_id = cwl_input["id"] cwl_source_id = cwl_input.get("source", None) if cwl_source_id is None: if "valueFrom" not in cwl_input and "default" not in cwl_input: msg = "Workflow step input must define a source, a valueFrom, or a default value. Obtained [%s]." % cwl_input raise MessageException(msg) assert cwl_input_id step_name, input_name = split_step_references( cwl_input_id, multiple=False, workflow_id=step_proxy.cwl_workflow_id) self.step_name = step_name self.input_name = input_name self.cwl_input_id = cwl_input_id self.cwl_source_id = cwl_source_id scatter_inputs = [ split_step_references(i, multiple=False, workflow_id=step_proxy.cwl_workflow_id)[1] for i in listify(step_proxy._step.tool.get("scatter", [])) ] scatter = self.input_name in scatter_inputs self.scatter = scatter
def create( self, trans, parent, # PRECONDITION: security checks on ability to add to parent occurred during load. name, collection_type, element_identifiers=None, elements=None, implicit_collection_info=None, ): """ """ if element_identifiers: validate_input_element_identifiers(element_identifiers) dataset_collection = self.__create_dataset_collection( trans=trans, collection_type=collection_type, element_identifiers=element_identifiers, elements=elements, ) if isinstance(parent, model.History): dataset_collection_instance = self.model.HistoryDatasetCollectionAssociation( collection=dataset_collection, name=name, ) if implicit_collection_info: for input_name, input_collection in implicit_collection_info[ "implicit_inputs"]: dataset_collection_instance.add_implicit_input_collection( input_name, input_collection) for output_dataset in implicit_collection_info.get( "outputs_datasets"): output_dataset.hidden_beneath_collection_instance = dataset_collection_instance trans.sa_session.add(output_dataset) dataset_collection_instance.implicit_output_name = implicit_collection_info[ "implicit_output_name"] log.debug("Created collection with %d elements" % (len(dataset_collection_instance.collection.elements))) # Handle setting hid parent.add_dataset_collection(dataset_collection_instance) elif isinstance(parent, model.LibraryFolder): dataset_collection_instance = self.model.LibraryDatasetCollectionAssociation( collection=dataset_collection, folder=parent, name=name, ) else: message = "Internal logic error - create called with unknown parent type %s" % type( parent) log.exception(message) raise MessageException(message) return self.__persist(dataset_collection_instance)
def update(self, trans, payload, **kwd): """ PUT /api/tags/ Apply a new set of tags to an item; previous tags are deleted. """ item_id = payload.get("item_id") item_class = payload.get("item_class") item_tags = payload.get("item_tags") if item_id is None: raise MessageException("Please provide the item id (item_id).") if item_class is None: raise MessageException("Please provide the item class (item_class).") if item_tags and len(item_tags) > 0: item_tags = ",".join(item_tags) item = self._get_item(trans, item_class, trans.security.decode_id(item_id)) user = trans.user self.get_tag_handler(trans).delete_item_tags(user, item) self.get_tag_handler(trans).apply_item_tags(user, item, item_tags) trans.sa_session.flush()
def _create_instance_for_collection(self, trans, parent, name, dataset_collection, implicit_output_name=None, implicit_inputs=None, tags=None, set_hid=True, flush=True): if isinstance(parent, model.History): dataset_collection_instance = model.HistoryDatasetCollectionAssociation( collection=dataset_collection, name=name, ) if implicit_inputs: for input_name, input_collection in implicit_inputs: dataset_collection_instance.add_implicit_input_collection( input_name, input_collection) if implicit_output_name: dataset_collection_instance.implicit_output_name = implicit_output_name log.debug("Created collection with %d elements" % (len(dataset_collection_instance.collection.elements))) if set_hid: parent.add_dataset_collection(dataset_collection_instance) elif isinstance(parent, model.LibraryFolder): dataset_collection_instance = model.LibraryDatasetCollectionAssociation( collection=dataset_collection, folder=parent, name=name, ) else: message = "Internal logic error - create called with unknown parent type %s" % type( parent) log.exception(message) raise MessageException(message) # Tags may be coming in as a dictionary of tag model objects if copying them from other # existing Galaxy objects or as a list of strings if the tags are coming from user supplied # values. if isinstance(tags, list): assert implicit_inputs is None, implicit_inputs tags = self.tag_handler.add_tags_from_list( trans.user, dataset_collection_instance, tags) else: tags = self._append_tags(dataset_collection_instance, implicit_inputs, tags) return self.__persist(dataset_collection_instance, flush=flush)
def __load_element(self, trans, element_identifier, hide_source_items, copy_elements, history=None): # if not isinstance( element_identifier, dict ): # # Is allowing this to just be the id of an hda too clever? Somewhat # # consistent with other API methods though. # element_identifier = dict( src='hda', id=str( element_identifier ) ) # Previously created collection already found in request, just pass # through as is. if "__object__" in element_identifier: the_object = element_identifier["__object__"] if the_object is not None and the_object.id: context = self.model.context if the_object not in context: the_object = context.query(type(the_object)).get(the_object.id) return the_object # dataset_identifier is dict {src=hda|ldda|hdca|new_collection, id=<encoded_id>} try: src_type = element_identifier.get('src', 'hda') except AttributeError: raise MessageException("Dataset collection element definition (%s) not dictionary-like." % element_identifier) encoded_id = element_identifier.get('id', None) if not src_type or not encoded_id: message_template = "Problem decoding element identifier %s - must contain a 'src' and a 'id'." message = message_template % element_identifier raise RequestParameterInvalidException(message) tags = element_identifier.pop('tags', None) tag_str = '' if tags: tag_str = ",".join(str(_) for _ in tags) if src_type == 'hda': decoded_id = int(trans.app.security.decode_id(encoded_id)) hda = self.hda_manager.get_accessible(decoded_id, trans.user) if copy_elements: element = self.hda_manager.copy(hda, history=history or trans.history, hide_copy=True) else: element = hda if hide_source_items and self.hda_manager.get_owned(hda.id, user=trans.user, current_history=history or trans.history): hda.visible = False self.tag_handler.apply_item_tags(user=trans.user, item=element, tags_str=tag_str) elif src_type == 'ldda': element = self.ldda_manager.get(trans, encoded_id, check_accessible=True) element = element.to_history_dataset_association(history or trans.history, add_to_history=True, visible=not hide_source_items) self.tag_handler.apply_item_tags(user=trans.user, item=element, tags_str=tag_str) elif src_type == 'hdca': # TODO: Option to copy? Force copy? Copy or allow if not owned? element = self.__get_history_collection_instance(trans, encoded_id).collection # TODO: ldca. else: raise RequestParameterInvalidException("Unknown src_type parameter supplied '%s'." % src_type) return element
def _get(self, url, params=None): response = requests.get(url, params=params) if response.ok: return response.json() else: code = response.status_code message = response.text try: trs_error_dict = response.json() code = int(trs_error_dict["code"]) message = trs_error_dict["message"] except Exception: pass raise MessageException.from_code(code, message)
def delete_custom_builds(self, trans, id, key, payload={}, **kwd): """ DELETE /api/users/{id}/custom_builds/{key} Delete a custom build. :param id: the encoded id of the user :type id: str :param id: custom build key to be deleted :type id: str """ user = self._get_user(trans, id) dbkeys = json.loads(user.preferences['dbkeys']) if 'dbkeys' in user.preferences else {} if key and key in dbkeys: del dbkeys[key] user.preferences['dbkeys'] = json.dumps(dbkeys) trans.sa_session.flush() return {'message': 'Deleted %s.' % key} else: raise MessageException('Could not find and delete build (%s).' % key)
def _add_entry(self, entry, allow_duplicates=True, persist=False, entry_source=None, **kwd): # accepts dict or list of columns if isinstance(entry, dict): fields = [] for column_name in self.get_column_name_list(): if column_name not in entry: log.debug( "Using default column value for column '%s' when adding data table entry (%s) to table '%s'.", column_name, entry, self.name) field_value = self.get_empty_field_by_name(column_name) else: field_value = entry[column_name] fields.append(field_value) else: fields = entry if self.largest_index < len(fields): fields = self._replace_field_separators(fields) if (allow_duplicates and self.allow_duplicate_entries ) or fields not in self.get_fields(): self.data.append(fields) else: raise MessageException( f"Attempted to add fields ({fields}) to data table '{self.name}', but this entry already exists and allow_duplicates is False." ) else: raise MessageException( f"Attempted to add fields ({fields}) to data table '{self.name}', but there were not enough fields specified ( {len(fields)} < {self.largest_index + 1} )." ) filename = None if persist: filename = self.get_filename_for_source(entry_source) if filename is None: # If we reach this point, there is no data table with a corresponding .loc file. raise MessageException( f"Unable to determine filename for persisting data table '{self.name}' values: '{self.fields}'." ) else: log.debug("Persisting changes to file: %s", filename) with FileLock(filename): try: if os.path.exists(filename): data_table_fh = open(filename, 'r+b') if os.stat(filename).st_size > 0: # ensure last existing line ends with new line data_table_fh.seek(-1, 2) # last char in file last_char = data_table_fh.read(1) if last_char not in [b'\n', b'\r']: data_table_fh.write(b'\n') else: data_table_fh = open(filename, 'wb') except OSError as e: log.exception('Error opening data table file (%s): %s', filename, e) raise fields = f"{self.separator.join(fields)}\n" data_table_fh.write(fields.encode('utf-8'))
def _check_can_convert_to_pdf_or_raise(): """Checks if the HTML to PDF converter is available.""" if not weasyprint: raise MessageException("PDF conversion service not available.")
def add_custom_builds(self, trans, id, key, payload={}, **kwd): """ PUT /api/users/{id}/custom_builds/{key} Add new custom build. :param id: the encoded id of the user :type id: str :param id: custom build key :type id: str :param payload: data with new build details :type payload: dict """ user = self._get_user(trans, id) dbkeys = json.loads(user.preferences['dbkeys'] ) if 'dbkeys' in user.preferences else {} name = payload.get('name') len_type = payload.get('len|type') len_value = payload.get('len|value') if len_type not in ['file', 'fasta', 'text'] or not len_value: raise MessageException('Please specify a valid data source type.') if not name or not key: raise MessageException( 'You must specify values for all the fields.') elif key in dbkeys: raise MessageException( 'There is already a custom build with that key. Delete it first if you want to replace it.' ) else: # Have everything needed; create new build. build_dict = {'name': name} if len_type in ['text', 'file']: # Create new len file new_len = trans.app.model.HistoryDatasetAssociation( extension='len', create_dataset=True, sa_session=trans.sa_session) trans.sa_session.add(new_len) new_len.name = name new_len.visible = False new_len.state = trans.app.model.Job.states.OK new_len.info = 'custom build .len file' try: trans.app.object_store.create(new_len.dataset) except ObjectInvalid: raise MessageException( 'Unable to create output dataset: object store is full.' ) trans.sa_session.flush() counter = 0 lines_skipped = 0 f = open(new_len.file_name, 'w') # LEN files have format: # <chrom_name><tab><chrom_length> for line in len_value.split('\n'): # Splits at the last whitespace in the line lst = line.strip().rsplit(None, 1) if not lst or len(lst) < 2: lines_skipped += 1 continue chrom, length = lst[0], lst[1] try: length = int(length) except ValueError: lines_skipped += 1 continue if chrom != escape(chrom): build_dict[ 'message'] = 'Invalid chromosome(s) with HTML detected and skipped.' lines_skipped += 1 continue counter += 1 f.write('%s\t%s\n' % (chrom, length)) f.close() build_dict['len'] = new_len.id build_dict['count'] = counter else: build_dict['fasta'] = trans.security.decode_id(len_value) dataset = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation).get( build_dict['fasta']) try: new_len = dataset.get_converted_dataset(trans, 'len') new_linecount = new_len.get_converted_dataset( trans, 'linecount') build_dict['len'] = new_len.id build_dict['linecount'] = new_linecount.id except: raise MessageException('Failed to convert dataset.') dbkeys[key] = build_dict user.preferences['dbkeys'] = json.dumps(dbkeys) trans.sa_session.flush() return build_dict
def upload(self, trans, history_id, provider, bucket_name, objects, credentials, input_args=None): """ Implements the logic of uploading a file from a cloud-based storage (e.g., Amazon S3) and persisting it as a Galaxy dataset. :type trans: galaxy.web.framework.webapp.GalaxyWebTransaction :param trans: Galaxy web transaction :type history_id: string :param history_id: the (decoded) id of history to which the object should be uploaded to. :type provider: string :param provider: the name of cloud-based resource provided. A list of supported providers is given in `SUPPORTED_PROVIDERS` variable. :type bucket_name: string :param bucket_name: the name of a bucket from which data should be uploaded (e.g., a bucket name on AWS S3). :type objects: list of string :param objects: the name of objects to be uploaded. :type credentials: dict :param credentials: a dictionary containing all the credentials required to authenticated to the specified provider (e.g., {"secret_key": YOUR_AWS_SECRET_TOKEN, "access_key": YOUR_AWS_ACCESS_TOKEN}). :type input_args: dict :param input_args: a [Optional] a dictionary of input parameters: dbkey, file_type, space_to_tab, to_posix_lines (see galaxy/webapps/galaxy/api/cloud.py) :rtype: list of galaxy.model.Dataset :return: a list of datasets created for the uploaded files. """ if CloudProviderFactory is None: raise Exception(NO_CLOUDBRIDGE_ERROR_MESSAGE) if input_args is None: input_args = {} connection = self._configure_provider(provider, credentials) try: bucket = connection.storage.buckets.get(bucket_name) if bucket is None: raise RequestParameterInvalidException( "The bucket `{}` not found.".format(bucket_name)) except Exception as e: raise ItemAccessibilityException( "Could not get the bucket `{}`: {}".format( bucket_name, str(e))) datasets = [] for obj in objects: try: key = bucket.objects.get(obj) except Exception as e: raise MessageException( "The following error occurred while getting the object {}: {}" .format(obj, str(e))) if key is None: raise ObjectNotFound( "Could not get the object `{}`.".format(obj)) params = Params(self._get_inputs(obj, key, input_args), sanitize=False) incoming = params.__dict__ history = trans.sa_session.query( trans.app.model.History).get(history_id) if not history: raise ObjectNotFound("History with ID `{}` not found.".format( trans.app.security.encode_id(history_id))) output = trans.app.toolbox.get_tool('upload1').handle_input( trans, incoming, history=history) job_errors = output.get('job_errors', []) if job_errors: raise ValueError( 'Following error occurred while uploading the given object(s) from {}: {}' .format(provider, job_errors)) else: for d in output['out_data']: datasets.append(d[1].dataset) return datasets
def set_information(self, trans, id, payload={}, **kwd): """ POST /api/users/{id}/information Save a user's email, username, addresses etc. :param id: the encoded id of the user :type id: str :param payload: data with new settings :type payload: dict """ user = self._get_user(trans, id) email = payload.get('email') username = payload.get('username') if email or username: message = self._validate_email_publicname( email, username) or validate_email(trans, email, user) if not message and username: message = validate_publicname(trans, username, user) if message: raise MessageException(message) if user.email != email: # Update user email and user's private role name which must match private_role = trans.app.security_agent.get_private_user_role( user) private_role.name = email private_role.description = 'Private role for ' + email user.email = email trans.sa_session.add(user) trans.sa_session.add(private_role) trans.sa_session.flush() if trans.app.config.user_activation_on: # Deactivate the user if email was changed and activation is on. user.active = False if self.send_verification_email(trans, user.email, user.username): message = 'The login information has been updated with the changes.<br>Verification email has been sent to your new email address. Please verify it by clicking the activation link in the email.<br>Please check your spam/trash folder in case you cannot find the message.' else: message = 'Unable to send activation email, please contact your local Galaxy administrator.' if trans.app.config.error_email_to is not None: message += ' Contact: %s' % trans.app.config.error_email_to raise MessageException(message) if user.username != username: # Update public name user.username = username # Update user custom form user_info_form_id = payload.get('info|form_id') if user_info_form_id: prefix = 'info|' user_info_form = trans.sa_session.query( trans.app.model.FormDefinition).get( trans.security.decode_id(user_info_form_id)) user_info_values = {} for item in payload: if item.startswith(prefix): user_info_values[item[len(prefix):]] = payload[item] form_values = trans.model.FormValues(user_info_form, user_info_values) trans.sa_session.add(form_values) user.values = form_values # Update user addresses address_dicts = {} address_count = 0 for item in payload: match = re.match(r'^address_(?P<index>\d+)\|(?P<attribute>\S+)', item) if match: groups = match.groupdict() index = int(groups['index']) attribute = groups['attribute'] address_dicts[index] = address_dicts.get(index) or {} address_dicts[index][attribute] = payload[item] address_count = max(address_count, index + 1) user.addresses = [] for index in range(0, address_count): d = address_dicts[index] if d.get('id'): try: user_address = trans.sa_session.query( trans.app.model.UserAddress).get( trans.security.decode_id(d['id'])) except Exception as e: raise MessageException( 'Failed to access user address (%s). %s' % (d['id'], e)) else: user_address = trans.model.UserAddress() trans.log_event('User address added') for field in AddressField.fields(): if str(field[2]).lower() == 'required' and not d.get(field[0]): raise MessageException('Address %s: %s (%s) required.' % (index + 1, field[1], field[0])) setattr(user_address, field[0], str(d.get(field[0], ''))) user_address.user = user user.addresses.append(user_address) trans.sa_session.add(user_address) trans.sa_session.add(user) trans.sa_session.flush() trans.log_event('User information added') return {'message': 'User information has been saved.'}
def create(self, trans, parent, name, collection_type, element_identifiers=None, elements=None, implicit_collection_info=None, trusted_identifiers=None, hide_source_items=False, tags=None): """ PRECONDITION: security checks on ability to add to parent occurred during load. """ # Trust embedded, newly created objects created by tool subsystem. if trusted_identifiers is None: trusted_identifiers = implicit_collection_info is not None if element_identifiers and not trusted_identifiers: validate_input_element_identifiers(element_identifiers) dataset_collection = self.create_dataset_collection( trans=trans, collection_type=collection_type, element_identifiers=element_identifiers, elements=elements, hide_source_items=hide_source_items, ) if isinstance(parent, model.History): dataset_collection_instance = self.model.HistoryDatasetCollectionAssociation( collection=dataset_collection, name=name, ) if implicit_collection_info: for input_name, input_collection in implicit_collection_info[ "implicit_inputs"]: dataset_collection_instance.add_implicit_input_collection( input_name, input_collection) for output_dataset in implicit_collection_info.get("outputs"): if output_dataset not in trans.sa_session: output_dataset = trans.sa_session.query( type(output_dataset)).get(output_dataset.id) if isinstance(output_dataset, model.HistoryDatasetAssociation): output_dataset.hidden_beneath_collection_instance = dataset_collection_instance elif isinstance(output_dataset, model.HistoryDatasetCollectionAssociation): dataset_collection_instance.add_implicit_input_collection( input_name, input_collection) else: # dataset collection, don't need to do anything... pass trans.sa_session.add(output_dataset) dataset_collection_instance.implicit_output_name = implicit_collection_info[ "implicit_output_name"] log.debug("Created collection with %d elements" % (len(dataset_collection_instance.collection.elements))) # Handle setting hid parent.add_dataset_collection(dataset_collection_instance) elif isinstance(parent, model.LibraryFolder): dataset_collection_instance = self.model.LibraryDatasetCollectionAssociation( collection=dataset_collection, folder=parent, name=name, ) else: message = "Internal logic error - create called with unknown parent type %s" % type( parent) log.exception(message) raise MessageException(message) tags = tags or {} if implicit_collection_info: for k, v in implicit_collection_info.get('implicit_inputs', []): for tag in [t for t in v.tags if t.user_tname == 'name']: tags[tag.value] = tag for _, tag in tags.items(): dataset_collection_instance.tags.append(tag.copy()) return self.__persist(dataset_collection_instance)
def error(message): raise MessageException(message, type='error')
def send(self, trans, history_id, bucket_name, authz_id, dataset_ids=None, overwrite_existing=False): """ Implements the logic of sending dataset(s) from a given history to a given cloud-based storage (e.g., Amazon S3). :type trans: galaxy.webapps.base.webapp.GalaxyWebTransaction :param trans: Galaxy web transaction :type history_id: string :param history_id: the (encoded) id of history from which the object should be sent. :type bucket_name: string :param bucket_name: the name of a bucket to which data should be sent (e.g., a bucket name on AWS S3). :type authz_id: int :param authz_id: the ID of CloudAuthz to be used for authorizing access to the resource provider. You may get a list of the defined authorizations via `/api/cloud/authz`. Also, you can use `/api/cloud/authz/create` to define a new authorization. :type dataset_ids: set :param dataset_ids: [Optional] The list of (decoded) dataset ID(s) belonging to the given history which should be sent to the given provider. If not provided, Galaxy sends all the datasets belonging to the given history. :type overwrite_existing: boolean :param overwrite_existing: [Optional] If set to "True", and an object with same name of the dataset to be sent already exist in the bucket, Galaxy replaces the existing object with the dataset to be sent. If set to "False", Galaxy appends datetime to the dataset name to prevent overwriting the existing object. :rtype: tuple :return: A tuple of two lists of labels of the objects that were successfully and unsuccessfully sent to cloud. """ if CloudProviderFactory is None: raise Exception(NO_CLOUDBRIDGE_ERROR_MESSAGE) if not hasattr(trans.app, 'authnz_manager'): err_msg = "The OpenID Connect protocol, a required feature for sending data to cloud, " \ "is not enabled on this Galaxy instance." log.debug(err_msg) raise MessageException(err_msg) cloudauthz = trans.app.authnz_manager.try_get_authz_config( trans.sa_session, trans.user.id, authz_id) history = trans.sa_session.query( trans.app.model.History).get(history_id) if not history: raise ObjectNotFound("History with ID `{}` not found.".format( trans.app.security.encode_id(history_id))) sent = [] failed = [] for hda in history.datasets: if hda.deleted or hda.purged or hda.state != "ok" or hda.creating_job.tool_id == SEND_TOOL: continue if dataset_ids is None or hda.dataset.id in dataset_ids: try: object_label = hda.name.replace(" ", "_") args = { # We encode ID here because the tool wrapper expects # an encoded ID and attempts decoding it. "authz_id": trans.security.encode_id(cloudauthz.id), "bucket": bucket_name, "object_label": object_label, "filename": hda, "overwrite_existing": overwrite_existing } incoming = (util.Params(args, sanitize=False)).__dict__ d2c = trans.app.toolbox.get_tool(SEND_TOOL, SEND_TOOL_VERSION) if not d2c: log.debug( f"Failed to get the `send` tool per user `{trans.user.id}` request." ) failed.append( json.dumps({ "object": object_label, "error": "Unable to get the `send` tool." })) continue res = d2c.execute(trans, incoming, history=history) job = res[0] sent.append( json.dumps({ "object": object_label, "job_id": trans.app.security.encode_id(job.id) })) except Exception as e: err_msg = "maybe invalid or unauthorized credentials. {}".format( util.unicodify(e)) log.debug( "Failed to send the dataset `{}` per user `{}` request to cloud, {}" .format(object_label, trans.user.id, err_msg)) failed.append( json.dumps({ "object": object_label, "error": err_msg })) return sent, failed
def get(self, trans, history_id, bucket_name, objects, authz_id, input_args=None): """ Implements the logic of getting a file from a cloud-based storage (e.g., Amazon S3) and persisting it as a Galaxy dataset. This manager does NOT require use credentials, instead, it uses a more secure method, which leverages CloudAuthz (https://github.com/galaxyproject/cloudauthz) and automatically requests temporary credentials to access the defined resources. :type trans: galaxy.webapps.base.webapp.GalaxyWebTransaction :param trans: Galaxy web transaction :type history_id: string :param history_id: the (decoded) id of history to which the object should be received to. :type bucket_name: string :param bucket_name: the name of a bucket from which data should be fetched (e.g., a bucket name on AWS S3). :type objects: list of string :param objects: the name of objects to be fetched. :type authz_id: int :param authz_id: the ID of CloudAuthz to be used for authorizing access to the resource provider. You may get a list of the defined authorizations sending GET to `/api/cloud/authz`. Also, you can POST to `/api/cloud/authz` to define a new authorization. :type input_args: dict :param input_args: a [Optional] a dictionary of input parameters: dbkey, file_type, space_to_tab, to_posix_lines (see galaxy/webapps/galaxy/api/cloud.py) :rtype: list of galaxy.model.Dataset :return: a list of datasets created for the fetched files. """ if CloudProviderFactory is None: raise Exception(NO_CLOUDBRIDGE_ERROR_MESSAGE) if input_args is None: input_args = {} if not hasattr(trans.app, 'authnz_manager'): err_msg = "The OpenID Connect protocol, a required feature for getting data from cloud, " \ "is not enabled on this Galaxy instance." log.debug(err_msg) raise MessageException(err_msg) cloudauthz = trans.app.authnz_manager.try_get_authz_config( trans.sa_session, trans.user.id, authz_id) credentials = trans.app.authnz_manager.get_cloud_access_credentials( cloudauthz, trans.sa_session, trans.user.id, trans.request) connection = self.configure_provider(cloudauthz.provider, credentials) try: bucket = connection.storage.buckets.get(bucket_name) if bucket is None: raise RequestParameterInvalidException( f"The bucket `{bucket_name}` not found.") except Exception as e: raise ItemAccessibilityException( "Could not get the bucket `{}`: {}".format( bucket_name, util.unicodify(e))) datasets = [] for obj in objects: try: key = bucket.objects.get(obj) except Exception as e: raise MessageException( "The following error occurred while getting the object {}: {}" .format(obj, util.unicodify(e))) if key is None: log.exception( "Could not get object `{}` for user `{}`. Object may not exist, or the provided credentials are " "invalid or not authorized to read the bucket/object.". format(obj, trans.user.id)) raise ObjectNotFound( "Could not get the object `{}`. Please check if the object exists, and credentials are valid and " "authorized to read the bucket and object. ".format(obj)) params = Params(self._get_inputs(obj, key, input_args), sanitize=False) incoming = params.__dict__ history = trans.sa_session.query( trans.app.model.History).get(history_id) if not history: raise ObjectNotFound("History with ID `{}` not found.".format( trans.app.security.encode_id(history_id))) output = trans.app.toolbox.get_tool('upload1').handle_input( trans, incoming, history=history) job_errors = output.get('job_errors', []) if job_errors: raise ValueError( 'Following error occurred while getting the given object(s) from {}: {}' .format(cloudauthz.provider, job_errors)) else: for d in output['out_data']: datasets.append(d[1].dataset) return datasets
def create( self, trans, parent, # PRECONDITION: security checks on ability to add to parent # occurred during load. name, collection_type, element_identifiers=None, elements=None, implicit_collection_info=None, trusted_identifiers=None, # Trust preloaded element objects ): """ """ # Trust embedded, newly created objects created by tool subsystem. if trusted_identifiers is None: trusted_identifiers = implicit_collection_info is not None if element_identifiers and not trusted_identifiers: validate_input_element_identifiers(element_identifiers) dataset_collection = self.__create_dataset_collection( trans=trans, collection_type=collection_type, element_identifiers=element_identifiers, elements=elements, ) if isinstance(parent, model.History): dataset_collection_instance = self.model.HistoryDatasetCollectionAssociation( collection=dataset_collection, name=name, ) if implicit_collection_info: for input_name, input_collection in implicit_collection_info[ "implicit_inputs"]: dataset_collection_instance.add_implicit_input_collection( input_name, input_collection) for output_dataset in implicit_collection_info.get("outputs"): if isinstance(output_dataset, model.HistoryDatasetAssociation): output_dataset.hidden_beneath_collection_instance = dataset_collection_instance elif isinstance(output_dataset, model.HistoryDatasetCollectionAssociation): dataset_collection_instance.add_implicit_input_collection( input_name, input_collection) else: # dataset collection, don't need to do anything... pass trans.sa_session.add(output_dataset) dataset_collection_instance.implicit_output_name = implicit_collection_info[ "implicit_output_name"] log.debug("Created collection with %d elements" % (len(dataset_collection_instance.collection.elements))) # Handle setting hid parent.add_dataset_collection(dataset_collection_instance) elif isinstance(parent, model.LibraryFolder): dataset_collection_instance = self.model.LibraryDatasetCollectionAssociation( collection=dataset_collection, folder=parent, name=name, ) else: message = "Internal logic error - create called with unknown parent type %s" % type( parent) log.exception(message) raise MessageException(message) return self.__persist(dataset_collection_instance)