def list_commands_handler_get_req(artifact_types): """Retrieves the commands that can process the given artifact types Parameters ---------- artifact_types : str Comma-separated list of artifact types Returns ------- dict of objects A dictionary containing the commands information {'status': str, 'message': str, 'commands': list of dicts of {'id': int, 'command': str, 'output': list of [str, str]}} """ artifact_types = artifact_types.split(',') cmd_info = [ {'id': cmd.id, 'command': cmd.name, 'output': cmd.outputs} for cmd in Command.get_commands_by_input_type(artifact_types)] return {'status': 'success', 'message': '', 'commands': cmd_info}
def list_commands_handler_get_req(artifact_types): """Retrieves the commands that can process the given artifact types Parameters ---------- artifact_types : str Comma-separated list of artifact types Returns ------- dict of objects A dictionary containing the commands information {'status': str, 'message': str, 'commands': list of dicts of {'id': int, 'command': str, 'output': list of [str, str]}} """ artifact_types = artifact_types.split(',') cmd_info = [{ 'id': cmd.id, 'command': cmd.name, 'output': cmd.outputs } for cmd in Command.get_commands_by_input_type(artifact_types)] return {'status': 'success', 'message': '', 'commands': cmd_info}
def list_options_handler_get_req(command_id): """Returns the available default parameters set for the given command Parameters ---------- command_id : int The command id Returns ------- dict of objects A dictionary containing the commands information {'status': str, 'message': str, 'options': list of dicts of {'id: str', 'name': str, 'values': dict of {str: str}}} """ command = Command(command_id) options = [{ 'id': p.id, 'name': p.name, 'values': p.values } for p in command.default_parameter_sets] return { 'status': 'success', 'message': '', 'options': options, 'req_options': command.required_parameters }
def workflow_handler_post_req(user_id, command_id, params): """Creates a new workflow in the system Parameters ---------- user_id : str The user creating the workflow command_id : int The first command to execute in the workflow params : str JSON representations of the parameters for the first command of the workflow Returns ------- dict of objects A dictionary containing the commands information {'status': str, 'message': str, 'workflow_id': int} """ parameters = Parameters.load(Command(command_id), json_str=params) status = 'success' message = '' try: wf = ProcessingWorkflow.from_scratch(User(user_id), parameters) except Exception as exc: wf = None wf_id = None job_info = None status = 'error' message = str(exc) if wf is not None: # this is safe as we are creating the workflow for the first time # and there is only one node. Remember networkx doesn't assure order # of nodes job = list(wf.graph.nodes())[0] inputs = [a.id for a in job.input_artifacts] job_cmd = job.command wf_id = wf.id job_info = { 'id': job.id, 'inputs': inputs, 'label': job_cmd.name, 'outputs': job_cmd.outputs } return { 'status': status, 'message': message, 'workflow_id': wf_id, 'job': job_info }
def list_options_handler_get_req(command_id, artifact_id=None): """Returns the available default parameters set for the given command Parameters ---------- command_id : int The command id artifact_id : int, optional The artifact id so to limit options based on how it has already been processed Returns ------- dict of objects A dictionary containing the commands information {'status': str, 'message': str, 'options': list of dicts of {'id: str', 'name': str, 'values': dict of {str: str}}} """ def _helper_process_params(params): return dumps({k: str(v).lower() for k, v in params.items()}, sort_keys=True) command = Command(command_id) rparamers = command.required_parameters.keys() eparams = [] if artifact_id is not None: artifact = Artifact(artifact_id) for job in artifact.jobs(cmd=command): jstatus = job.status outputs = job.outputs if job.status == 'success' else None # this ignore any jobs that weren't successful or are in # construction, or the results have been deleted [outputs == {}] if jstatus not in {'success', 'in_construction'} or outputs == {}: continue params = job.parameters.values.copy() for k in rparamers: del params[k] eparams.append(_helper_process_params(params)) options = [{ 'id': p.id, 'name': p.name, 'values': p.values } for p in command.default_parameter_sets if _helper_process_params(p.values) not in eparams] return { 'status': 'success', 'message': '', 'options': options, 'req_options': command.required_parameters, 'opt_options': command.optional_parameters }
def test_workflow_handler_patch_req(self): # Create a new workflow so it is in construction exp_command = Command(1) json_str = ( '{"input_data": 1, "max_barcode_errors": 1.5, ' '"barcode_type": "golay_12", "max_bad_run_length": 3, ' '"rev_comp": false, "phred_quality_threshold": 3, ' '"rev_comp_barcode": false, "rev_comp_mapping_barcodes": false, ' '"min_per_read_length_fraction": 0.75, "sequence_max_n": 0}') exp_params = Parameters.load(exp_command, json_str=json_str) exp_user = User('*****@*****.**') name = "Test processing workflow" # tests success wf = ProcessingWorkflow.from_scratch(exp_user, exp_params, name=name, force=True) graph = wf.graph nodes = list(graph.nodes()) job_id = nodes[0].id value = { 'dflt_params': 10, 'connections': { job_id: { 'demultiplexed': 'input_data' } } } obs = workflow_handler_patch_req('add', '/%s/' % wf.id, req_value=dumps(value)) new_jobs = set(wf.graph.nodes()) - set(nodes) self.assertEqual(len(new_jobs), 1) new_job = new_jobs.pop() exp = { 'status': 'success', 'message': '', 'job': { 'id': new_job.id, 'inputs': [job_id], 'label': 'Pick closed-reference OTUs', 'outputs': [['OTU table', 'BIOM']] } } self.assertEqual(obs, exp) obs = workflow_handler_patch_req('remove', '/%s/%s/' % (wf.id, new_job.id)) exp = {'status': 'success', 'message': ''} jobs = set(wf.graph.nodes()) - set(nodes) self.assertEqual(jobs, set())
def test_artifact_summary_post_request(self): # No access with self.assertRaises(QiitaHTTPError): artifact_summary_post_request(User('*****@*****.**'), 1) # Returns already existing job job = ProcessingJob.create( User('*****@*****.**'), Parameters.load(Command(7), values_dict={'input_data': 2})) job._set_status('queued') obs = artifact_summary_post_request(User('*****@*****.**'), 2) exp = {'job': [job.id, 'queued', None]} self.assertEqual(obs, exp)
def test_patch(self): # Create a new job - through a workflow since that is the only way # of creating jobs in the interface exp_command = Command(1) json_str = ( '{"input_data": 1, "max_barcode_errors": 1.5, ' '"barcode_type": "golay_12", "max_bad_run_length": 3, ' '"rev_comp": false, "phred_quality_threshold": 3, ' '"rev_comp_barcode": false, "rev_comp_mapping_barcodes": false, ' '"min_per_read_length_fraction": 0.75, "sequence_max_n": 0}') exp_params = Parameters.load(exp_command, json_str=json_str) exp_user = User('*****@*****.**') name = "Test processing workflow" # tests success wf = ProcessingWorkflow.from_scratch(exp_user, exp_params, name=name, force=True) graph = wf.graph nodes = graph.nodes() job_id = nodes[0].id response = self.patch('/study/process/job/', { 'op': 'remove', 'path': job_id }) self.assertEqual(response.code, 200) exp = { 'status': 'error', 'message': "Can't delete job %s. It is 'in_construction' " "status. Please use /study/process/workflow/" % job_id } self.assertEqual(loads(response.body), exp) # Test success ProcessingJob(job_id)._set_error('Killed for testing') response = self.patch('/study/process/job/', { 'op': 'remove', 'path': job_id }) self.assertEqual(response.code, 200) exp = {'status': 'success', 'message': ''} self.assertEqual(loads(response.body), exp)
def artifact_summary_post_request(user_id, artifact_id): """Launches the HTML summary generation and returns the job information Parameters ---------- user_id : str The user making the request artifact_id : int or str The artifact id Returns ------- dict of objects A dictionary containing the artifact summary information {'status': str, 'message': str, 'job': list of [str, str, str]} """ artifact_id = int(artifact_id) artifact = Artifact(artifact_id) access_error = check_access(artifact.study.id, user_id) if access_error: return access_error # Check if the summary is being generated or has been already generated command = Command.get_html_generator(artifact.artifact_type) jobs = artifact.jobs(cmd=command) jobs = [j for j in jobs if j.status in ['queued', 'running', 'success']] if jobs: # The HTML summary is either being generated or already generated. # Return the information of that job so we only generate the HTML # once job = jobs[0] else: # Create a new job to generate the HTML summary and return the newly # created job information job = ProcessingJob.create( User(user_id), Parameters.load(command, values_dict={'input_data': artifact_id})) job.submit() return { 'status': 'success', 'message': '', 'job': [job.id, job.status, job.step] }
def list_commands_handler_get_req(id, exclude_analysis): """Retrieves the commands that can process the given artifact types Parameters ---------- id : string id, it can be the integer or the name of the artifact:network-root exclude_analysis : bool If True, return commands that are not part of the analysis pipeline Returns ------- dict of objects A dictionary containing the commands information {'status': str, 'message': str, 'commands': list of dicts of {'id': int, 'command': str, 'output': list of [str, str]}} """ if id.isdigit(): commands = Artifact(id).get_commands else: pieces = id.split(':') if len(pieces) == 1: aid = pieces[0] root = '' else: aid = pieces[0] root = pieces[1] prep_type = None if root.isdigit(): artifact = Artifact(root) if artifact.analysis is None: prep_type = artifact.prep_templates[0].data_type commands = Command.get_commands_by_input_type( [aid], exclude_analysis=exclude_analysis, prep_type=prep_type) cmd_info = [{ 'id': cmd.id, 'command': cmd.name, 'output': cmd.outputs } for cmd in commands] return {'status': 'success', 'message': '', 'commands': cmd_info}
def artifact_summary_post_request(user_id, artifact_id): """Launches the HTML summary generation and returns the job information Parameters ---------- user_id : str The user making the request artifact_id : int or str The artifact id Returns ------- dict of objects A dictionary containing the artifact summary information {'status': str, 'message': str, 'job': list of [str, str, str]} """ artifact_id = int(artifact_id) artifact = Artifact(artifact_id) access_error = check_access(artifact.study.id, user_id) if access_error: return access_error # Check if the summary is being generated or has been already generated command = Command.get_html_generator(artifact.artifact_type) jobs = artifact.jobs(cmd=command) jobs = [j for j in jobs if j.status in ['queued', 'running', 'success']] if jobs: # The HTML summary is either being generated or already generated. # Return the information of that job so we only generate the HTML # once job = jobs[0] else: # Create a new job to generate the HTML summary and return the newly # created job information job = ProcessingJob.create( User(user_id), Parameters.load(command, values_dict={'input_data': artifact_id})) job.submit() return {'status': 'success', 'message': '', 'job': [job.id, job.status, job.step]}
def artifact_summary_post_request(user, artifact_id): """Launches the HTML summary generation and returns the job information Parameters ---------- user : qiita_db.user.User The user making the request artifact_id : int or str The artifact id Returns ------- dict of objects A dictionary containing the job summary information {'job': [str, str, str]} """ artifact_id = int(artifact_id) artifact = Artifact(artifact_id) check_artifact_access(user, artifact) # Check if the summary is being generated or has been already generated command = Command.get_html_generator(artifact.artifact_type) jobs = artifact.jobs(cmd=command) jobs = [j for j in jobs if j.status in ['queued', 'running', 'success']] if jobs: # The HTML summary is either being generated or already generated. # Return the information of that job so we only generate the HTML # once - Magic number 0 -> we are ensuring that there is only one # job generating the summary, so we can use the index 0 to access to # that job job = jobs[0] else: # Create a new job to generate the HTML summary and return the newly # created job information job = ProcessingJob.create( user, Parameters.load(command, values_dict={'input_data': artifact_id}), True) job.submit() return {'job': [job.id, job.status, job.step]}
def test_artifact_summary_post_request(self): # No access obs = artifact_summary_post_request('*****@*****.**', 1) exp = { 'status': 'error', 'message': 'User does not have access to study' } self.assertEqual(obs, exp) # Returns already existing job job = ProcessingJob.create( User('*****@*****.**'), Parameters.load(Command(7), values_dict={'input_data': 2})) job._set_status('queued') obs = artifact_summary_post_request('*****@*****.**', 2) exp = { 'status': 'success', 'message': '', 'job': [job.id, 'queued', None] } self.assertEqual(obs, exp)
def artifact_summary_post_request(user, artifact_id): """Launches the HTML summary generation and returns the job information Parameters ---------- user : qiita_db.user.User The user making the request artifact_id : int or str The artifact id Returns ------- dict of objects A dictionary containing the job summary information {'job': [str, str, str]} """ artifact_id = int(artifact_id) artifact = Artifact(artifact_id) check_artifact_access(user, artifact) # Check if the summary is being generated or has been already generated command = Command.get_html_generator(artifact.artifact_type) jobs = artifact.jobs(cmd=command) jobs = [j for j in jobs if j.status in ['queued', 'running', 'success']] if jobs: # The HTML summary is either being generated or already generated. # Return the information of that job so we only generate the HTML # once - Magic number 0 -> we are ensuring that there is only one # job generating the summary, so we can use the index 0 to access to # that job job = jobs[0] else: # Create a new job to generate the HTML summary and return the newly # created job information job = ProcessingJob.create(user, Parameters.load( command, values_dict={'input_data': artifact_id}), True) job.submit() return {'job': [job.id, job.status, job.step]}
def workflow_handler_post_req(user_id, command_id, params): """Creates a new workflow in the system Parameters ---------- user_id : str The user creating the workflow command_id : int The first command to execute in the workflow params : str JSON representations of the parameters for the first command of the workflow Returns ------- dict of objects A dictionary containing the commands information {'status': str, 'message': str, 'workflow_id': int} """ parameters = Parameters.load(Command(command_id), json_str=params) wf = ProcessingWorkflow.from_scratch(User(user_id), parameters) # this is safe as we are creating the workflow for the first time and there # is only one node. Remember networkx doesn't assure order of nodes job = wf.graph.nodes()[0] inputs = [a.id for a in job.input_artifacts] job_cmd = job.command return { 'status': 'success', 'message': '', 'workflow_id': wf.id, 'job': { 'id': job.id, 'inputs': inputs, 'label': job_cmd.name, 'outputs': job_cmd.outputs } }
def artifact_post_req(user_id, filepaths, artifact_type, name, prep_template_id, artifact_id=None): """Creates the initial artifact for the prep template Parameters ---------- user_id : str User adding the atrifact filepaths : dict of str Comma-separated list of files to attach to the artifact, keyed by file type artifact_type : str The type of the artifact name : str Name to give the artifact prep_template_id : int or str castable to int Prep template to attach the artifact to artifact_id : int or str castable to int, optional The id of the imported artifact Returns ------- dict of objects A dictionary containing the new artifact ID {'status': status, 'message': message, 'artifact': id} """ prep_template_id = int(prep_template_id) prep = PrepTemplate(prep_template_id) study_id = prep.study_id # First check if the user has access to the study access_error = check_access(study_id, user_id) if access_error: return access_error user = User(user_id) if artifact_id: # if the artifact id has been provided, import the artifact qiita_plugin = Software.from_name_and_version('Qiita', 'alpha') cmd = qiita_plugin.get_command('copy_artifact') params = Parameters.load(cmd, values_dict={ 'artifact': artifact_id, 'prep_template': prep.id }) job = ProcessingJob.create(user, params, True) else: uploads_path = get_mountpoint('uploads')[0][1] path_builder = partial(join, uploads_path, str(study_id)) cleaned_filepaths = {} for ftype, file_list in filepaths.items(): # JavaScript sends us this list as a comma-separated list for fp in file_list.split(','): # JavaScript will send this value as an empty string if the # list of files was empty. In such case, the split will # generate a single element containing the empty string. Check # for that case here and, if fp is not the empty string, # proceed to check if the file exists if fp: # Check if filepath being passed exists for study full_fp = path_builder(fp) exists = check_fp(study_id, full_fp) if exists['status'] != 'success': return { 'status': 'error', 'message': 'File does not exist: %s' % fp } if ftype not in cleaned_filepaths: cleaned_filepaths[ftype] = [] cleaned_filepaths[ftype].append(full_fp) # This should never happen, but it doesn't hurt to actually have # a explicit check, in case there is something odd with the JS if not cleaned_filepaths: return { 'status': 'error', 'message': "Can't create artifact, no files provided." } # This try/except will catch the case when the plugins are not # activated so there is no Validate for the given artifact_type try: command = Command.get_validator(artifact_type) except QiitaDBError as e: return {'status': 'error', 'message': str(e)} job = ProcessingJob.create( user, Parameters.load(command, values_dict={ 'template': prep_template_id, 'files': dumps(cleaned_filepaths), 'artifact_type': artifact_type, 'name': name, 'analysis': None, }), True) # Submit the job job.submit() r_client.set(PREP_TEMPLATE_KEY_FORMAT % prep.id, dumps({ 'job_id': job.id, 'is_qiita_job': True })) return {'status': 'success', 'message': ''}
def artifact_summary_get_request(user, artifact_id): """Returns the information for the artifact summary page Parameters ---------- user : qiita_db.user.User The user making the request artifact_id : int or str The artifact id Returns ------- dict of objects A dictionary containing the artifact summary information {'name': str, 'artifact_id': int, 'visibility': str, 'editable': bool, 'buttons': str, 'processing_parameters': dict of {str: object}, 'files': list of (int, str), 'is_from_analysis': bool, 'summary': str or None, 'job': [str, str, str], 'errored_jobs': list of [str, str]} """ artifact_id = int(artifact_id) artifact = Artifact(artifact_id) artifact_type = artifact.artifact_type check_artifact_access(user, artifact) visibility = artifact.visibility summary = artifact.html_summary_fp job_info = None errored_summary_jobs = [] # Check if the HTML summary exists if summary: # Magic number 1: If the artifact has a summary, the call # artifact.html_summary_fp returns a tuple with 2 elements. The first # element is the filepath id, while the second one is the actual # actual filepath. We are only interested on the actual filepath, # hence the 1 value. summary = relpath(summary[1], qiita_config.base_data_dir) else: # Check if the summary is being generated command = Command.get_html_generator(artifact_type) all_jobs = set(artifact.jobs(cmd=command)) jobs = [] errored_summary_jobs = [] for j in all_jobs: if j.status in ['queued', 'running']: jobs.append(j) elif j.status in ['error']: errored_summary_jobs.append(j) if jobs: # There is already a job generating the HTML. Also, there should be # at most one job, because we are not allowing here to start more # than one job = jobs[0] job_info = [job.id, job.status, job.step] # Check if the artifact is editable by the given user study = artifact.study analysis = artifact.analysis editable = study.can_edit(user) if study else analysis.can_edit(user) buttons = [] btn_base = ( '<button onclick="if (confirm(\'Are you sure you want to %s ' 'artifact id: {0}?\')) {{ set_artifact_visibility(\'%s\', {0}) }}" ' 'class="btn btn-primary btn-sm">%s</button>').format(artifact_id) if not analysis: # If the artifact is part of a study, the buttons shown depend in # multiple factors (see each if statement for an explanation of those) if qiita_config.require_approval: if visibility == 'sandbox' and artifact.parents: # The request approval button only appears if the artifact is # sandboxed and the qiita_config specifies that the approval # should be requested buttons.append( btn_base % ('request approval for', 'awaiting_approval', 'Request approval')) elif user.level == 'admin' and visibility == 'awaiting_approval': # The approve artifact button only appears if the user is an # admin the artifact is waiting to be approvaed and the qiita # config requires artifact approval buttons.append(btn_base % ('approve', 'private', 'Approve artifact')) if visibility == 'private': # The make public button only appears if the artifact is private buttons.append(btn_base % ('make public', 'public', 'Make public')) # The revert to sandbox button only appears if the artifact is not # sandboxed nor public if visibility not in {'sandbox', 'public'}: buttons.append(btn_base % ('revert to sandbox', 'sandbox', 'Revert to sandbox')) if user.level == 'admin': if artifact.can_be_submitted_to_ebi: buttons.append( '<a class="btn btn-primary btn-sm" ' 'href="/ebi_submission/%d">' '<span class="glyphicon glyphicon-export"></span>' ' Submit to EBI</a>' % artifact_id) if artifact.can_be_submitted_to_vamps: if not artifact.is_submitted_to_vamps: buttons.append( '<a class="btn btn-primary btn-sm" href="/vamps/%d">' '<span class="glyphicon glyphicon-export"></span>' ' Submit to VAMPS</a>' % artifact_id) files = [(x['fp_id'], "%s (%s)" % (basename(x['fp']), x['fp_type'].replace('_', ' ')), x['checksum'], naturalsize(x['fp_size'])) for x in artifact.filepaths if x['fp_type'] != 'directory'] # TODO: https://github.com/biocore/qiita/issues/1724 Remove this hardcoded # values to actually get the information from the database once it stores # the information if artifact_type in ['SFF', 'FASTQ', 'FASTA', 'FASTA_Sanger', 'per_sample_FASTQ']: # If the artifact is one of the "raw" types, only the owner of the # study and users that has been shared with can see the files study = artifact.study has_access = study.has_access(user, no_public=True) if (not study.public_raw_download and not has_access): files = [] proc_params = artifact.processing_parameters if proc_params: cmd = proc_params.command sw = cmd.software processing_info = { 'command': cmd.name, 'software': sw.name, 'software_version': sw.version, 'processing_parameters': proc_params.values, 'command_active': cmd.active, 'software_deprecated': sw.deprecated, } else: processing_info = {} return {'name': artifact.name, 'artifact_id': artifact_id, 'artifact_type': artifact_type, 'visibility': visibility, 'editable': editable, 'buttons': ' '.join(buttons), 'processing_info': processing_info, 'files': files, 'is_from_analysis': artifact.analysis is not None, 'summary': summary, 'job': job_info, 'artifact_timestamp': artifact.timestamp.strftime( "%Y-%m-%d %H:%m"), 'errored_summary_jobs': errored_summary_jobs}
def artifact_summary_get_request(user_id, artifact_id): """Returns the information for the artifact summary page Parameters ---------- user_id : str The user making the request artifact_id : int or str The artifact id Returns ------- dict of objects A dictionary containing the artifact summary information {'status': str, 'message': str, 'name': str, 'summary': str, 'job': list of [str, str, str]} """ artifact_id = int(artifact_id) artifact = Artifact(artifact_id) access_error = check_access(artifact.study.id, user_id) if access_error: return access_error user = User(user_id) visibility = artifact.visibility summary = artifact.html_summary_fp job_info = None errored_jobs = [] processing_jobs = [] for j in artifact.jobs(): if j.command.software.type == "artifact transformation": status = j.status if status == 'success': continue j_msg = j.log.msg if status == 'error' else None processing_jobs.append( [j.id, j.command.name, j.status, j.step, j_msg]) # Check if the HTML summary exists if summary: with open(summary[1]) as f: summary = f.read() else: # Check if the summary is being generated command = Command.get_html_generator(artifact.artifact_type) all_jobs = set(artifact.jobs(cmd=command)) jobs = [j for j in all_jobs if j.status in ['queued', 'running']] errored_jobs = [(j.id, j.log.msg) for j in all_jobs if j.status in ['error']] if jobs: # There is already a job generating the HTML. Also, there should be # at most one job, because we are not allowing here to start more # than one job = jobs[0] job_info = [job.id, job.status, job.step] buttons = [] btn_base = ( '<button onclick="if (confirm(\'Are you sure you want to %s ' 'artifact id: {0}?\')) {{ set_artifact_visibility(\'%s\', {0}) }}" ' 'class="btn btn-primary btn-sm">%s</button>').format(artifact_id) if qiita_config.require_approval: if visibility == 'sandbox': # The request approval button only appears if the artifact is # sandboxed and the qiita_config specifies that the approval should # be requested buttons.append( btn_base % ('request approval for', 'awaiting_approval', 'Request approval')) elif user.level == 'admin' and visibility == 'awaiting_approval': # The approve artifact button only appears if the user is an admin # the artifact is waiting to be approvaed and the qiita config # requires artifact approval buttons.append(btn_base % ('approve', 'private', 'Approve artifact')) if visibility == 'private': # The make public button only appears if the artifact is private buttons.append(btn_base % ('make public', 'public', 'Make public')) # The revert to sandbox button only appears if the artifact is not # sandboxed nor public if visibility not in {'sandbox', 'public'}: buttons.append(btn_base % ('revert to sandbox', 'sandbox', 'Revert to sandbox')) if artifact.can_be_submitted_to_ebi: if not artifact.is_submitted_to_ebi: buttons.append( '<a class="btn btn-primary btn-sm" ' 'href="/ebi_submission/%d">' '<span class="glyphicon glyphicon-export"></span>' ' Submit to EBI</a>' % artifact_id) if artifact.can_be_submitted_to_vamps: if not artifact.is_submitted_to_vamps: buttons.append( '<a class="btn btn-primary btn-sm" href="/vamps/%d">' '<span class="glyphicon glyphicon-export"></span>' ' Submit to VAMPS</a>' % artifact_id) files = [(f_id, "%s (%s)" % (basename(fp), f_type.replace('_', ' '))) for f_id, fp, f_type in artifact.filepaths if f_type != 'directory'] # TODO: https://github.com/biocore/qiita/issues/1724 Remove this hardcoded # values to actually get the information from the database once it stores # the information if artifact.artifact_type in ['SFF', 'FASTQ', 'FASTA', 'FASTA_Sanger', 'per_sample_FASTQ']: # If the artifact is one of the "raw" types, only the owner of the # study and users that has been shared with can see the files if not artifact.study.has_access(user, no_public=True): files = [] return {'status': 'success', 'message': '', 'name': artifact.name, 'summary': summary, 'job': job_info, 'errored_jobs': errored_jobs, 'processing_jobs': processing_jobs, 'visibility': visibility, 'buttons': ' '.join(buttons), 'files': files, 'editable': artifact.study.can_edit(user)}
def render(self, study_id, preprocessed_data): user = self.current_user ppd_id = preprocessed_data.id vamps_status = preprocessed_data.is_submitted_to_vamps filepaths = preprocessed_data.filepaths is_local_request = is_localhost(self.request.headers['host']) show_ebi_btn = user.level == "admin" processing_status, processing_status_msg = \ get_artifact_processing_status(preprocessed_data) processed_data = sorted([pd.id for pd in preprocessed_data.children]) # Get all the ENA terms for the investigation type ontology = Ontology(convert_to_id('ENA', 'ontology')) # make "Other" show at the bottom of the drop down menu ena_terms = [] for v in sorted(ontology.terms): if v != 'Other': ena_terms.append('<option value="%s">%s</option>' % (v, v)) ena_terms.append('<option value="Other">Other</option>') # New Type is for users to add a new user-defined investigation type user_defined_terms = ontology.user_defined_terms + ['New Type'] # ppd can only have 1 prep template prep_template = preprocessed_data.prep_templates[0] # this block might seem wrong but is here due to a possible # pathological case that we used to have in the system: preprocessed # data without valid prep_templates prep_templates = preprocessed_data.prep_templates if len(prep_templates) == 1: prep_template_id = prep_template.id raw_data_id = prep_template.artifact.id inv_type = prep_template.investigation_type or "None selected" else: prep_template_id = None raw_data_id = None inv_type = "None Selected" process_params = { param.id: (generate_param_str(param), param.name) for param in Command(3).default_parameter_sets } # We just need to provide an ID for the default parameters, # so we can initialize the interface default_params = min(process_params.keys()) ebi_link = None if preprocessed_data.is_submitted_to_ebi: ebi_link = EBI_LINKIFIER.format( Study(study_id).ebi_study_accession) return self.render_string( "study_description_templates/preprocessed_data_info_tab.html", ppd_id=ppd_id, show_ebi_btn=show_ebi_btn, filepaths=filepaths, is_local_request=is_local_request, prep_template_id=prep_template_id, raw_data_id=raw_data_id, inv_type=inv_type, ena_terms=ena_terms, vamps_status=vamps_status, user_defined_terms=user_defined_terms, process_params=process_params, default_params=default_params, study_id=preprocessed_data.study.id, processing_status=processing_status, processing_status_msg=processing_status_msg, processed_data=processed_data, ebi_link=ebi_link)
# Nov 28, 2017 (only in py file) # Adding a new command into Qiita/Alpha: delete_analysis from qiita_db.software import Software, Command from qiita_db.sql_connection import TRN # Create the delete study command Command.create(Software.from_name_and_version('Qiita', 'alpha'), 'delete_analysis', 'Deletes a full analysis', {'analysis_id': ['integer', None]}) # Make sure that all validate commands have the "analysis" parameter with TRN: # Get all validate commands that are missing the analysis parameter sql = """SELECT command_id FROM qiita.software_command sc WHERE name = 'Validate' AND NOT ( SELECT EXISTS(SELECT * FROM qiita.command_parameter WHERE parameter_name = 'analysis' AND command_id = sc.command_id));""" TRN.add(sql) sql = """INSERT INTO qiita.command_parameter (command_id, parameter_name, parameter_type, required, default_value, name_order, check_biom_merge) VALUES (6, 'analysis', 'analysis', false, NULL, NULL, false)""" sql_params = [[cmd_id, 'analysis', 'analysis', False, None, None, False] for cmd_id in TRN.execute_fetchflatten()] TRN.add(sql, sql_params, many=True) TRN.execute()
def artifact_summary_get_request(user_id, artifact_id): """Returns the information for the artifact summary page Parameters ---------- user_id : str The user making the request artifact_id : int or str The artifact id Returns ------- dict of objects A dictionary containing the artifact summary information {'status': str, 'message': str, 'name': str, 'summary': str, 'job': list of [str, str, str]} """ artifact_id = int(artifact_id) artifact = Artifact(artifact_id) access_error = check_access(artifact.study.id, user_id) if access_error: return access_error user = User(user_id) visibility = artifact.visibility summary = artifact.html_summary_fp job_info = None errored_jobs = [] processing_jobs = [] for j in artifact.jobs(): if j.command.software.type == "artifact transformation": status = j.status if status == 'success': continue j_msg = j.log.msg if status == 'error' else None processing_jobs.append( [j.id, j.command.name, j.status, j.step, j_msg]) # Check if the HTML summary exists if summary: with open(summary[1]) as f: summary = f.read() else: # Check if the summary is being generated command = Command.get_html_generator(artifact.artifact_type) all_jobs = set(artifact.jobs(cmd=command)) jobs = [j for j in all_jobs if j.status in ['queued', 'running']] errored_jobs = [(j.id, j.log.msg) for j in all_jobs if j.status in ['error']] if jobs: # There is already a job generating the HTML. Also, there should be # at most one job, because we are not allowing here to start more # than one job = jobs[0] job_info = [job.id, job.status, job.step] buttons = [] btn_base = ( '<button onclick="if (confirm(\'Are you sure you want to %s ' 'artifact id: {0}?\')) {{ set_artifact_visibility(\'%s\', {0}) }}" ' 'class="btn btn-primary btn-sm">%s</button>').format(artifact_id) if qiita_config.require_approval: if visibility == 'sandbox': # The request approval button only appears if the artifact is # sandboxed and the qiita_config specifies that the approval should # be requested buttons.append(btn_base % ('request approval for', 'awaiting_approval', 'Request approval')) elif user.level == 'admin' and visibility == 'awaiting_approval': # The approve artifact button only appears if the user is an admin # the artifact is waiting to be approvaed and the qiita config # requires artifact approval buttons.append(btn_base % ('approve', 'private', 'Approve artifact')) if visibility == 'private': # The make public button only appears if the artifact is private buttons.append(btn_base % ('make public', 'public', 'Make public')) # The revert to sandbox button only appears if the artifact is not # sandboxed nor public if visibility not in {'sandbox', 'public'}: buttons.append(btn_base % ('revert to sandbox', 'sandbox', 'Revert to sandbox')) if user.level == 'admin': if artifact.can_be_submitted_to_ebi: if not artifact.is_submitted_to_ebi: buttons.append( '<a class="btn btn-primary btn-sm" ' 'href="/ebi_submission/%d">' '<span class="glyphicon glyphicon-export"></span>' ' Submit to EBI</a>' % artifact_id) if artifact.can_be_submitted_to_vamps: if not artifact.is_submitted_to_vamps: buttons.append( '<a class="btn btn-primary btn-sm" href="/vamps/%d">' '<span class="glyphicon glyphicon-export"></span>' ' Submit to VAMPS</a>' % artifact_id) files = [(f_id, "%s (%s)" % (basename(fp), f_type.replace('_', ' '))) for f_id, fp, f_type in artifact.filepaths if f_type != 'directory'] # TODO: https://github.com/biocore/qiita/issues/1724 Remove this hardcoded # values to actually get the information from the database once it stores # the information if artifact.artifact_type in [ 'SFF', 'FASTQ', 'FASTA', 'FASTA_Sanger', 'per_sample_FASTQ' ]: # If the artifact is one of the "raw" types, only the owner of the # study and users that has been shared with can see the files if not artifact.study.has_access(user, no_public=True): files = [] return { 'status': 'success', 'message': '', 'name': artifact.name, 'summary': summary, 'job': job_info, 'errored_jobs': errored_jobs, 'processing_jobs': processing_jobs, 'visibility': visibility, 'buttons': ' '.join(buttons), 'files': files, 'editable': artifact.study.can_edit(user), 'study_id': artifact.study.id, 'prep_id': artifact.prep_templates[0].id }
def artifact_summary_get_request(user, artifact_id): """Returns the information for the artifact summary page Parameters ---------- user : qiita_db.user.User The user making the request artifact_id : int or str The artifact id Returns ------- dict of objects A dictionary containing the artifact summary information {'name': str, 'artifact_id': int, 'visibility': str, 'editable': bool, 'buttons': str, 'processing_parameters': dict of {str: object}, 'files': list of (int, str), 'is_from_analysis': bool, 'processing_jobs': list of [str, str, str, str, str], 'summary': str or None, 'job': [str, str, str], 'errored_jobs': list of [str, str]} """ artifact_id = int(artifact_id) artifact = Artifact(artifact_id) artifact_type = artifact.artifact_type check_artifact_access(user, artifact) visibility = artifact.visibility summary = artifact.html_summary_fp job_info = None errored_jobs = [] processing_jobs = [] for j in artifact.jobs(): if j.command.software.type == "artifact transformation": status = j.status if status == 'success': continue j_msg = j.log.msg if status == 'error' else None processing_jobs.append( [j.id, j.command.name, j.status, j.step, j_msg]) # Check if the HTML summary exists if summary: # Magic number 1: If the artifact has a summary, the call # artifact.html_summary_fp returns a tuple with 2 elements. The first # element is the filepath id, while the second one is the actual # actual filepath. We are only interested on the actual filepath, # hence the 1 value. summary = relpath(summary[1], qiita_config.base_data_dir) else: # Check if the summary is being generated command = Command.get_html_generator(artifact_type) all_jobs = set(artifact.jobs(cmd=command)) jobs = [j for j in all_jobs if j.status in ['queued', 'running']] errored_jobs = [(j.id, j.log.msg) for j in all_jobs if j.status in ['error']] if jobs: # There is already a job generating the HTML. Also, there should be # at most one job, because we are not allowing here to start more # than one job = jobs[0] job_info = [job.id, job.status, job.step] # Check if the artifact is editable by the given user study = artifact.study analysis = artifact.analysis editable = study.can_edit(user) if study else analysis.can_edit(user) buttons = [] btn_base = ( '<button onclick="if (confirm(\'Are you sure you want to %s ' 'artifact id: {0}?\')) {{ set_artifact_visibility(\'%s\', {0}) }}" ' 'class="btn btn-primary btn-sm">%s</button>').format(artifact_id) if analysis: # If the artifact is part of an analysis, we don't require admin # approval, and the artifact can be made public only if all the # artifacts used to create the initial artifact set are public if analysis.can_be_publicized and visibility != 'public': buttons.append(btn_base % ('make public', 'public', 'Make public')) else: # If the artifact is part of a study, the buttons shown depend in # multiple factors (see each if statement for an explanation of those) if qiita_config.require_approval: if visibility == 'sandbox': # The request approval button only appears if the artifact is # sandboxed and the qiita_config specifies that the approval # should be requested buttons.append(btn_base % ('request approval for', 'awaiting_approval', 'Request approval')) elif user.level == 'admin' and visibility == 'awaiting_approval': # The approve artifact button only appears if the user is an # admin the artifact is waiting to be approvaed and the qiita # config requires artifact approval buttons.append(btn_base % ('approve', 'private', 'Approve artifact')) if visibility == 'private': # The make public button only appears if the artifact is private buttons.append(btn_base % ('make public', 'public', 'Make public')) # The revert to sandbox button only appears if the artifact is not # sandboxed nor public if visibility not in {'sandbox', 'public'}: buttons.append( btn_base % ('revert to sandbox', 'sandbox', 'Revert to sandbox')) if user.level == 'admin': if artifact.can_be_submitted_to_ebi: buttons.append( '<a class="btn btn-primary btn-sm" ' 'href="/ebi_submission/%d">' '<span class="glyphicon glyphicon-export"></span>' ' Submit to EBI</a>' % artifact_id) if artifact.can_be_submitted_to_vamps: if not artifact.is_submitted_to_vamps: buttons.append( '<a class="btn btn-primary btn-sm" href="/vamps/%d">' '<span class="glyphicon glyphicon-export"></span>' ' Submit to VAMPS</a>' % artifact_id) files = [(f_id, "%s (%s)" % (basename(fp), f_type.replace('_', ' '))) for f_id, fp, f_type in artifact.filepaths if f_type != 'directory'] # TODO: https://github.com/biocore/qiita/issues/1724 Remove this hardcoded # values to actually get the information from the database once it stores # the information if artifact_type in [ 'SFF', 'FASTQ', 'FASTA', 'FASTA_Sanger', 'per_sample_FASTQ' ]: # If the artifact is one of the "raw" types, only the owner of the # study and users that has been shared with can see the files if not artifact.study.has_access(user, no_public=True): files = [] processing_parameters = (artifact.processing_parameters.values if artifact.processing_parameters is not None else {}) return { 'name': artifact.name, 'artifact_id': artifact_id, 'artifact_type': artifact_type, 'visibility': visibility, 'editable': editable, 'buttons': ' '.join(buttons), 'processing_parameters': processing_parameters, 'files': files, 'is_from_analysis': artifact.analysis is not None, 'processing_jobs': processing_jobs, 'summary': summary, 'job': job_info, 'artifact_timestamp': artifact.timestamp.strftime("%Y-%m-%d %H:%m"), 'errored_jobs': errored_jobs }
def artifact_summary_get_request(user, artifact_id): """Returns the information for the artifact summary page Parameters ---------- user : qiita_db.user.User The user making the request artifact_id : int or str The artifact id Returns ------- dict of objects A dictionary containing the artifact summary information {'name': str, 'artifact_id': int, 'visibility': str, 'editable': bool, 'buttons': str, 'processing_parameters': dict of {str: object}, 'files': list of (int, str), 'is_from_analysis': bool, 'summary': str or None, 'job': [str, str, str], 'errored_jobs': list of [str, str]} """ artifact_id = int(artifact_id) artifact = Artifact(artifact_id) artifact_type = artifact.artifact_type check_artifact_access(user, artifact) visibility = artifact.visibility summary = artifact.html_summary_fp job_info = None errored_summary_jobs = [] # Check if the HTML summary exists if summary: # Magic number 1: If the artifact has a summary, the call # artifact.html_summary_fp returns a tuple with 2 elements. The first # element is the filepath id, while the second one is the actual # actual filepath. We are only interested on the actual filepath, # hence the 1 value. summary = relpath(summary[1], qiita_config.base_data_dir) else: # Check if the summary is being generated command = Command.get_html_generator(artifact_type) all_jobs = set(artifact.jobs(cmd=command)) jobs = [] errored_summary_jobs = [] for j in all_jobs: if j.status in ['queued', 'running']: jobs.append(j) elif j.status in ['error']: errored_summary_jobs.append(j) if jobs: # There is already a job generating the HTML. Also, there should be # at most one job, because we are not allowing here to start more # than one job = jobs[0] job_info = [job.id, job.status, job.step] # Check if the artifact is editable by the given user study = artifact.study analysis = artifact.analysis if artifact_type == 'job-output-folder': editable = False else: editable = study.can_edit(user) if study else analysis.can_edit(user) buttons = [] btn_base = ( '<button onclick="if (confirm(\'Are you sure you want to %s ' 'artifact id: {0}?\')) {{ set_artifact_visibility(\'%s\', {0}) }}" ' 'class="btn btn-primary btn-sm">%s</button>').format(artifact_id) if not analysis and artifact_type != 'job-output-folder': # If the artifact is part of a study, the buttons shown depend in # multiple factors (see each if statement for an explanation of those) if qiita_config.require_approval: if visibility == 'sandbox' and artifact.parents: # The request approval button only appears if the artifact is # sandboxed and the qiita_config specifies that the approval # should be requested buttons.append(btn_base % ('request approval for', 'awaiting_approval', 'Request approval')) elif user.level == 'admin' and visibility == 'awaiting_approval': # The approve artifact button only appears if the user is an # admin the artifact is waiting to be approvaed and the qiita # config requires artifact approval buttons.append(btn_base % ('approve', 'private', 'Approve artifact')) if visibility == 'private': # The make public button only appears if the artifact is private buttons.append(btn_base % ('make public', 'public', 'Make public')) # The revert to sandbox button only appears if the artifact is not # sandboxed nor public if visibility not in {'sandbox', 'public'}: buttons.append( btn_base % ('revert to sandbox', 'sandbox', 'Revert to sandbox')) if user.level == 'admin' and not study.autoloaded: if artifact.can_be_submitted_to_ebi: buttons.append( '<a class="btn btn-primary btn-sm" ' 'href="/ebi_submission/%d">' '<span class="glyphicon glyphicon-export"></span>' ' Submit to EBI</a>' % artifact_id) if artifact.can_be_submitted_to_vamps: if not artifact.is_submitted_to_vamps: buttons.append( '<a class="btn btn-primary btn-sm" href="/vamps/%d">' '<span class="glyphicon glyphicon-export"></span>' ' Submit to VAMPS</a>' % artifact_id) if visibility != 'public': # Have no fear, this is just python to generate html with an onclick in # javascript that makes an ajax call to a separate url, takes the # response and writes it to the newly uncollapsed div. Do note that # you have to be REALLY CAREFUL with properly escaping quotation marks. private_download = ( '<button class="btn btn-primary btn-sm" type="button" ' 'aria-expanded="false" aria-controls="privateDownloadLink" ' 'onclick="generate_private_download_link(%d)">Generate ' 'Download Link</button><div class="collapse" ' 'id="privateDownloadLink"><div class="card card-body" ' 'id="privateDownloadText">Generating Download Link...' '</div></div>') % artifact_id buttons.append(private_download) files = [(x['fp_id'], "%s (%s)" % (basename(x['fp']), x['fp_type'].replace('_', ' ')), x['checksum'], naturalsize(x['fp_size'], gnu=True)) for x in artifact.filepaths if x['fp_type'] != 'directory'] # TODO: https://github.com/biocore/qiita/issues/1724 Remove this hardcoded # values to actually get the information from the database once it stores # the information if artifact_type in [ 'SFF', 'FASTQ', 'FASTA', 'FASTA_Sanger', 'per_sample_FASTQ' ]: # If the artifact is one of the "raw" types, only the owner of the # study and users that has been shared with can see the files study = artifact.study has_access = study.has_access(user, no_public=True) if (not study.public_raw_download and not has_access): files = [] proc_params = artifact.processing_parameters if proc_params: cmd = proc_params.command sw = cmd.software processing_info = { 'command': cmd.name, 'software': sw.name, 'software_version': sw.version, 'processing_parameters': proc_params.values, 'command_active': cmd.active, 'software_deprecated': sw.deprecated, } else: processing_info = {} return { 'name': artifact.name, 'artifact_id': artifact_id, 'artifact_type': artifact_type, 'visibility': visibility, 'editable': editable, 'buttons': ' '.join(buttons), 'processing_info': processing_info, 'files': files, 'is_from_analysis': artifact.analysis is not None, 'summary': summary, 'job': job_info, 'artifact_timestamp': artifact.timestamp.strftime("%Y-%m-%d %H:%m"), 'being_deleted': artifact.being_deleted_by is not None, 'errored_summary_jobs': errored_summary_jobs }
def test_download_study(self): tmp_dir = mkdtemp() self._clean_up_files.append(tmp_dir) biom_fp = join(tmp_dir, 'otu_table.biom') smr_dir = join(tmp_dir, 'sortmerna_picked_otus') log_dir = join(smr_dir, 'seqs_otus.log') tgz = join(tmp_dir, 'sortmerna_picked_otus.tgz') with biom_open(biom_fp, 'w') as f: et.to_hdf5(f, "test") makedirs(smr_dir) with open(log_dir, 'w') as f: f.write('\n') with open(tgz, 'w') as f: f.write('\n') files_biom = [(biom_fp, 'biom'), (smr_dir, 'directory'), (tgz, 'tgz')] params = Parameters.from_default_params( Command(3).default_parameter_sets.next(), {'input_data': 1}) a = Artifact.create(files_biom, "BIOM", parents=[Artifact(2)], processing_parameters=params) for _, fp, _ in a.filepaths: self._clean_up_files.append(fp) response = self.get('/download_study_bioms/1') self.assertEqual(response.code, 200) exp = ( '- 1256812 /protected/processed_data/1_study_1001_closed_' 'reference_otu_table.biom processed_data/1_study_1001_closed_' 'reference_otu_table.biom\n' '- 36615 /protected/templates/1_prep_1_qiime_[0-9]*-' '[0-9]*.txt mapping_files/4_mapping_file.txt\n' '- 1256812 /protected/processed_data/' '1_study_1001_closed_reference_otu_table.biom processed_data/' '1_study_1001_closed_reference_otu_table.biom\n' '- 36615 /protected/templates/1_prep_1_qiime_[0-9]*-' '[0-9]*.txt mapping_files/5_mapping_file.txt\n' '- 1256812 /protected/processed_data/' '1_study_1001_closed_reference_otu_table_Silva.biom processed_data' '/1_study_1001_closed_reference_otu_table_Silva.biom\n' '- 36615 /protected/templates/1_prep_1_qiime_[0-9]*-' '[0-9]*.txt mapping_files/6_mapping_file.txt\n' '- 36615 /protected/templates/1_prep_2_qiime_[0-9]*-' '[0-9]*.txt mapping_files/7_mapping_file.txt\n' '- 39752 /protected/BIOM/{0}/otu_table.biom ' 'BIOM/{0}/otu_table.biom\n' '- 1 /protected/BIOM/{0}/sortmerna_picked_otus/seqs_otus.log ' 'BIOM/{0}/sortmerna_picked_otus/seqs_otus.log\n' '- 36615 /protected/templates/1_prep_1_qiime_[0-9]*-[0-9]*.' 'txt mapping_files/{0}_mapping_file.txt\n'.format(a.id)) self.assertRegexpMatches(response.body, exp) response = self.get('/download_study_bioms/200') self.assertEqual(response.code, 405) # changing user so we can test the failures BaseHandler.get_current_user = Mock( return_value=User("*****@*****.**")) response = self.get('/download_study_bioms/1') self.assertEqual(response.code, 405) a.visibility = 'public' response = self.get('/download_study_bioms/1') self.assertEqual(response.code, 200) exp = ('- 39752 /protected/BIOM/{0}/otu_table.biom ' 'BIOM/{0}/otu_table.biom\n' '- 1 /protected/BIOM/{0}/sortmerna_picked_otus/seqs_otus.log ' 'BIOM/{0}/sortmerna_picked_otus/seqs_otus.log\n' '- 36615 /protected/templates/1_prep_1_qiime_[0-9]*-[0-9]*.' 'txt mapping_files/{0}_mapping_file.txt\n'.format(a.id)) self.assertRegexpMatches(response.body, exp)
def create_command(software, name, description, parameters, outputs=None, analysis_only=False): r"""Replicates the Command.create code at the time the patch was written""" # Perform some sanity checks in the parameters dictionary if not parameters: raise QiitaDBError( "Error creating command %s. At least one parameter should " "be provided." % name) sql_param_values = [] sql_artifact_params = [] for pname, vals in parameters.items(): if len(vals) != 2: raise QiitaDBError( "Malformed parameters dictionary, the format should be " "{param_name: [parameter_type, default]}. Found: " "%s for parameter name %s" % (vals, pname)) ptype, dflt = vals # Check that the type is one of the supported types supported_types = ['string', 'integer', 'float', 'reference', 'boolean', 'prep_template', 'analysis'] if ptype not in supported_types and not ptype.startswith( ('choice', 'mchoice', 'artifact')): supported_types.extend(['choice', 'mchoice', 'artifact']) raise QiitaDBError( "Unsupported parameters type '%s' for parameter %s. " "Supported types are: %s" % (ptype, pname, ', '.join(supported_types))) if ptype.startswith(('choice', 'mchoice')) and dflt is not None: choices = set(loads(ptype.split(':')[1])) dflt_val = dflt if ptype.startswith('choice'): # In the choice case, the dflt value is a single string, # create a list with it the string on it to use the # issuperset call below dflt_val = [dflt_val] else: # jsonize the list to store it in the DB dflt = dumps(dflt) if not choices.issuperset(dflt_val): raise QiitaDBError( "The default value '%s' for the parameter %s is not " "listed in the available choices: %s" % (dflt, pname, ', '.join(choices))) if ptype.startswith('artifact'): atypes = loads(ptype.split(':')[1]) sql_artifact_params.append( [pname, 'artifact', atypes]) else: if dflt is not None: sql_param_values.append([pname, ptype, False, dflt]) else: sql_param_values.append([pname, ptype, True, None]) with TRN: sql = """SELECT EXISTS(SELECT * FROM qiita.software_command WHERE software_id = %s AND name = %s)""" TRN.add(sql, [software.id, name]) if TRN.execute_fetchlast(): raise QiitaDBDuplicateError( "command", "software: %d, name: %s" % (software.id, name)) # Add the command to the DB sql = """INSERT INTO qiita.software_command (name, software_id, description, is_analysis) VALUES (%s, %s, %s, %s) RETURNING command_id""" sql_params = [name, software.id, description, analysis_only] TRN.add(sql, sql_params) c_id = TRN.execute_fetchlast() # Add the parameters to the DB sql = """INSERT INTO qiita.command_parameter (command_id, parameter_name, parameter_type, required, default_value) VALUES (%s, %s, %s, %s, %s) RETURNING command_parameter_id""" sql_params = [[c_id, pname, p_type, reqd, default] for pname, p_type, reqd, default in sql_param_values] TRN.add(sql, sql_params, many=True) TRN.execute() # Add the artifact parameters sql_type = """INSERT INTO qiita.parameter_artifact_type (command_parameter_id, artifact_type_id) VALUES (%s, %s)""" supported_types = [] for pname, p_type, atypes in sql_artifact_params: sql_params = [c_id, pname, p_type, True, None] TRN.add(sql, sql_params) pid = TRN.execute_fetchlast() sql_params = [[pid, convert_to_id(at, 'artifact_type')] for at in atypes] TRN.add(sql_type, sql_params, many=True) supported_types.extend([atid for _, atid in sql_params]) # If the software type is 'artifact definition', there are a couple # of extra steps if software.type == 'artifact definition': # If supported types is not empty, link the software with these # types if supported_types: sql = """INSERT INTO qiita.software_artifact_type (software_id, artifact_type_id) VALUES (%s, %s)""" sql_params = [[software.id, atid] for atid in supported_types] TRN.add(sql, sql_params, many=True) # If this is the validate command, we need to add the # provenance and name parameters. These are used internally, # that's why we are adding them here if name == 'Validate': sql = """INSERT INTO qiita.command_parameter (command_id, parameter_name, parameter_type, required, default_value) VALUES (%s, 'name', 'string', 'False', 'dflt_name'), (%s, 'provenance', 'string', 'False', NULL) """ TRN.add(sql, [c_id, c_id]) # Add the outputs to the command if outputs: sql = """INSERT INTO qiita.command_output (name, command_id, artifact_type_id) VALUES (%s, %s, %s)""" sql_args = [[pname, c_id, convert_to_id(at, 'artifact_type')] for pname, at in outputs.items()] TRN.add(sql, sql_args, many=True) TRN.execute() return Command(c_id)
def artifact_post_req(user_id, filepaths, artifact_type, name, prep_template_id, artifact_id=None): """Creates the initial artifact for the prep template Parameters ---------- user_id : str User adding the atrifact filepaths : dict of str Comma-separated list of files to attach to the artifact, keyed by file type artifact_type : str The type of the artifact name : str Name to give the artifact prep_template_id : int or str castable to int Prep template to attach the artifact to artifact_id : int or str castable to int, optional The id of the imported artifact Returns ------- dict of objects A dictionary containing the new artifact ID {'status': status, 'message': message, 'artifact': id} """ prep_template_id = int(prep_template_id) prep = PrepTemplate(prep_template_id) study_id = prep.study_id # First check if the user has access to the study access_error = check_access(study_id, user_id) if access_error: return access_error if artifact_id: # if the artifact id has been provided, import the artifact job_id = safe_submit(user_id, copy_raw_data, prep, artifact_id) is_qiita_job = False else: uploads_path = get_mountpoint('uploads')[0][1] path_builder = partial(join, uploads_path, str(study_id)) cleaned_filepaths = {} for ftype, file_list in viewitems(filepaths): # JavaScript sends us this list as a comma-separated list for fp in file_list.split(','): # JavaScript will send this value as an empty string if the # list of files was empty. In such case, the split will # generate a single element containing the empty string. Check # for that case here and, if fp is not the empty string, # proceed to check if the file exists if fp: # Check if filepath being passed exists for study full_fp = path_builder(fp) exists = check_fp(study_id, full_fp) if exists['status'] != 'success': return {'status': 'error', 'message': 'File does not exist: %s' % fp} if ftype not in cleaned_filepaths: cleaned_filepaths[ftype] = [] cleaned_filepaths[ftype].append(full_fp) # This should never happen, but it doesn't hurt to actually have # a explicit check, in case there is something odd with the JS if not cleaned_filepaths: return {'status': 'error', 'message': "Can't create artifact, no files provided."} command = Command.get_validator(artifact_type) job = ProcessingJob.create( User(user_id), Parameters.load(command, values_dict={ 'template': prep_template_id, 'files': dumps(cleaned_filepaths), 'artifact_type': artifact_type })) job.submit() job_id = job.id is_qiita_job = True r_client.set(PREP_TEMPLATE_KEY_FORMAT % prep.id, dumps({'job_id': job_id, 'is_qiita_job': is_qiita_job})) return {'status': 'success', 'message': ''}
def test_job_ajax_patch_req(self): # Create a new job - through a workflow since that is the only way # of creating jobs in the interface exp_command = Command(1) json_str = ( '{"input_data": 1, "max_barcode_errors": 1.5, ' '"barcode_type": "golay_12", "max_bad_run_length": 3, ' '"rev_comp": false, "phred_quality_threshold": 3, ' '"rev_comp_barcode": false, "rev_comp_mapping_barcodes": false, ' '"min_per_read_length_fraction": 0.75, "sequence_max_n": 0}') exp_params = Parameters.load(exp_command, json_str=json_str) exp_user = User('*****@*****.**') name = "Test processing workflow" # tests success wf = ProcessingWorkflow.from_scratch(exp_user, exp_params, name=name, force=True) graph = wf.graph nodes = list(graph.nodes()) job_id = nodes[0].id # Incorrect path parameter obs = job_ajax_patch_req('remove', '/%s/somethingelse' % job_id) exp = { 'status': 'error', 'message': 'Incorrect path parameter: missing job id' } self.assertEqual(obs, exp) obs = job_ajax_patch_req('remove', '/') exp = { 'status': 'error', 'message': 'Incorrect path parameter: missing job id' } self.assertEqual(obs, exp) # Job id is not like a job id obs = job_ajax_patch_req('remove', '/notAJobId') exp = { 'status': 'error', 'message': 'Incorrect path parameter: ' 'notAJobId is not a recognized job id' } self.assertEqual(obs, exp) # Job doesn't exist obs = job_ajax_patch_req('remove', '/6d368e16-2242-4cf8-87b4-a5dc40bc890b') exp = { 'status': 'error', 'message': 'Incorrect path parameter: ' '6d368e16-2242-4cf8-87b4-a5dc40bc890b is not a ' 'recognized job id' } self.assertEqual(obs, exp) # in_construction job obs = job_ajax_patch_req('remove', '/%s' % job_id) exp = { 'status': 'error', 'message': "Can't delete job %s. It is 'in_construction' " "status. Please use /study/process/workflow/" % job_id } self.assertEqual(obs, exp) # job status != 'error' job = ProcessingJob(job_id) job._set_status('queued') obs = job_ajax_patch_req('remove', '/%s' % job_id) exp = { 'status': 'error', 'message': 'Only jobs in "error" status can be deleted.' } self.assertEqual(obs, exp) # Operation not supported job._set_status('queued') obs = job_ajax_patch_req('add', '/%s' % job_id) exp = { 'status': 'error', 'message': 'Operation "add" not supported. Current supported ' 'operations: remove' } self.assertEqual(obs, exp) # Test success job._set_error('Killed for testing') obs = job_ajax_patch_req('remove', '/%s' % job_id) exp = {'status': 'success', 'message': ''} self.assertEqual(obs, exp)
def test_artifact_summary_get_request(self): # Artifact w/o summary obs = artifact_summary_get_request('*****@*****.**', 1) exp_p_jobs = [[ '063e553b-327c-4818-ab4a-adfe58e49860', 'Split libraries FASTQ', 'queued', None, None ], [ 'bcc7ebcd-39c1-43e4-af2d-822e3589f14d', 'Split libraries', 'running', 'demultiplexing', None ]] exp_files = [ (1L, '1_s_G1_L001_sequences.fastq.gz (raw forward seqs)'), (2L, '1_s_G1_L001_sequences_barcodes.fastq.gz (raw barcodes)') ] exp = { 'status': 'success', 'message': '', 'name': 'Raw data 1', 'summary': None, 'job': None, 'processing_jobs': exp_p_jobs, 'errored_jobs': [], 'visibility': 'private', 'buttons': ('<button onclick="if (confirm(\'Are you sure you ' 'want to make public artifact id: 1?\')) { ' 'set_artifact_visibility(\'public\', 1) }" ' 'class="btn btn-primary btn-sm">Make public' '</button> <button onclick="if (confirm(\'Are you ' 'sure you want to revert to sandbox artifact id: ' '1?\')) { set_artifact_visibility(\'sandbox\', 1) ' '}" class="btn btn-primary btn-sm">Revert to ' 'sandbox</button>'), 'files': exp_files, 'editable': True, 'prep_id': 1, 'study_id': 1 } self.assertEqual(obs, exp) # Artifact with summary being generated job = ProcessingJob.create( User('*****@*****.**'), Parameters.load(Command(7), values_dict={'input_data': 1})) job._set_status('queued') obs = artifact_summary_get_request('*****@*****.**', 1) exp = { 'status': 'success', 'message': '', 'name': 'Raw data 1', 'summary': None, 'job': [job.id, 'queued', None], 'processing_jobs': exp_p_jobs, 'errored_jobs': [], 'visibility': 'private', 'buttons': ('<button onclick="if (confirm(\'Are you sure you ' 'want to make public artifact id: 1?\')) { ' 'set_artifact_visibility(\'public\', 1) }" ' 'class="btn btn-primary btn-sm">Make public' '</button> <button onclick="if (confirm(\'Are you ' 'sure you want to revert to sandbox artifact id: ' '1?\')) { set_artifact_visibility(\'sandbox\', 1) ' '}" class="btn btn-primary btn-sm">Revert to ' 'sandbox</button>'), 'files': exp_files, 'editable': True, 'prep_id': 1, 'study_id': 1 } self.assertEqual(obs, exp) # Artifact with summary fd, fp = mkstemp(suffix=".html") close(fd) with open(fp, 'w') as f: f.write('<b>HTML TEST - not important</b>\n') a = Artifact(1) a.html_summary_fp = fp self._files_to_remove.extend([fp, a.html_summary_fp[1]]) exp_files.append( (a.html_summary_fp[0], '%s (html summary)' % basename(a.html_summary_fp[1]))) obs = artifact_summary_get_request('*****@*****.**', 1) exp = { 'status': 'success', 'message': '', 'name': 'Raw data 1', 'summary': '<b>HTML TEST - not important</b>\n', 'job': None, 'processing_jobs': exp_p_jobs, 'errored_jobs': [], 'visibility': 'private', 'buttons': ('<button onclick="if (confirm(\'Are you sure you ' 'want to make public artifact id: 1?\')) { ' 'set_artifact_visibility(\'public\', 1) }" ' 'class="btn btn-primary btn-sm">Make public' '</button> <button onclick="if (confirm(\'Are you ' 'sure you want to revert to sandbox artifact id: ' '1?\')) { set_artifact_visibility(\'sandbox\', 1) ' '}" class="btn btn-primary btn-sm">Revert to ' 'sandbox</button>'), 'files': exp_files, 'editable': True, 'prep_id': 1, 'study_id': 1 } self.assertEqual(obs, exp) # No access obs = artifact_summary_get_request('*****@*****.**', 1) exp = { 'status': 'error', 'message': 'User does not have access to study' } self.assertEqual(obs, exp) # A non-owner/share user can't see the files a.visibility = 'public' obs = artifact_summary_get_request('*****@*****.**', 1) exp = { 'status': 'success', 'message': '', 'name': 'Raw data 1', 'summary': '<b>HTML TEST - not important</b>\n', 'job': None, 'processing_jobs': exp_p_jobs, 'errored_jobs': [], 'visibility': 'public', 'buttons': '', 'files': [], 'editable': False, 'prep_id': 1, 'study_id': 1 } self.assertEqual(obs, exp) # returnig to private a.visibility = 'sandbox' # admin gets buttons obs = artifact_summary_get_request('*****@*****.**', 2) exp_p_jobs = [[ 'd19f76ee-274e-4c1b-b3a2-a12d73507c55', 'Pick closed-reference OTUs', 'error', 'generating demux file', 'Error message' ]] exp_files = [(3L, '1_seqs.fna (preprocessed fasta)'), (4L, '1_seqs.qual (preprocessed fastq)'), (5L, '1_seqs.demux (preprocessed demux)')] exp = { 'status': 'success', 'files': exp_files, 'errored_jobs': [], 'editable': True, 'visibility': 'private', 'job': None, 'message': '', 'name': 'Demultiplexed 1', 'processing_jobs': exp_p_jobs, 'summary': None, 'buttons': ('<button onclick="if (confirm(\'Are you sure you ' 'want to make public artifact id: 2?\')) { ' 'set_artifact_visibility(\'public\', 2) }" ' 'class="btn btn-primary btn-sm">Make public' '</button> <button onclick="if (confirm(\'Are you ' 'sure you want to revert to sandbox artifact id: ' '2?\')) { set_artifact_visibility(\'sandbox\', 2) ' '}" class="btn btn-primary btn-sm">Revert to ' 'sandbox</button> <a class="btn btn-primary ' 'btn-sm" href="/vamps/2"><span class="glyphicon ' 'glyphicon-export"></span> Submit to VAMPS</a>'), 'study_id': 1, 'prep_id': 1 } self.assertEqual(obs, exp)
def test_complete_job(self): # Complete success pt = npt.assert_warns( QiitaDBWarning, PrepTemplate.create, pd.DataFrame({'new_col': {'1.SKD6.640190': 1}}), Study(1), '16S') c_job = ProcessingJob.create( User('*****@*****.**'), Parameters.load( Command.get_validator('BIOM'), values_dict={'template': pt.id, 'files': dumps({'BIOM': ['file']}), 'artifact_type': 'BIOM'}), True) c_job._set_status('running') fd, fp = mkstemp(suffix='_table.biom') close(fd) with open(fp, 'w') as f: f.write('\n') self._clean_up_files.append(fp) exp_artifact_count = get_count('qiita.artifact') + 1 payload = dumps( {'success': True, 'error': '', 'artifacts': {'OTU table': {'filepaths': [(fp, 'biom')], 'artifact_type': 'BIOM'}}}) job = self._create_job('complete_job', {'job_id': c_job.id, 'payload': payload}) private_task(job.id) self.assertEqual(job.status, 'success') self.assertEqual(c_job.status, 'success') self.assertEqual(get_count('qiita.artifact'), exp_artifact_count) # Complete job error payload = dumps({'success': False, 'error': 'Job failure'}) job = self._create_job( 'complete_job', {'job_id': 'bcc7ebcd-39c1-43e4-af2d-822e3589f14d', 'payload': payload}) private_task(job.id) self.assertEqual(job.status, 'success') c_job = ProcessingJob('bcc7ebcd-39c1-43e4-af2d-822e3589f14d') self.assertEqual(c_job.status, 'error') self.assertEqual(c_job.log, LogEntry.newest_records(numrecords=1)[0]) self.assertEqual(c_job.log.msg, 'Job failure') # Complete internal error pt = npt.assert_warns( QiitaDBWarning, PrepTemplate.create, pd.DataFrame({'new_col': {'1.SKD6.640190': 1}}), Study(1), '16S') c_job = ProcessingJob.create( User('*****@*****.**'), Parameters.load( Command.get_validator('BIOM'), values_dict={'template': pt.id, 'files': dumps({'BIOM': ['file']}), 'artifact_type': 'BIOM'}), True) c_job._set_status('running') fp = '/surprised/if/this/path/exists.biom' payload = dumps( {'success': True, 'error': '', 'artifacts': {'OTU table': {'filepaths': [(fp, 'biom')], 'artifact_type': 'BIOM'}}}) job = self._create_job('complete_job', {'job_id': c_job.id, 'payload': payload}) private_task(job.id) self.assertEqual(job.status, 'success') self.assertEqual(c_job.status, 'error') self.assertIn('No such file or directory', c_job.log.msg)
def test_complete_job(self): # Complete success pt = npt.assert_warns(QiitaDBWarning, PrepTemplate.create, pd.DataFrame({'new_col': { '1.SKD6.640190': 1 }}), Study(1), '16S') c_job = ProcessingJob.create( User('*****@*****.**'), Parameters.load(Command.get_validator('BIOM'), values_dict={ 'template': pt.id, 'files': dumps({'BIOM': ['file']}), 'artifact_type': 'BIOM' }), True) c_job._set_status('running') fd, fp = mkstemp(suffix='_table.biom') close(fd) with open(fp, 'w') as f: f.write('\n') self._clean_up_files.append(fp) exp_artifact_count = get_count('qiita.artifact') + 1 payload = dumps({ 'success': True, 'error': '', 'artifacts': { 'OTU table': { 'filepaths': [(fp, 'biom')], 'artifact_type': 'BIOM' } } }) job = self._create_job('complete_job', { 'job_id': c_job.id, 'payload': payload }) private_task(job.id) self.assertEqual(job.status, 'success') self.assertEqual(c_job.status, 'success') self.assertEqual(get_count('qiita.artifact'), exp_artifact_count) # Complete job error payload = dumps({'success': False, 'error': 'Job failure'}) job = self._create_job('complete_job', { 'job_id': 'bcc7ebcd-39c1-43e4-af2d-822e3589f14d', 'payload': payload }) private_task(job.id) self.assertEqual(job.status, 'success') c_job = ProcessingJob('bcc7ebcd-39c1-43e4-af2d-822e3589f14d') self.assertEqual(c_job.status, 'error') self.assertEqual(c_job.log, LogEntry.newest_records(numrecords=1)[0]) self.assertEqual(c_job.log.msg, 'Job failure') # Complete internal error pt = npt.assert_warns(QiitaDBWarning, PrepTemplate.create, pd.DataFrame({'new_col': { '1.SKD6.640190': 1 }}), Study(1), '16S') c_job = ProcessingJob.create( User('*****@*****.**'), Parameters.load(Command.get_validator('BIOM'), values_dict={ 'template': pt.id, 'files': dumps({'BIOM': ['file']}), 'artifact_type': 'BIOM' }), True) c_job._set_status('running') fp = '/surprised/if/this/path/exists.biom' payload = dumps({ 'success': True, 'error': '', 'artifacts': { 'OTU table': { 'filepaths': [(fp, 'biom')], 'artifact_type': 'BIOM' } } }) job = self._create_job('complete_job', { 'job_id': c_job.id, 'payload': payload }) private_task(job.id) self.assertEqual(job.status, 'success') self.assertEqual(c_job.status, 'error') self.assertIn('No such file or directory', c_job.log.msg)
def test_get_analysis_graph_handler(self): response = self.get('/analysis/description/1/graph/') self.assertEqual(response.code, 200) # The job id is randomly generated in the test environment. Gather # it here. There is only 1 job in the first artifact of the analysis job_id = Analysis(1).artifacts[0].jobs()[0].id obs = loads(response.body) exp = { 'edges': [[8, job_id], [job_id, 9]], 'nodes': [['job', 'job', job_id, 'Single Rarefaction', 'success'], ['artifact', 'BIOM', 9, 'noname\n(BIOM)', 'artifact'], ['artifact', 'BIOM', 8, 'noname\n(BIOM)', 'artifact']], 'workflow': None } self.assertItemsEqual(obs, exp) self.assertItemsEqual(obs['edges'], exp['edges']) self.assertItemsEqual(obs['nodes'], exp['nodes']) self.assertIsNone(obs['workflow']) # Create a new analysis with 2 starting BIOMs to be able to test # the different if statements of the request BaseHandler.get_current_user = Mock( return_value=User('*****@*****.**')) user = User('*****@*****.**') dflt_analysis = user.default_analysis dflt_analysis.add_samples({ 4: ['1.SKB8.640193', '1.SKD8.640184', '1.SKB7.640196'], 6: ['1.SKB8.640193', '1.SKD8.640184', '1.SKB7.640196'] }) args = {'name': 'New Test Graph Analysis', 'description': 'Desc'} response = self.post('/analysis/create/', args) new_id = response.effective_url.split('/')[-2] a = Analysis(new_id) # Wait until all the jobs are done so the BIOM tables exist for j in a.jobs: wait_for_processing_job(j.id) artifacts = a.artifacts self.assertEqual(len(artifacts), 2) # Create a new workflow starting on the first artifact # Magic number 9 -> Summarize Taxa command params = Parameters.load(Command(9), values_dict={ 'metadata_category': 'None', 'sort': 'False', 'biom_table': artifacts[0].id }) wf = ProcessingWorkflow.from_scratch(user, params) # There is only one job in the workflow job_id = wf.graph.nodes()[0].id response = self.get('/analysis/description/%s/graph/' % new_id) self.assertEqual(response.code, 200) obs = loads(response.body) exp = { 'edges': [[artifacts[0].id, job_id], [job_id, '%s:taxa_summary' % job_id]], 'nodes': [['job', 'job', job_id, 'Summarize Taxa', 'in_construction'], [ 'artifact', 'BIOM', artifacts[0].id, 'noname\n(BIOM)', 'artifact' ], [ 'artifact', 'BIOM', artifacts[1].id, 'noname\n(BIOM)', 'artifact' ], [ 'type', 'taxa_summary', '%s:taxa_summary' % job_id, 'taxa_summary\n(taxa_summary)', 'type' ]], 'workflow': wf.id } # Check that the keys are the same self.assertItemsEqual(obs, exp) # Check the edges self.assertItemsEqual(obs['edges'], exp['edges']) # Check the edges self.assertItemsEqual(obs['nodes'], exp['nodes']) # Check the edges self.assertEqual(obs['workflow'], exp['workflow']) # Add a job to the second BIOM to make sure that the edges and nodes # are respected. Magic number 12 -> Single Rarefaction job2 = wf.add(DefaultParameters(16), req_params={ 'depth': '100', 'biom_table': artifacts[1].id }) job_id_2 = job2.id response = self.get('/analysis/description/%s/graph/' % new_id) self.assertEqual(response.code, 200) obs = loads(response.body) exp = { 'edges': [[artifacts[0].id, job_id], [job_id, '%s:taxa_summary' % job_id], [artifacts[1].id, job_id_2], [job_id_2, '%s:rarefied_table' % job_id_2]], 'nodes': [['job', 'job', job_id, 'Summarize Taxa', 'in_construction'], ['job', 'job', job_id_2, 'Single Rarefaction', 'in_construction'], [ 'artifact', 'BIOM', artifacts[0].id, 'noname\n(BIOM)', 'artifact' ], [ 'artifact', 'BIOM', artifacts[1].id, 'noname\n(BIOM)', 'artifact' ], [ 'type', 'taxa_summary', '%s:taxa_summary' % job_id, 'taxa_summary\n(taxa_summary)', 'type' ], [ 'type', 'BIOM', '%s:rarefied_table' % job_id_2, 'rarefied_table\n(BIOM)', 'type' ]], 'workflow': wf.id } # Check that the keys are the same self.assertItemsEqual(obs, exp) # Check the edges self.assertItemsEqual(obs['edges'], exp['edges']) # Check the edges self.assertItemsEqual(obs['nodes'], exp['nodes']) # Check the edges self.assertEqual(obs['workflow'], exp['workflow']) # Add a second Workflow to the second artifact to force the raise of # the error. This situation should never happen when using # the interface wf.remove(job2) params = Parameters.load(Command(9), values_dict={ 'metadata_category': 'None', 'sort': 'False', 'biom_table': artifacts[1].id }) wf = ProcessingWorkflow.from_scratch(user, params) response = self.get('/analysis/description/%s/graph/' % new_id) self.assertEqual(response.code, 500)
def _construct_job_graph(self, analysis, commands, comm_opts=None, rarefaction_depth=None, merge_duplicated_sample_ids=False): """Builds the job graph for running an analysis Parameters ---------- analysis: Analysis object Analysis to finalize. commands : list of tuples Commands to add as jobs in the analysis. Format [(data_type, command name), ...] comm_opts : dict of dicts, optional Options for commands. Format {command name: {opt1: value,...},...} Default None (use default options). rarefaction_depth : int, optional Rarefaction depth for analysis' biom tables. Default None. merge_duplicated_sample_ids : bool, optional If the duplicated sample ids in the selected studies should be merged or prepended with the artifact ids. False (default) prepends the artifact id """ self._logger = stderr self.analysis = analysis analysis_id = analysis.id # Add jobs to analysis if comm_opts is None: comm_opts = {} analysis.status = "running" # creating bioms at this point cause all this section runs on a worker # node, currently an ipython job analysis.build_files(rarefaction_depth, merge_duplicated_sample_ids) mapping_file = analysis.mapping_file tree_commands = ["Beta Diversity", "Alpha Rarefaction"] for data_type, biom_fp in viewitems(analysis.biom_tables): biom_table = load_table(biom_fp) # getting reference_id and software_command_id from the first # sample of the biom. This decision was discussed on the qiita # meeting on 02/24/16 metadata = biom_table.metadata(biom_table.ids()[0]) rid = metadata['reference_id'] sci = metadata['command_id'] if rid != 'na': reference = Reference(rid) tree = reference.tree_fp else: reference = None tree = '' cmd = Command(sci) if sci != 'na' else None for cmd_data_type, command in commands: if data_type != cmd_data_type: continue # get opts set by user, else make it empty dict opts = comm_opts.get(command, {}) opts["--otu_table_fp"] = biom_fp opts["--mapping_fp"] = mapping_file if command in tree_commands: if tree != '': opts["--tree_fp"] = tree else: opts["--parameter_fp"] = join( get_db_files_base_dir(), "reference", "params_qiime.txt") if command == "Alpha Rarefaction": opts["-n"] = 4 Job.create(data_type, command, opts, analysis, reference, cmd, return_existing=True) # Add the jobs job_nodes = [] for job in analysis.jobs: node_name = "%d_JOB_%d" % (analysis_id, job.id) job_nodes.append(node_name) job_name = "%s: %s" % (job.datatype, job.command[0]) self._job_graph.add_node(node_name, func=system_call_from_job, args=(job.id,), job_name=job_name, requires_deps=False) # tgz-ing the analysis results tgz_node_name = "TGZ_ANALYSIS_%d" % (analysis_id) job_name = "tgz_analysis_%d" % (analysis_id) self._job_graph.add_node(tgz_node_name, func=_generate_analysis_tgz, args=(analysis,), job_name=job_name, requires_deps=False) # Adding the dependency edges to the graph for job_node_name in job_nodes: self._job_graph.add_edge(job_node_name, tgz_node_name) # Finalize the analysis. node_name = "FINISH_ANALYSIS_%d" % analysis.id self._job_graph.add_node(node_name, func=_finish_analysis, args=(analysis,), job_name='Finalize analysis', requires_deps=False) self._job_graph.add_edge(tgz_node_name, node_name)
def test_artifact_summary_get_request(self): user = User('*****@*****.**') main_buttons = ( '<button onclick="if (confirm(' "\'Are you sure you want to make " "public artifact id: 1?')) { set_artifact_visibility('public', 1) " '}" class="btn btn-primary btn-sm">Make public</button> <button ' 'onclick="if (confirm(' "'Are you sure you want to revert to " "sandbox artifact id: 1?')) { set_artifact_visibility('sandbox', 1" ') }" class="btn btn-primary btn-sm">Revert to sandbox</button> ') private_download_button = ( '<button class="btn btn-primary btn-sm" type="button" ' 'aria-expanded="false" aria-controls="privateDownloadLink" ' 'onclick="generate_private_download_link(%s)">Generate Download ' 'Link</button><div class="collapse" id="privateDownloadLink"><div ' 'class="card card-body" id="privateDownloadText">Generating ' 'Download Link...</div></div>') # Artifact w/o summary obs = artifact_summary_get_request(user, 1) exp_files = [ (1, '1_s_G1_L001_sequences.fastq.gz (raw forward seqs)', '2125826711', '58 Bytes'), (2, '1_s_G1_L001_sequences_barcodes.fastq.gz (raw barcodes)', '2125826711', '58 Bytes') ] exp = { 'name': 'Raw data 1', 'artifact_id': 1, 'artifact_type': 'FASTQ', 'artifact_timestamp': '2012-10-01 09:10', 'visibility': 'private', 'editable': True, 'buttons': main_buttons + private_download_button % 1, 'processing_info': {}, 'files': exp_files, 'is_from_analysis': False, 'summary': None, 'job': None, 'errored_summary_jobs': [] } self.assertEqual(obs, exp) # Artifact with summary being generated job = ProcessingJob.create( User('*****@*****.**'), Parameters.load(Command(7), values_dict={'input_data': 1})) job._set_status('queued') obs = artifact_summary_get_request(user, 1) exp = { 'name': 'Raw data 1', 'artifact_id': 1, 'artifact_type': 'FASTQ', 'artifact_timestamp': '2012-10-01 09:10', 'visibility': 'private', 'editable': True, 'buttons': main_buttons + private_download_button % 1, 'processing_info': {}, 'files': exp_files, 'is_from_analysis': False, 'summary': None, 'job': [job.id, 'queued', None], 'errored_summary_jobs': [] } self.assertEqual(obs, exp) # Artifact with summary fd, fp = mkstemp(suffix=".html") close(fd) with open(fp, 'w') as f: f.write('<b>HTML TEST - not important</b>\n') a = Artifact(1) a.set_html_summary(fp) self._files_to_remove.extend([fp, a.html_summary_fp[1]]) exp_files.append((a.html_summary_fp[0], '%s (html summary)' % basename(a.html_summary_fp[1]), '1642196267', '33 Bytes')) exp_summary_path = relpath(a.html_summary_fp[1], qiita_config.base_data_dir) obs = artifact_summary_get_request(user, 1) exp = { 'name': 'Raw data 1', 'artifact_id': 1, 'artifact_type': 'FASTQ', 'artifact_timestamp': '2012-10-01 09:10', 'visibility': 'private', 'editable': True, 'buttons': main_buttons + private_download_button % 1, 'processing_info': {}, 'files': exp_files, 'is_from_analysis': False, 'summary': exp_summary_path, 'job': None, 'errored_summary_jobs': [] } self.assertEqual(obs, exp) # No access demo_u = User('*****@*****.**') with self.assertRaises(QiitaHTTPError): obs = artifact_summary_get_request(demo_u, 1) # A non-owner/share user can't see the files a.visibility = 'public' obs = artifact_summary_get_request(demo_u, 1) exp = { 'name': 'Raw data 1', 'artifact_id': 1, 'artifact_type': 'FASTQ', 'artifact_timestamp': '2012-10-01 09:10', 'visibility': 'public', 'editable': False, 'buttons': '', 'processing_info': {}, 'files': [], 'is_from_analysis': False, 'summary': exp_summary_path, 'job': None, 'errored_summary_jobs': [] } self.assertEqual(obs, exp) # testing sandbox a.visibility = 'sandbox' obs = artifact_summary_get_request(user, 1) exp = { 'name': 'Raw data 1', 'artifact_id': 1, 'artifact_type': 'FASTQ', 'artifact_timestamp': '2012-10-01 09:10', 'visibility': 'sandbox', 'editable': True, 'buttons': private_download_button % 1, 'processing_info': {}, 'files': exp_files, 'is_from_analysis': False, 'summary': exp_summary_path, 'job': None, 'errored_summary_jobs': [] } self.assertEqual(obs, exp) # returnig to private a.visibility = 'private' # admin gets buttons obs = artifact_summary_get_request(User('*****@*****.**'), 2) exp_files = [(3, '1_seqs.fna (preprocessed fasta)', '', '0 Bytes'), (4, '1_seqs.qual (preprocessed fastq)', '', '0 Bytes'), (5, '1_seqs.demux (preprocessed demux)', '', '0 Bytes')] exp = { 'name': 'Demultiplexed 1', 'artifact_id': 2, 'artifact_type': 'Demultiplexed', 'artifact_timestamp': '2012-10-01 10:10', 'visibility': 'private', 'editable': True, 'buttons': ('<button onclick="if (confirm(\'Are you sure you ' 'want to make public artifact id: 2?\')) { ' 'set_artifact_visibility(\'public\', 2) }" ' 'class="btn btn-primary btn-sm">Make public' '</button> <button onclick="if (confirm(\'Are you ' 'sure you want to revert to sandbox artifact id: ' '2?\')) { set_artifact_visibility(\'sandbox\', 2) ' '}" class="btn btn-primary btn-sm">Revert to ' 'sandbox</button> <a class="btn btn-primary ' 'btn-sm" href="/ebi_submission/2"><span ' 'class="glyphicon glyphicon-export"></span> ' 'Submit to EBI</a> <a class="btn btn-primary ' 'btn-sm" href="/vamps/2"><span class="glyphicon ' 'glyphicon-export"></span> Submit to VAMPS</a> ' + private_download_button % 2), 'processing_info': { 'command_active': True, 'software_deprecated': False, 'command': 'Split libraries FASTQ', 'processing_parameters': { 'max_barcode_errors': '1.5', 'sequence_max_n': '0', 'max_bad_run_length': '3', 'phred_offset': 'auto', 'rev_comp': 'False', 'phred_quality_threshold': '3', 'input_data': '1', 'rev_comp_barcode': 'False', 'rev_comp_mapping_barcodes': 'False', 'min_per_read_length_fraction': '0.75', 'barcode_type': 'golay_12' }, 'software_version': '1.9.1', 'software': 'QIIME' }, 'files': exp_files, 'is_from_analysis': False, 'summary': None, 'job': None, 'errored_summary_jobs': [] } self.assertEqual(obs, exp) # analysis artifact obs = artifact_summary_get_request(user, 8) exp = { 'name': 'noname', 'artifact_id': 8, 'artifact_type': 'BIOM', # this value changes on build so copy from obs 'artifact_timestamp': obs['artifact_timestamp'], 'visibility': 'sandbox', 'editable': True, 'buttons': private_download_button % 8, 'processing_info': {}, 'files': [(22, 'biom_table.biom (biom)', '1756512010', '1.1 MB')], 'is_from_analysis': True, 'summary': None, 'job': None, 'errored_summary_jobs': [] } self.assertEqual(obs, exp)
def test_artifact_summary_get_request(self): user = User('*****@*****.**') # Artifact w/o summary obs = artifact_summary_get_request(user, 1) exp_p_jobs = [[ '063e553b-327c-4818-ab4a-adfe58e49860', 'Split libraries FASTQ', 'queued', None, None ], [ 'bcc7ebcd-39c1-43e4-af2d-822e3589f14d', 'Split libraries', 'running', 'demultiplexing', None ]] exp_files = [ (1L, '1_s_G1_L001_sequences.fastq.gz (raw forward seqs)'), (2L, '1_s_G1_L001_sequences_barcodes.fastq.gz (raw barcodes)') ] exp = { 'name': 'Raw data 1', 'artifact_id': 1, 'visibility': 'private', 'editable': True, 'buttons': ('<button onclick="if (confirm(\'Are you sure you ' 'want to make public artifact id: 1?\')) { ' 'set_artifact_visibility(\'public\', 1) }" ' 'class="btn btn-primary btn-sm">Make public' '</button> <button onclick="if (confirm(\'Are you ' 'sure you want to revert to sandbox artifact id: ' '1?\')) { set_artifact_visibility(\'sandbox\', 1) ' '}" class="btn btn-primary btn-sm">Revert to ' 'sandbox</button>'), 'processing_parameters': {}, 'files': exp_files, 'summary': None, 'job': None, 'processing_jobs': exp_p_jobs, 'errored_jobs': [] } self.assertEqual(obs, exp) # Artifact with summary being generated job = ProcessingJob.create( User('*****@*****.**'), Parameters.load(Command(7), values_dict={'input_data': 1})) job._set_status('queued') obs = artifact_summary_get_request(user, 1) exp = { 'name': 'Raw data 1', 'artifact_id': 1, 'visibility': 'private', 'editable': True, 'buttons': ('<button onclick="if (confirm(\'Are you sure you ' 'want to make public artifact id: 1?\')) { ' 'set_artifact_visibility(\'public\', 1) }" ' 'class="btn btn-primary btn-sm">Make public' '</button> <button onclick="if (confirm(\'Are you ' 'sure you want to revert to sandbox artifact id: ' '1?\')) { set_artifact_visibility(\'sandbox\', 1) ' '}" class="btn btn-primary btn-sm">Revert to ' 'sandbox</button>'), 'processing_parameters': {}, 'files': exp_files, 'summary': None, 'job': [job.id, 'queued', None], 'processing_jobs': exp_p_jobs, 'errored_jobs': [] } self.assertEqual(obs, exp) # Artifact with summary fd, fp = mkstemp(suffix=".html") close(fd) with open(fp, 'w') as f: f.write('<b>HTML TEST - not important</b>\n') a = Artifact(1) a.set_html_summary(fp) self._files_to_remove.extend([fp, a.html_summary_fp[1]]) exp_files.append( (a.html_summary_fp[0], '%s (html summary)' % basename(a.html_summary_fp[1]))) exp_summary_path = relpath(a.html_summary_fp[1], qiita_config.base_data_dir) obs = artifact_summary_get_request(user, 1) exp = { 'name': 'Raw data 1', 'artifact_id': 1, 'visibility': 'private', 'editable': True, 'buttons': ('<button onclick="if (confirm(\'Are you sure you ' 'want to make public artifact id: 1?\')) { ' 'set_artifact_visibility(\'public\', 1) }" ' 'class="btn btn-primary btn-sm">Make public' '</button> <button onclick="if (confirm(\'Are you ' 'sure you want to revert to sandbox artifact id: ' '1?\')) { set_artifact_visibility(\'sandbox\', 1) ' '}" class="btn btn-primary btn-sm">Revert to ' 'sandbox</button>'), 'processing_parameters': {}, 'files': exp_files, 'summary': exp_summary_path, 'job': None, 'processing_jobs': exp_p_jobs, 'errored_jobs': [] } self.assertEqual(obs, exp) # No access demo_u = User('*****@*****.**') with self.assertRaises(QiitaHTTPError): obs = artifact_summary_get_request(demo_u, 1) # A non-owner/share user can't see the files a.visibility = 'public' obs = artifact_summary_get_request(demo_u, 1) exp = { 'name': 'Raw data 1', 'artifact_id': 1, 'visibility': 'public', 'editable': False, 'buttons': '', 'processing_parameters': {}, 'files': [], 'summary': exp_summary_path, 'job': None, 'processing_jobs': exp_p_jobs, 'errored_jobs': [] } self.assertEqual(obs, exp) # returnig to private a.visibility = 'private' # admin gets buttons obs = artifact_summary_get_request(User('*****@*****.**'), 2) exp_p_jobs = [[ 'd19f76ee-274e-4c1b-b3a2-a12d73507c55', 'Pick closed-reference OTUs', 'error', 'generating demux file', 'Error message' ]] exp_files = [(3L, '1_seqs.fna (preprocessed fasta)'), (4L, '1_seqs.qual (preprocessed fastq)'), (5L, '1_seqs.demux (preprocessed demux)')] exp = { 'name': 'Demultiplexed 1', 'artifact_id': 2, 'visibility': 'private', 'editable': True, 'buttons': ('<button onclick="if (confirm(\'Are you sure you ' 'want to make public artifact id: 2?\')) { ' 'set_artifact_visibility(\'public\', 2) }" ' 'class="btn btn-primary btn-sm">Make public' '</button> <button onclick="if (confirm(\'Are you ' 'sure you want to revert to sandbox artifact id: ' '2?\')) { set_artifact_visibility(\'sandbox\', 2) ' '}" class="btn btn-primary btn-sm">Revert to ' 'sandbox</button> <a class="btn btn-primary ' 'btn-sm" href="/ebi_submission/2"><span ' 'class="glyphicon glyphicon-export"></span> ' 'Submit to EBI</a> <a class="btn btn-primary ' 'btn-sm" href="/vamps/2"><span class="glyphicon ' 'glyphicon-export"></span> Submit to VAMPS</a>'), 'processing_parameters': { 'max_barcode_errors': 1.5, 'sequence_max_n': 0, 'max_bad_run_length': 3, 'phred_offset': u'auto', 'rev_comp': False, 'phred_quality_threshold': 3, 'input_data': 1, 'rev_comp_barcode': False, 'rev_comp_mapping_barcodes': False, 'min_per_read_length_fraction': 0.75, 'barcode_type': u'golay_12' }, 'files': exp_files, 'summary': None, 'job': None, 'processing_jobs': exp_p_jobs, 'errored_jobs': [] } self.assertEqual(obs, exp) # analysis artifact obs = artifact_summary_get_request(user, 8) exp = { 'name': 'noname', 'artifact_id': 8, 'visibility': 'sandbox', 'editable': True, 'buttons': '', 'processing_parameters': {}, 'files': [(27, 'biom_table.biom (biom)')], 'summary': None, 'job': None, 'processing_jobs': [], 'errored_jobs': [] } self.assertEqual(obs, exp)
def test_download_study(self): tmp_dir = mkdtemp() self._clean_up_files.append(tmp_dir) biom_fp = join(tmp_dir, 'otu_table.biom') smr_dir = join(tmp_dir, 'sortmerna_picked_otus') log_dir = join(smr_dir, 'seqs_otus.log') tgz = join(tmp_dir, 'sortmerna_picked_otus.tgz') with biom_open(biom_fp, 'w') as f: et.to_hdf5(f, "test") makedirs(smr_dir) with open(log_dir, 'w') as f: f.write('\n') with open(tgz, 'w') as f: f.write('\n') files_biom = [(biom_fp, 'biom'), (smr_dir, 'directory'), (tgz, 'tgz')] params = Parameters.from_default_params( next(Command(3).default_parameter_sets), {'input_data': 1}) a = Artifact.create(files_biom, "BIOM", parents=[Artifact(2)], processing_parameters=params) for x in a.filepaths: self._clean_up_files.append(x['fp']) response = self.get('/download_study_bioms/1') self.assertEqual(response.code, 200) exp = ('- 1256812 /protected/processed_data/' '1_study_1001_closed_reference_otu_table.biom processed_data/' '1_study_1001_closed_reference_otu_table.biom\n' '- [0-9]* /protected/templates/1_prep_1_[0-9]*-[0-9]*.txt ' 'mapping_files/4_mapping_file.txt\n' '- 1256812 /protected/processed_data/' '1_study_1001_closed_reference_otu_table.biom processed_data/' '1_study_1001_closed_reference_otu_table.biom\n' '- [0-9]* /protected/templates/1_prep_1_[0-9]*-[0-9]*.txt ' 'mapping_files/5_mapping_file.txt\n' '- 1256812 /protected/processed_data/1_study_1001_' 'closed_reference_otu_table_Silva.biom processed_data/' '1_study_1001_closed_reference_otu_table_Silva.biom\n' '- [0-9]* /protected/templates/1_prep_1_[0-9]*-[0-9]*.txt ' 'mapping_files/6_mapping_file.txt\n' '- 1093210 /protected/BIOM/7/biom_table.biom ' 'BIOM/7/biom_table.biom\n' '- [0-9]* /protected/templates/1_prep_2_[0-9]*-[0-9]*.txt ' 'mapping_files/7_mapping_file.txt\n' '- [0-9]* /protected/BIOM/{0}/otu_table.biom ' 'BIOM/{0}/otu_table.biom\n' '- 1 /protected/BIOM/10/sortmerna_picked_otus/seqs_otus.log ' 'BIOM/{0}/sortmerna_picked_otus/seqs_otus.log\n' '- [0-9]* /protected/templates/1_prep_1_[0-9]*-[0-9]*.txt ' 'mapping_files/{0}_mapping_file.txt\n'.format(a.id)) self.assertRegex(response.body.decode('ascii'), exp) response = self.get('/download_study_bioms/200') self.assertEqual(response.code, 405) # changing user so we can test the failures BaseHandler.get_current_user = Mock( return_value=User("*****@*****.**")) response = self.get('/download_study_bioms/1') self.assertEqual(response.code, 405) a.visibility = 'public' response = self.get('/download_study_bioms/1') # returning visibility a.visibility = 'private' self.assertEqual(response.code, 200) # we should have the same files than the previous test, except artifact # and mapping file 7: position 6 and 7; thus removing 6 twice exp = exp.split('\n') exp.pop(6) exp.pop(6) exp = '\n'.join(exp) self.assertRegex(response.body.decode('ascii'), exp)
job = ProcessingJob.create(user, params) job._set_error(info['message']) payload = {'job_id': job.id} r_client.set(key, dumps(payload)) else: # The key doesn't contain any information. Delete the key r_client.delete(key) with TRN: # Retrieve the Qiita plugin qiita_plugin = Software.from_name_and_version('Qiita', 'alpha') # Create the submit command for VAMPS command parameters = {'artifact': ['integer', None]} Command.create(qiita_plugin, "submit_to_VAMPS", "submits an artifact to VAMPS", parameters) # Create the copy artifact command parameters = { 'artifact': ['integer', None], 'prep_template': ['prep_template', None] } Command.create(qiita_plugin, "copy_artifact", "Creates a copy of an artifact", parameters) # Create the submit command for EBI command parameters = { 'artifact': ['integer', None], 'submission_type': ['choice:["ADD", "MODIFY"]', 'ADD'] } Command.create(qiita_plugin, "submit_to_EBI", "submits an artifact to EBI",
def render(self, study, prep_template, full_access, ena_terms, user_defined_terms): user = self.current_user is_local_request = is_localhost(self.request.headers['host']) template_fps = [] qiime_fps = [] # Unfortunately, both the prep template and the qiime mapping files # have the sample type. The way to differentiate them is if we have # the substring 'qiime' in the basename for id_, fp in prep_template.get_filepaths(): if 'qiime' in basename(fp): qiime_fps.append( download_link_or_path(is_local_request, fp, id_, 'Qiime mapping')) else: template_fps.append( download_link_or_path(is_local_request, fp, id_, 'Prep template')) # Since get_filepaths returns the paths sorted from newest to oldest, # the first in both list is the latest one current_template_fp = template_fps[0] current_qiime_fp = qiime_fps[0] if len(template_fps) > 1: show_old_templates = True old_templates = template_fps[1:] else: show_old_templates = False old_templates = None if len(qiime_fps) > 1: show_old_qiime_fps = True old_qiime_fps = qiime_fps[1:] else: show_old_qiime_fps = False old_qiime_fps = None filetypes = sorted(((ft, ft_id, fp_type_by_ft[ft]) for ft, ft_id in viewitems(get_artifact_types())), key=itemgetter(1)) files = [f for _, f in get_files_from_uploads_folders(str(study.id))] other_studies_rd = sorted(viewitems(_get_accessible_raw_data(user))) # A prep template can be modified if its status is sandbox is_editable = prep_template.status == 'sandbox' raw_data = prep_template.artifact preprocess_options = [] preprocessed_data = None show_preprocess_btn = True no_preprocess_msg = None preprocessing_status = 'Not processed' preprocessing_status_msg = "" if raw_data: raw_data_ft = raw_data.artifact_type # If the prep template has a raw data associated, it can be # preprocessed. Retrieve the pre-processing parameters # Hardcoding the command ids until the interface is refactored if raw_data_ft in ('SFF', 'FASTA'): param_iter = Command(2).default_parameter_sets elif raw_data_ft == 'FASTQ': param_iter = [ p for p in Command(1).default_parameter_sets if p.values['barcode_type'] != 'not-barcoded' ] elif raw_data_ft == 'per_sample_FASTQ': param_iter = [ p for p in Command(1).default_parameter_sets if p.values['barcode_type'] == 'not-barcoded' ] else: raise NotImplementedError( "Pre-processing of %s files currently not supported." % raw_data_ft) preprocess_options = [] for param in param_iter: text = ("<b>%s:</b> %s" % (k, v) for k, v in viewitems(param.values)) preprocess_options.append( (param.id, param.name, '<br>'.join(text))) preprocessed_data = raw_data.children # Check if the template have all the required columns for # preprocessing raw_data_files = raw_data.filepaths if len(raw_data_files) == 0: show_preprocess_btn = False no_preprocess_msg = ( "Preprocessing disabled because there are no files " "linked with the Raw Data") else: if prep_template.data_type() in TARGET_GENE_DATA_TYPES: raw_forward_fps = [ fp for _, fp, ftype in raw_data_files if ftype == 'raw_forward_seqs' ] key = ('demultiplex_multiple' if len(raw_forward_fps) > 1 else 'demultiplex') missing_cols = prep_template.check_restrictions( [PREP_TEMPLATE_COLUMNS_TARGET_GENE[key]]) if raw_data_ft == 'per_sample_FASTQ': show_preprocess_btn = 'run_prefix' not in missing_cols else: show_preprocess_btn = len(missing_cols) == 0 no_preprocess_msg = None if not show_preprocess_btn: no_preprocess_msg = ( "Preprocessing disabled due to missing columns in " "the prep template: %s" % ', '.join(missing_cols)) # Check the processing status preprocessing_status, preprocessing_status_msg = \ get_artifact_processing_status(raw_data) ebi_link = None if prep_template.is_submitted_to_ebi: ebi_link = EBI_LINKIFIER.format(study.ebi_study_accession) return self.render_string( "study_description_templates/prep_template_info_tab.html", raw_data=raw_data, current_template_fp=current_template_fp, current_qiime_fp=current_qiime_fp, show_old_templates=show_old_templates, old_templates=old_templates, show_old_qiime_fps=show_old_qiime_fps, old_qiime_fps=old_qiime_fps, filetypes=filetypes, files=files, other_studies_rd=other_studies_rd, prep_template=prep_template, study=study, ena_terms=ena_terms, user_defined_terms=user_defined_terms, investigation_type=prep_template.investigation_type, is_editable=is_editable, preprocess_options=preprocess_options, preprocessed_data=preprocessed_data, preprocessing_status=preprocessing_status, preprocessing_status_message=preprocessing_status_msg, show_preprocess_btn=show_preprocess_btn, no_preprocess_msg=no_preprocess_msg, ebi_link=ebi_link)