def post(self): """Retrieves the archiving information Returns ------- dict The contents of the analysis keyed by sample id Notes ----- Argument "path" must be the Qiita job_id which is used to infer the merging scheme. Argument "features" is a list of feature identifier, e.g. Deblur sequences. Feature identifiers not found in the archive won't be included in the return dictionary. """ job_id = self.get_argument('job_id') features = self.request.arguments['features'] ms = Archive.get_merging_scheme_from_job(ProcessingJob(job_id)) response = Archive.retrieve_feature_values(archive_merging_scheme=ms, features=features) self.write(response)
def execute(job_id): """Executes a job through the plugin system Parameters ---------- job_id : str The id of the job to execute """ # Create the new job job = ProcessingJob(job_id) job_dir = join(get_work_base_dir(), job.id) software = job.command.software plugin_start_script = software.start_script plugin_env_script = software.environment_script # Get the command to start the plugin cmd = '%s "%s" "%s" "%s" "%s" "%s"' % ( qiita_config.plugin_launcher, plugin_env_script, plugin_start_script, qiita_config.base_url, job.id, job_dir) # Start the plugin std_out, std_err, return_value = system_call(cmd) if return_value != 0: # Something wrong happened during the plugin start procedure job.status = 'error' log = LogEntry.create( 'Runtime', "Error starting plugin '%s':\nStd output:%s\nStd error:%s" % (software.name, std_out, std_err)) job.log = log
def prep_template_jobs_get_req(prep_id, user_id): """Returns graph of all artifacts created from the prep base artifact Parameters ---------- prep_id : int Prep template ID to get graph for user_id : str User making the request Returns ------- dict with the jobs information Notes ----- Nodes are identified by the corresponding Artifact ID. """ prep = PrepTemplate(int(prep_id)) access_error = check_access(prep.study_id, user_id) if access_error: return access_error job_info = r_client.get(PREP_TEMPLATE_KEY_FORMAT % prep_id) result = {} if job_info: job_info = defaultdict(lambda: '', loads(job_info)) job_id = job_info['job_id'] job = ProcessingJob(job_id) result[job.id] = {'status': job.status, 'step': job.step, 'error': job.log.msg if job.log else ""} return result
def job_ajax_get_req(job_id): """Returns the job information Parameters ---------- job_id : str The job id Returns ------- dict of objects A dictionary containing the job information {'status': str, 'message': str, 'job_id': str, 'job_status': str, 'job_step': str, 'job_parameters': dict of {str: str}} """ job = ProcessingJob(job_id) return { 'status': 'success', 'message': '', 'job_id': job.id, 'job_status': job.status, 'job_step': job.step, 'job_parameters': job.parameters.values }
def wait_for_processing_job(job_id): """Waits until a processing job is completed Parameters ---------- job_id : str Job id """ job = ProcessingJob(job_id) while job.status not in ('success', 'error'): sleep(0.05) sleep(0.05)
def test_artifact_post_request(self): # No access with self.assertRaises(QiitaHTTPError): artifact_post_req(User('*****@*****.**'), 1) artifact_post_req(User('*****@*****.**'), 2) # Wait until the job is completed wait_for_prep_information_job(1) # Check that the delete function has been actually called job = ProcessingJob(loads(r_client.get('prep_template_1'))['job_id']) self.assertEqual(job.status, 'error') self.assertIn('Cannot delete artifact 2', job.log.msg)
def display_template(self, study_id, msg): """Simple function to avoid duplication of code""" study_id = int(study_id) study = Study(study_id) user = self.current_user level = 'info' message = '' remote_url = '' remote_files = [] check_access(user, study, no_public=True, raise_error=True) job_info = r_client.get(UPLOAD_STUDY_FORMAT % study_id) if job_info: job_info = defaultdict(lambda: '', loads(job_info)) job_id = job_info['job_id'] job = ProcessingJob(job_id) job_status = job.status processing = job_status not in ('success', 'error') url = job.parameters.values['url'] if processing: if job.command.name == 'list_remote_files': message = 'Retrieving remote files: listing %s' % url else: message = 'Retrieving remote files: download %s' % url elif job_status == 'error': level = 'danger' message = job.log.msg.replace('\n', '</br>') # making errors nicer for users if 'No such file' in message: message = 'URL not valid: <i>%s</i>, please review.' % url else: remote_url = job_info['url'] remote_files = job_info['files'] level = job_info['alert_type'] message = job_info['alert_msg'].replace('\n', '</br>') # getting the ontologies self.render('upload.html', study_title=study.title, study_info=study.info, study_id=study_id, is_admin=user.level == 'admin', extensions=','.join(qiita_config.valid_upload_extension), max_upload_size=qiita_config.max_upload_size, level=level, message=message, remote_url=remote_url, remote_files=remote_files, files=get_files_from_uploads_folders(str(study_id)))
def wait_for_processing_job(job_id): """Waits until a processing job is completed Parameters ---------- job_id : str Job id """ job = ProcessingJob(job_id) while job.status not in ('success', 'error'): sleep(0.8) # this print is useful for debugging if job.status == 'error': print("==> %s: %s" % (job.id, job.log.msg)) sleep(0.8)
def patch(self): """Updates / stores feature information in the archive. Notes ----- Argument "path" must be the Qiita job_id which is used to infer the merging scheme. Argument "value" is a json string, i.e. result of a json.dump(obj) of a dictionary, keyed with feature identifiers. """ req_path = self.get_argument('path') req_value = self.get_argument('value') ms = Archive.get_merging_scheme_from_job(ProcessingJob(req_path)) self.write(Archive.insert_features(ms, loads(req_value)))
def test_patch(self): # Create a new job - through a workflow since that is the only way # of creating jobs in the interface exp_command = Command(1) json_str = ( '{"input_data": 1, "max_barcode_errors": 1.5, ' '"barcode_type": "golay_12", "max_bad_run_length": 3, ' '"rev_comp": false, "phred_quality_threshold": 3, ' '"rev_comp_barcode": false, "rev_comp_mapping_barcodes": false, ' '"min_per_read_length_fraction": 0.75, "sequence_max_n": 0}') exp_params = Parameters.load(exp_command, json_str=json_str) exp_user = User('*****@*****.**') name = "Test processing workflow" # tests success wf = ProcessingWorkflow.from_scratch(exp_user, exp_params, name=name, force=True) graph = wf.graph nodes = graph.nodes() job_id = nodes[0].id response = self.patch('/study/process/job/', { 'op': 'remove', 'path': job_id }) self.assertEqual(response.code, 200) exp = { 'status': 'error', 'message': "Can't delete job %s. It is 'in_construction' " "status. Please use /study/process/workflow/" % job_id } self.assertEqual(loads(response.body), exp) # Test success ProcessingJob(job_id)._set_error('Killed for testing') response = self.patch('/study/process/job/', { 'op': 'remove', 'path': job_id }) self.assertEqual(response.code, 200) exp = {'status': 'success', 'message': ''} self.assertEqual(loads(response.body), exp)
def get(self): user = self.current_user is_local_request = is_localhost(self.request.headers['host']) uanalyses = user.shared_analyses | user.private_analyses user_analysis_ids = set([a.id for a in uanalyses]) panalyses = Analysis.get_by_status('public') public_analysis_ids = set([a.id for a in panalyses]) public_analysis_ids = public_analysis_ids - user_analysis_ids user_analyses = generate_analysis_list(user_analysis_ids) public_analyses = generate_analysis_list(public_analysis_ids, True) dlop = partial(download_link_or_path, is_local_request) messages = {'info': '', 'danger': ''} for analysis_id in user_analysis_ids: job_info = r_client.get('analysis_delete_%d' % analysis_id) if job_info: job_info = defaultdict(lambda: '', loads(job_info)) job_id = job_info['job_id'] job = ProcessingJob(job_id) job_status = job.status processing = job_status not in ('success', 'error') if processing: messages['info'] += ('Analysis %s is being deleted<br/>' % analysis_id) elif job_status == 'error': messages['danger'] += (job.log.msg.replace('\n', '<br/>') + '<br/>') else: if job_info['alert_type'] not in messages: messages[job_info['alert_type']] = [] messages[job_info['alert_type']] += ( job.log.msg.replace('\n', '<br/>') + '<br/>') self.render("list_analyses.html", user_analyses=user_analyses, public_analyses=public_analyses, messages=messages, dlop=dlop)
def job_ajax_get_req(job_id): """Returns the job information Parameters ---------- job_id : str The job id Returns ------- dict of objects A dictionary containing the job information {'status': str, 'message': str, 'job_id': str, 'job_status': str, 'job_step': str, 'job_parameters': dict of {str: str}} """ job = ProcessingJob(job_id) cmd = job.command sw = cmd.software job_status = job.status job_error = job.log.msg if job.log is not None else None return { 'status': 'success', 'message': '', 'job_id': job.id, 'job_external_id': job.external_id, 'job_status': job_status, 'job_step': job.step, 'job_parameters': job.parameters.values, 'job_error': job_error, 'command': cmd.name, 'command_description': cmd.description, 'software': sw.name, 'software_version': sw.version }
def get_sample_template_processing_status(st_id): # Initialize variables here processing = False alert_type = '' alert_msg = '' job_info = r_client.get(SAMPLE_TEMPLATE_KEY_FORMAT % st_id) if job_info: job_info = defaultdict(lambda: '', loads(job_info)) job_id = job_info['job_id'] job = ProcessingJob(job_id) job_status = job.status processing = job_status not in ('success', 'error') if processing: alert_type = 'info' alert_msg = 'This sample template is currently being processed' elif job_status == 'error': alert_type = 'danger' alert_msg = job.log.msg.replace('\n', '</br>') else: alert_type = job_info['alert_type'] alert_msg = job_info['alert_msg'].replace('\n', '</br>') return processing, alert_type, alert_msg
def workflow_handler_patch_req(req_op, req_path, req_value=None, req_from=None): """Patches a workflow Parameters ---------- req_op : str The operation to perform on the workflow req_path : str Path parameter with the workflow to patch req_value : str, optional The value that needs to be modified req_from : str, optional The original path of the element Returns ------- dict of {str: str} A dictionary of the form: {'status': str, 'message': str} in which status is the status of the request ('error' or 'success') and message is a human readable string with the error message in case that status is 'error'. """ if req_op == 'add': req_path = [v for v in req_path.split('/') if v] if len(req_path) != 1: return {'status': 'error', 'message': 'Incorrect path parameter'} req_path = req_path[0] try: wf = ProcessingWorkflow(req_path) except QiitaDBUnknownIDError: return { 'status': 'error', 'message': 'Workflow %s does not exist' % req_path } req_value = loads(req_value) dflt_params = DefaultParameters(req_value['dflt_params']) req_params = req_value.get('req_params', None) opt_params = req_value.get('opt_params', None) connections = { ProcessingJob(k): v for k, v in req_value['connections'].items() } job = wf.add(dflt_params, connections=connections, req_params=req_params, opt_params=opt_params) job_cmd = job.command return { 'status': 'success', 'message': '', 'job': { 'id': job.id, 'inputs': list(req_value['connections'].keys()), 'label': job_cmd.name, 'outputs': job_cmd.outputs } } elif req_op == 'remove': req_path = [v for v in req_path.split('/') if v] if len(req_path) != 2: return {'status': 'error', 'message': 'Incorrect path parameter'} wf_id = req_path[0] job_id = req_path[1] wf = ProcessingWorkflow(wf_id) job = ProcessingJob(job_id) wf.remove(job, cascade=True) return {'status': 'success', 'message': ''} else: return { 'status': 'error', 'message': 'Operation "%s" not supported. Current supported ' 'operations: add' % req_op }
def test_complete_job(self): # Complete success pt = npt.assert_warns(QiitaDBWarning, PrepTemplate.create, pd.DataFrame({'new_col': { '1.SKD6.640190': 1 }}), Study(1), '16S') c_job = ProcessingJob.create( User('*****@*****.**'), Parameters.load(Command.get_validator('BIOM'), values_dict={ 'template': pt.id, 'files': dumps({'BIOM': ['file']}), 'artifact_type': 'BIOM' }), True) c_job._set_status('running') fd, fp = mkstemp(suffix='_table.biom') close(fd) with open(fp, 'w') as f: f.write('\n') self._clean_up_files.append(fp) exp_artifact_count = get_count('qiita.artifact') + 1 payload = dumps({ 'success': True, 'error': '', 'artifacts': { 'OTU table': { 'filepaths': [(fp, 'biom')], 'artifact_type': 'BIOM' } } }) job = self._create_job('complete_job', { 'job_id': c_job.id, 'payload': payload }) private_task(job.id) self.assertEqual(job.status, 'success') self.assertEqual(c_job.status, 'success') self.assertEqual(get_count('qiita.artifact'), exp_artifact_count) # Complete job error payload = dumps({'success': False, 'error': 'Job failure'}) job = self._create_job('complete_job', { 'job_id': 'bcc7ebcd-39c1-43e4-af2d-822e3589f14d', 'payload': payload }) private_task(job.id) self.assertEqual(job.status, 'success') c_job = ProcessingJob('bcc7ebcd-39c1-43e4-af2d-822e3589f14d') self.assertEqual(c_job.status, 'error') self.assertEqual(c_job.log, LogEntry.newest_records(numrecords=1)[0]) self.assertEqual(c_job.log.msg, 'Job failure') # Complete internal error pt = npt.assert_warns(QiitaDBWarning, PrepTemplate.create, pd.DataFrame({'new_col': { '1.SKD6.640190': 1 }}), Study(1), '16S') c_job = ProcessingJob.create( User('*****@*****.**'), Parameters.load(Command.get_validator('BIOM'), values_dict={ 'template': pt.id, 'files': dumps({'BIOM': ['file']}), 'artifact_type': 'BIOM' }), True) c_job._set_status('running') fp = '/surprised/if/this/path/exists.biom' payload = dumps({ 'success': True, 'error': '', 'artifacts': { 'OTU table': { 'filepaths': [(fp, 'biom')], 'artifact_type': 'BIOM' } } }) job = self._create_job('complete_job', { 'job_id': c_job.id, 'payload': payload }) private_task(job.id) self.assertEqual(job.status, 'success') self.assertEqual(c_job.status, 'error') self.assertIn('No such file or directory', c_job.log.msg)
def test_job_ajax_patch_req(self): # Create a new job - through a workflow since that is the only way # of creating jobs in the interface exp_command = Command(1) json_str = ( '{"input_data": 1, "max_barcode_errors": 1.5, ' '"barcode_type": "golay_12", "max_bad_run_length": 3, ' '"rev_comp": false, "phred_quality_threshold": 3, ' '"rev_comp_barcode": false, "rev_comp_mapping_barcodes": false, ' '"min_per_read_length_fraction": 0.75, "sequence_max_n": 0}') exp_params = Parameters.load(exp_command, json_str=json_str) exp_user = User('*****@*****.**') name = "Test processing workflow" # tests success wf = ProcessingWorkflow.from_scratch(exp_user, exp_params, name=name, force=True) graph = wf.graph nodes = list(graph.nodes()) job_id = nodes[0].id # Incorrect path parameter obs = job_ajax_patch_req('remove', '/%s/somethingelse' % job_id) exp = { 'status': 'error', 'message': 'Incorrect path parameter: missing job id' } self.assertEqual(obs, exp) obs = job_ajax_patch_req('remove', '/') exp = { 'status': 'error', 'message': 'Incorrect path parameter: missing job id' } self.assertEqual(obs, exp) # Job id is not like a job id obs = job_ajax_patch_req('remove', '/notAJobId') exp = { 'status': 'error', 'message': 'Incorrect path parameter: ' 'notAJobId is not a recognized job id' } self.assertEqual(obs, exp) # Job doesn't exist obs = job_ajax_patch_req('remove', '/6d368e16-2242-4cf8-87b4-a5dc40bc890b') exp = { 'status': 'error', 'message': 'Incorrect path parameter: ' '6d368e16-2242-4cf8-87b4-a5dc40bc890b is not a ' 'recognized job id' } self.assertEqual(obs, exp) # in_construction job obs = job_ajax_patch_req('remove', '/%s' % job_id) exp = { 'status': 'error', 'message': "Can't delete job %s. It is 'in_construction' " "status. Please use /study/process/workflow/" % job_id } self.assertEqual(obs, exp) # job status != 'error' job = ProcessingJob(job_id) job._set_status('queued') obs = job_ajax_patch_req('remove', '/%s' % job_id) exp = { 'status': 'error', 'message': 'Only jobs in "error" status can be deleted.' } self.assertEqual(obs, exp) # Operation not supported job._set_status('queued') obs = job_ajax_patch_req('add', '/%s' % job_id) exp = { 'status': 'error', 'message': 'Operation "add" not supported. Current supported ' 'operations: remove' } self.assertEqual(obs, exp) # Test success job._set_error('Killed for testing') obs = job_ajax_patch_req('remove', '/%s' % job_id) exp = {'status': 'success', 'message': ''} self.assertEqual(obs, exp)
def prep_template_ajax_get_req(user_id, prep_id): """Returns the prep tempalte information needed for the AJAX handler Parameters ---------- user_id : str The user id prep_id : int The prep template id Returns ------- dict of {str: object} A dictionary with the following keys: - status: str, whether the request is successful or not - message: str, if the request is unsuccessful, a human readable error - name: str, the name of the prep template - files: list of str, the files available to update the prep template - download_prep: int, the filepath_id of the prep file - download_qiime, int, the filepath_id of the qiime mapping file - num_samples: int, the number of samples present in the template - num_columns: int, the number of columns present in the template - investigation_type: str, the investigation type of the template - ontology: str, dict of {str, list of str} containing the information of the ENA ontology - artifact_attached: bool, whether the template has an artifact attached - study_id: int, the study id of the template """ # Currently there is no name attribute, but it will be soon name = "Prep information %d" % prep_id pt = PrepTemplate(prep_id) job_info = r_client.get(PREP_TEMPLATE_KEY_FORMAT % prep_id) if job_info: job_info = loads(job_info) job_id = job_info['job_id'] if job_id: if job_info['is_qiita_job']: job = ProcessingJob(job_id) processing = job.status in ('queued', 'running') success = job.status == 'success' alert_type = 'info' if processing or success else 'danger' alert_msg = (job.log.msg.replace('\n', '</br>') if job.log is not None else "") else: redis_info = loads(r_client.get(job_id)) processing = redis_info['status_msg'] == 'Running' success = redis_info['status_msg'] == 'Success' if redis_info['return'] is not None: alert_type = redis_info['return']['status'] alert_msg = redis_info['return']['message'].replace( '\n', '</br>') else: alert_type = 'info' alert_msg = '' if processing: alert_type = 'info' alert_msg = 'This prep template is currently being updated' elif success: payload = { 'job_id': None, 'status': alert_type, 'message': alert_msg, 'is_qiita_job': job_info['is_qiita_job'] } r_client.set(PREP_TEMPLATE_KEY_FORMAT % prep_id, dumps(payload)) else: processing = False alert_type = job_info['status'] alert_msg = job_info['message'].replace('\n', '</br>') else: processing = False alert_type = '' alert_msg = '' artifact_attached = pt.artifact is not None study_id = pt.study_id files = [ f for _, f in get_files_from_uploads_folders(study_id) if f.endswith(('.txt', '.tsv')) ] # The call to list is needed because keys is an iterator num_samples = len(list(pt.keys())) num_columns = len(pt.categories()) investigation_type = pt.investigation_type # Retrieve the information to download the prep template and QIIME # mapping file. See issue https://github.com/biocore/qiita/issues/1675 download_prep = [] download_qiime = [] for fp_id, fp in pt.get_filepaths(): if 'qiime' in basename(fp): download_qiime.append(fp_id) else: download_prep.append(fp_id) download_prep = download_prep[0] download_qiime = download_qiime[0] ontology = _get_ENA_ontology() editable = Study(study_id).can_edit(User(user_id)) and not processing return { 'status': 'success', 'message': '', 'name': name, 'files': files, 'download_prep': download_prep, 'download_qiime': download_qiime, 'num_samples': num_samples, 'num_columns': num_columns, 'investigation_type': investigation_type, 'ontology': ontology, 'artifact_attached': artifact_attached, 'study_id': study_id, 'editable': editable, 'data_type': pt.data_type(), 'alert_type': alert_type, 'is_submitted_to_ebi': pt.is_submitted_to_ebi, 'alert_message': alert_msg }
def study_get_req(study_id, user_id): """Returns information available for the given study Parameters ---------- study_id : int Study id to get prep template info for user_id : str User requesting the info Returns ------- dict Data types information in the form {'status': status, 'message': message, 'info': dict of objects status can be success, warning, or error depending on result message has the warnings or errors info contains study information seperated by data type, in the form {col_name: value, ...} with value being a string, int, or list of strings or ints """ access_error = check_access(study_id, user_id) if access_error: return access_error # Can only pass ids over API, so need to instantiate object study = Study(study_id) study_info = study.info # Add needed info that is not part of the initial info pull study_info['publication_doi'] = [] study_info['publication_pid'] = [] for pub, is_doi in study.publications: if is_doi: study_info['publication_doi'].append(pub) else: study_info['publication_pid'].append(pub) study_info['study_id'] = study.id study_info['study_title'] = study.title study_info['shared_with'] = [s.id for s in study.shared_with] study_info['status'] = study.status study_info['ebi_study_accession'] = study.ebi_study_accession study_info['ebi_submission_status'] = study.ebi_submission_status study_info['public_raw_download'] = study.public_raw_download study_info['notes'] = study.notes # Clean up StudyPerson objects to string for display pi = study_info['principal_investigator'] study_info['principal_investigator'] = { 'name': pi.name, 'email': pi.email, 'affiliation': pi.affiliation } lab_person = study_info['lab_person'] if lab_person: study_info['lab_person'] = { 'name': lab_person.name, 'email': lab_person.email, 'affiliation': lab_person.affiliation } samples = study.sample_template study_info['num_samples'] = 0 if samples is None else len(list(samples)) study_info['owner'] = study.owner.id # Study.has_access no_public=True, will return True only if the user_id is # the owner of the study or if the study is shared with the user_id; this # with study.public_raw_download will define has_access_to_raw_data study_info['has_access_to_raw_data'] = study.has_access( User(user_id), True) or study.public_raw_download study_info['show_biom_download_button'] = 'BIOM' in [ a.artifact_type for a in study.artifacts() ] study_info['show_raw_download_button'] = any( [True for pt in study.prep_templates() if pt.artifact is not None]) # getting study processing status from redis processing = False study_info['level'] = '' study_info['message'] = '' job_info = r_client.get(STUDY_KEY_FORMAT % study_id) if job_info: job_info = defaultdict(lambda: '', loads(job_info)) job_id = job_info['job_id'] job = ProcessingJob(job_id) job_status = job.status processing = job_status not in ('success', 'error') if processing: study_info['level'] = 'info' study_info['message'] = 'This study is currently being processed' elif job_status == 'error': study_info['level'] = 'danger' study_info['message'] = job.log.msg.replace('\n', '</br>') else: study_info['level'] = job_info['alert_type'] study_info['message'] = job_info['alert_msg'].replace( '\n', '</br>') return { 'status': 'success', 'message': '', 'study_info': study_info, 'editable': study.can_edit(User(user_id)) }
def prep_template_ajax_get_req(user_id, prep_id): """Returns the prep tempalte information needed for the AJAX handler Parameters ---------- user_id : str The user id prep_id : int The prep template id Returns ------- dict of {str: object} A dictionary with the following keys: - status: str, whether the request is successful or not - message: str, if the request is unsuccessful, a human readable error - name: str, the name of the prep template - files: list of str, the files available to update the prep template - download_prep: int, the filepath_id of the prep file - download_qiime, int, the filepath_id of the qiime mapping file - num_samples: int, the number of samples present in the template - num_columns: int, the number of columns present in the template - investigation_type: str, the investigation type of the template - ontology: str, dict of {str, list of str} containing the information of the ENA ontology - artifact_attached: bool, whether the template has an artifact attached - study_id: int, the study id of the template """ pt = PrepTemplate(prep_id) name = pt.name # Initialize variables here processing = False alert_type = '' alert_msg = '' job_info = r_client.get(PREP_TEMPLATE_KEY_FORMAT % prep_id) if job_info: job_info = defaultdict(lambda: '', loads(job_info)) job_id = job_info['job_id'] job = ProcessingJob(job_id) job_status = job.status processing = job_status not in ('success', 'error') if processing: alert_type = 'info' alert_msg = 'This prep template is currently being updated' elif job_status == 'error': alert_type = 'danger' alert_msg = job.log.msg.replace('\n', '</br>') else: alert_type = job_info['alert_type'] alert_msg = job_info['alert_msg'].replace('\n', '</br>') artifact_attached = pt.artifact is not None study_id = pt.study_id files = [f for _, f, _ in get_files_from_uploads_folders(study_id) if f.endswith(('.txt', '.tsv'))] # The call to list is needed because keys is an iterator num_samples = len(list(pt.keys())) num_columns = len(pt.categories()) investigation_type = pt.investigation_type download_prep_id = None download_qiime_id = None other_filepaths = [] for fp_id, fp in pt.get_filepaths(): fp = basename(fp) if 'qiime' in fp: if download_qiime_id is None: download_qiime_id = fp_id else: if download_prep_id is None: download_prep_id = fp_id else: other_filepaths.append(fp) ontology = _get_ENA_ontology() editable = Study(study_id).can_edit(User(user_id)) and not processing return {'status': 'success', 'message': '', 'name': name, 'files': files, 'download_prep_id': download_prep_id, 'download_qiime_id': download_qiime_id, 'other_filepaths': other_filepaths, 'num_samples': num_samples, 'num_columns': num_columns, 'investigation_type': investigation_type, 'ontology': ontology, 'artifact_attached': artifact_attached, 'study_id': study_id, 'editable': editable, 'data_type': pt.data_type(), 'alert_type': alert_type, 'is_submitted_to_ebi': pt.is_submitted_to_ebi, 'alert_message': alert_msg}
def job_ajax_patch_req(req_op, req_path, req_value=None, req_from=None): """Patches a job Parameters ---------- req_op : str The operation to perform on the job req_path : str Path parameter with the job to patch req_value : str, optional The value that needs to be modified req_from : str, optional The original path of the element Returns ------- dict of {str: str} A dictionary of the form: {'status': str, 'message': str} in which status is the status of the request ('error' or 'success') and message is a human readable string with the error message in case that status is 'error'. """ if req_op == 'remove': req_path = [v for v in req_path.split('/') if v] if len(req_path) != 1: return { 'status': 'error', 'message': 'Incorrect path parameter: missing job id' } # We have ensured that we only have one element on req_path job_id = req_path[0] try: job = ProcessingJob(job_id) except QiitaDBUnknownIDError: return { 'status': 'error', 'message': 'Incorrect path parameter: ' '%s is not a recognized job id' % job_id } except Exception as e: e = str(e) if "invalid input syntax for uuid" in e: return { 'status': 'error', 'message': 'Incorrect path parameter: ' '%s is not a recognized job id' % job_id } else: return { 'status': 'error', 'message': 'An error occured while accessing the ' 'job: %s' % e } job_status = job.status if job_status == 'in_construction': # A job that is in construction is in a workflow. Use the methods # defined for workflows to keep everything consistent. This message # should never be presented to the user, but rather to the # developer if it makes a mistake during changes in the interface return { 'status': 'error', 'message': "Can't delete job %s. It is 'in_construction' " "status. Please use /study/process/workflow/" % job_id } elif job_status == 'error': # When the job is in error status, we just need to hide it job.hide() return {'status': 'success', 'message': ''} else: # In any other state, we currently fail. Adding the else here # because it can be useful to have it for fixing issue #2307 return { 'status': 'error', 'message': 'Only jobs in "error" status can be deleted.' } else: return { 'status': 'error', 'message': 'Operation "%s" not supported. Current supported ' 'operations: remove' % req_op }
def correct_redis_data(key, cmd, values_dict, user): """Corrects the data stored in the redis DB Parameters ---------- key: str The redis key to fix cmd : qiita_db.software.Command Command to use to create the processing job values_dict : dict Dictionary used to instantiate the parameters of the command user : qiita_db.user. User The user that will own the job """ info = r_client.get(key) if info: info = loads(info) if info['job_id'] is not None: if 'is_qiita_job' in info: if info['is_qiita_job']: try: job = ProcessingJob(info['job_id']) payload = { 'job_id': info['job_id'], 'alert_type': info['status'], 'alert_msg': info['alert_msg'] } r_client.set(key, dumps(payload)) except (QiitaDBUnknownIDError, KeyError): # We shomehow lost the information of this job # Simply delete the key r_client.delete(key) else: # These jobs don't contain any information on the live # dump. We can safely delete the key r_client.delete(key) else: # These jobs don't contain any information on the live # dump. We can safely delete the key r_client.delete(key) else: # Job is null, we have the information here if info['status'] == 'success': # In the success case no information is stored. We can # safely delete the key r_client.delete(key) elif info['status'] == 'warning': # In case of warning the key message stores the warning # message. We need to create a new job, mark it as # successful and store the error message as expected by # the new structure params = Parameters.load(cmd, values_dict=values_dict) job = ProcessingJob.create(user, params) job._set_status('success') payload = { 'job_id': job.id, 'alert_type': 'warning', 'alert_msg': info['message'] } r_client.set(key, dumps(payload)) else: # The status is error. The key message stores the error # message. We need to create a new job and mark it as # failed with the given error message params = Parameters.load(cmd, values_dict=values_dict) job = ProcessingJob.create(user, params) job._set_error(info['message']) payload = {'job_id': job.id} r_client.set(key, dumps(payload)) else: # The key doesn't contain any information. Delete the key r_client.delete(key)