def post(self, preprocessed_data_id): user = self.current_user # make sure user is admin and can therefore actually submit to EBI if user.level != 'admin': raise HTTPError(403, reason="User %s cannot submit to EBI!" % user.id) submission_type = self.get_argument('submission_type') if submission_type not in ['ADD', 'MODIFY']: raise HTTPError(403, reason="User: %s, %s is not a recognized " "submission type" % (user.id, submission_type)) study = Artifact(preprocessed_data_id).study state = study.ebi_submission_status if state == 'submitting': message = "Cannot resubmit! Current state is: %s" % state self.display_template(preprocessed_data_id, message, 'danger') else: qiita_plugin = Software.from_name_and_version('Qiita', 'alpha') cmd = qiita_plugin.get_command('submit_to_EBI') params = Parameters.load( cmd, values_dict={'artifact': preprocessed_data_id, 'submission_type': submission_type}) job = ProcessingJob.create(user, params, True) r_client.set('ebi_submission_%s' % preprocessed_data_id, dumps({'job_id': job.id, 'is_qiita_job': True})) job.submit() level = 'success' message = 'EBI submission started. Job id: %s' % job.id self.redirect("%s/study/description/%d?level=%s&message=%s" % ( qiita_config.portal_dir, study.id, level, url_escape(message)))
def study_delete_req(study_id, user_id): """Delete a given study Parameters ---------- study_id : int Study id to delete user_id : str User requesting the deletion Returns ------- dict Status of deletion, in the format {status: status, message: message} """ access_error = check_access(study_id, user_id) if access_error: return access_error qiita_plugin = Software.from_name_and_version('Qiita', 'alpha') cmd = qiita_plugin.get_command('delete_study') params = Parameters.load(cmd, values_dict={'study': study_id}) job = ProcessingJob.create(User(user_id), params, True) # Store the job id attaching it to the sample template id r_client.set(STUDY_KEY_FORMAT % study_id, dumps({'job_id': job.id})) job.submit() return {'status': 'success', 'message': ''}
def post(self, preprocessed_data_id): user = self.current_user # make sure user is admin and can therefore actually submit to EBI if user.level != 'admin': raise HTTPError(403, reason="User %s cannot submit to EBI!" % user.id) submission_type = self.get_argument('submission_type') if submission_type not in ['ADD', 'MODIFY']: raise HTTPError(403, reason="User: %s, %s is not a recognized " "submission type" % (user.id, submission_type)) study = Artifact(preprocessed_data_id).study state = study.ebi_submission_status if state == 'submitting': message = "Cannot resubmit! Current state is: %s" % state self.display_template(preprocessed_data_id, message, 'danger') else: qiita_plugin = Software.from_name_and_version('Qiita', 'alpha') cmd = qiita_plugin.get_command('submit_to_EBI') params = Parameters.load( cmd, values_dict={'artifact': preprocessed_data_id, 'submission_type': submission_type}) job = ProcessingJob.create(user, params, True) r_client.set('ebi_submission_%s' % preprocessed_data_id, dumps({'job_id': job.id, 'is_qiita_job': True})) job.submit() level = 'success' message = 'EBI submission started. Job id: %s' % job.id self.redirect("%s/study/description/%d?level=%s&message=%s" % ( qiita_config.portal_dir, study.id, level, url_escape(message)))
def create_sample_template(job): """Creates a sample template Parameters ---------- job : qiita_db.processing_job.ProcessingJob The processing job performing the task """ with qdb.sql_connection.TRN: params = job.parameters.values fp = params['fp'] study = qdb.study.Study(int(params['study_id'])) is_mapping_file = params['is_mapping_file'] data_type = params['data_type'] with warnings.catch_warnings(record=True) as warns: if is_mapping_file: create_templates_from_qiime_mapping_file(fp, study, data_type) else: qdb.metadata_template.sample_template.SampleTemplate.create( qdb.metadata_template.util.load_template_to_dataframe(fp), study) remove(fp) if warns: msg = '\n'.join(set(str(w) for w in warns)) r_client.set( "sample_template_%s" % study.id, dumps({ 'job_id': job.id, 'alert_type': 'warning', 'alert_msg': msg })) job._set_status('success')
def create_sample_template(job): """Creates a sample template Parameters ---------- job : qiita_db.processing_job.ProcessingJob The processing job performing the task """ with qdb.sql_connection.TRN: params = job.parameters.values fp = params['fp'] study = qdb.study.Study(int(params['study_id'])) is_mapping_file = params['is_mapping_file'] data_type = params['data_type'] with warnings.catch_warnings(record=True) as warns: if is_mapping_file: create_templates_from_qiime_mapping_file(fp, study, data_type) else: qdb.metadata_template.sample_template.SampleTemplate.create( qdb.metadata_template.util.load_template_to_dataframe(fp), study) remove(fp) if warns: msg = '\n'.join(set(str(w.message) for w in warns)) r_client.set("sample_template_%s" % study.id, dumps({'job_id': job.id, 'alert_type': 'warning', 'alert_msg': msg})) job._set_status('success')
def artifact_post_req(user, artifact_id): """Deletes the artifact Parameters ---------- user : qiita_db.user.User The user requesting the action artifact_id : int Id of the artifact being deleted """ artifact_id = int(artifact_id) artifact = Artifact(artifact_id) check_artifact_access(user, artifact) analysis = artifact.analysis if analysis: # Do something when deleting in the analysis part to keep track of it redis_key = "analysis_%s" % analysis.id else: pt_id = artifact.prep_templates[0].id redis_key = PREP_TEMPLATE_KEY_FORMAT % pt_id qiita_plugin = Software.from_name_and_version('Qiita', 'alpha') cmd = qiita_plugin.get_command('delete_artifact') params = Parameters.load(cmd, values_dict={'artifact': artifact_id}) job = ProcessingJob.create(user, params) r_client.set(redis_key, dumps({'job_id': job.id, 'is_qiita_job': True})) job.submit()
def update_prep_template(job): """Updates a prep template Parameters ---------- job : qiita_db.processing_job.ProcessingJob The processing job performing the task """ with qdb.sql_connection.TRN: param_vals = job.parameters.values prep_id = param_vals['prep_template'] fp = param_vals['template_fp'] prep = qdb.metadata_template.prep_template.PrepTemplate(prep_id) with warnings.catch_warnings(record=True) as warns: df = qdb.metadata_template.util.load_template_to_dataframe(fp) prep.extend_and_update(df) remove(fp) # Join all the warning messages into one. Note that this info # will be ignored if an exception is raised if warns: msg = '\n'.join(set(str(w) for w in warns)) r_client.set( "prep_template_%s" % prep_id, dumps({ 'job_id': job.id, 'alert_type': 'warning', 'alert_msg': msg })) job._set_status('success')
def list_remote_files(job): """Lists valid study files on a remote server Parameters ---------- job : qiita_db.processing_job.ProcessingJob The processing job performing the task """ with qdb.sql_connection.TRN: url = job.parameters.values['url'] private_key = job.parameters.values['private_key'] study_id = job.parameters.values['study_id'] try: files = list_remote(url, private_key) r_client.set("upload_study_%s" % study_id, dumps({ 'job_id': job.id, 'url': url, 'files': files })) except Exception: job._set_error(traceback.format_exception(*exc_info())) else: job._set_status('success') finally: # making sure to always delete the key so Qiita never keeps it remove(private_key)
def study_delete_req(study_id, user_id): """Delete a given study Parameters ---------- study_id : int Study id to delete user_id : str User requesting the deletion Returns ------- dict Status of deletion, in the format {status: status, message: message} """ access_error = check_access(study_id, user_id) if access_error: return access_error qiita_plugin = Software.from_name_and_version('Qiita', 'alpha') cmd = qiita_plugin.get_command('delete_study') params = Parameters.load(cmd, values_dict={'study': study_id}) job = ProcessingJob.create(User(user_id), params, True) # Store the job id attaching it to the sample template id r_client.set(STUDY_KEY_FORMAT % study_id, dumps({'job_id': job.id})) job.submit() return {'status': 'success', 'message': ''}
def update_prep_template(job): """Updates a prep template Parameters ---------- job : qiita_db.processing_job.ProcessingJob The processing job performing the task """ with qdb.sql_connection.TRN: param_vals = job.parameters.values prep_id = param_vals['prep_template'] fp = param_vals['template_fp'] prep = qdb.metadata_template.prep_template.PrepTemplate(prep_id) with warnings.catch_warnings(record=True) as warns: df = qdb.metadata_template.util.load_template_to_dataframe(fp) prep.extend_and_update(df) remove(fp) # Join all the warning messages into one. Note that this info # will be ignored if an exception is raised if warns: msg = '\n'.join(set(str(w.message) for w in warns)) r_client.set("prep_template_%s" % prep_id, dumps({'job_id': job.id, 'alert_type': 'warning', 'alert_msg': msg})) job._set_status('success')
def artifact_post_req(user, artifact_id): """Deletes the artifact Parameters ---------- user : qiita_db.user.User The user requesting the action artifact_id : int Id of the artifact being deleted """ artifact_id = int(artifact_id) artifact = Artifact(artifact_id) check_artifact_access(user, artifact) analysis = artifact.analysis if analysis: # Do something when deleting in the analysis part to keep track of it redis_key = "analysis_%s" % analysis.id else: pt_id = artifact.prep_templates[0].id redis_key = PREP_TEMPLATE_KEY_FORMAT % pt_id qiita_plugin = Software.from_name_and_version('Qiita', 'alpha') cmd = qiita_plugin.get_command('delete_artifact') params = Parameters.load(cmd, values_dict={'artifact': artifact_id}) job = ProcessingJob.create(user, params, True) r_client.set(redis_key, dumps({'job_id': job.id, 'is_qiita_job': True})) job.submit() return {'job': job.id}
def sample_template_put_req(study_id, user_id, sample_template): """Updates a sample template using the given file Parameters ---------- study_id : int The current study object id user_id : str The current user object id sample_template : str filename to use for updating Returns ------- dict results dictonary in the format {'status': status, 'message': msg, 'file': sample_template} status can be success, warning, or error depending on result message has the warnings or errors file has the file name """ exists = _check_sample_template_exists(int(study_id)) if exists['status'] != 'success': return exists access_error = check_access(int(study_id), user_id) if access_error: return access_error fp_rsp = check_fp(study_id, sample_template) if fp_rsp['status'] != 'success': # Unknown filepath, so return the error message return fp_rsp fp_rsp = fp_rsp['file'] msg = '' status = 'success' # Offload the update of the sample template to the cluster qiita_plugin = Software.from_name_and_version('Qiita', 'alpha') cmd = qiita_plugin.get_command('update_sample_template') params = Parameters.load(cmd, values_dict={ 'study': int(study_id), 'template_fp': fp_rsp }) job = ProcessingJob.create(User(user_id), params) # Store the job id attaching it to the sample template id r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id, dumps({'job_id': job.id})) job.submit() return {'status': status, 'message': msg, 'file': sample_template}
def sample_template_handler_post_request(study_id, user, filepath, data_type=None): """Creates a new sample template Parameters ---------- study_id: int The study to add the sample information user: qiita_db.user import User The user performing the request filepath: str The path to the sample template file data_type: str, optional If filepath is a QIIME mapping file, the data type of the prep information file Returns ------- dict of {'job': str} job: the id of the job adding the sample information to the study Raises ------ HTTPError 404 if the filepath doesn't exist """ # Check if the current user has access to the study sample_template_checks(study_id, user) # Check if the file exists fp_rsp = check_fp(study_id, filepath) if fp_rsp['status'] != 'success': raise HTTPError(404, 'Filepath not found') filepath = fp_rsp['file'] is_mapping_file = looks_like_qiime_mapping_file(filepath) if is_mapping_file and not data_type: raise HTTPError( 400, 'Please, choose a data type if uploading a ' 'QIIME mapping file') qiita_plugin = Software.from_name_and_version('Qiita', 'alpha') cmd = qiita_plugin.get_command('create_sample_template') params = Parameters.load(cmd, values_dict={ 'fp': filepath, 'study_id': study_id, 'is_mapping_file': is_mapping_file, 'data_type': data_type }) job = ProcessingJob.create(user, params, True) r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id, dumps({'job_id': job.id})) job.submit() return {'job': job.id}
def sample_template_handler_post_request(study_id, user, filepath, data_type=None, direct_upload=False): """Creates a new sample template Parameters ---------- study_id: int The study to add the sample information user: qiita_db.user import User The user performing the request filepath: str The path to the sample template file data_type: str, optional If filepath is a QIIME mapping file, the data type of the prep information file direct_upload: boolean, optional If filepath is a direct upload; if False we need to process the filepath as part of the study upload folder Returns ------- dict of {'job': str} job: the id of the job adding the sample information to the study Raises ------ HTTPError 404 if the filepath doesn't exist """ # Check if the current user has access to the study sample_template_checks(study_id, user) # Check if the file exists if not direct_upload: fp_rsp = check_fp(study_id, filepath) if fp_rsp['status'] != 'success': raise HTTPError(404, reason='Filepath not found') filepath = fp_rsp['file'] is_mapping_file = looks_like_qiime_mapping_file(filepath) if is_mapping_file and not data_type: raise HTTPError(400, reason='Please, choose a data type if uploading ' 'a QIIME mapping file') qiita_plugin = Software.from_name_and_version('Qiita', 'alpha') cmd = qiita_plugin.get_command('create_sample_template') params = Parameters.load( cmd, values_dict={'fp': filepath, 'study_id': study_id, 'is_mapping_file': is_mapping_file, 'data_type': data_type}) job = ProcessingJob.create(user, params, True) r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id, dumps({'job_id': job.id})) job.submit() return {'job': job.id}
def post(self, study_id): method = self.get_argument('remote-request-type') url = self.get_argument('inputURL') ssh_key = self.request.files['ssh-key'][0]['body'] status = 'success' message = '' try: study = Study(int(study_id)) except QiitaDBUnknownIDError: raise HTTPError(404, reason="Study %s does not exist" % study_id) check_access(self.current_user, study, no_public=True, raise_error=True) _, upload_folder = get_mountpoint("uploads")[0] upload_folder = join(upload_folder, study_id) ssh_key_fp = join(upload_folder, '.key.txt') create_nested_path(upload_folder) with open(ssh_key_fp, 'wb') as f: f.write(ssh_key) chmod(ssh_key_fp, 0o600) qiita_plugin = Software.from_name_and_version('Qiita', 'alpha') if method == 'list': cmd = qiita_plugin.get_command('list_remote_files') params = Parameters.load(cmd, values_dict={ 'url': url, 'private_key': ssh_key_fp, 'study_id': study_id }) elif method == 'transfer': cmd = qiita_plugin.get_command('download_remote_files') params = Parameters.load(cmd, values_dict={ 'url': url, 'private_key': ssh_key_fp, 'destination': upload_folder }) else: status = 'error' message = 'Not a valid method' if status == 'success': job = ProcessingJob.create(self.current_user, params, True) job.submit() r_client.set(UPLOAD_STUDY_FORMAT % study_id, dumps({'job_id': job.id})) self.write({'status': status, 'message': message})
def post(self): if r_client.get('maintenance') is not None: raise HTTPError(503, "Site is down for maintenance") username = self.get_argument("username", "").strip().lower() passwd = self.get_argument("password", "") nextpage = self.get_argument("next", None) if nextpage is None: if "auth/" not in self.request.headers['Referer']: nextpage = self.request.headers['Referer'] else: nextpage = "%s/" % qiita_config.portal_dir msg = "" # check the user level try: if User(username).level == "unverified": # email not verified so dont log in msg = ("Email not verified. Please check your email and click " "the verify link. You may need to check your spam " "folder to find the email.<br/>If a verification email" " has not arrived in 15 minutes, please email <a href='" "mailto:[email protected]'>[email protected]</a>") except QiitaDBUnknownIDError: msg = "Unknown user" except RuntimeError: # means DB not available, so set maintenance mode and failover r_client.set( "maintenance", "Database connection unavailable, " "please try again later.") self.redirect("%s/" % qiita_config.portal_dir) return # Check the login information login = None try: login = User.login(username, passwd) except IncorrectEmailError: msg = "Unknown user" except IncorrectPasswordError: msg = "Incorrect password" except UnverifiedEmailError: msg = "You have not verified your email address" if login: # everything good so log in self.set_current_user(username) self.redirect(nextpage) else: self.render("index.html", message=msg, level='danger')
def post(self): analysis_id = int(self.get_argument('analysis_id')) user = self.current_user check_analysis_access(user, Analysis(analysis_id)) qiita_plugin = Software.from_name_and_version('Qiita', 'alpha') cmd = qiita_plugin.get_command('delete_analysis') params = Parameters.load(cmd, values_dict={'analysis_id': analysis_id}) job = ProcessingJob.create(user, params, True) # Store the job id attaching it to the sample template id r_client.set('analysis_delete_%d' % analysis_id, dumps({'job_id': job.id})) job.submit() self.redirect("%s/analysis/list/" % (qiita_config.portal_dir))
def test_post_select_samples(self): # just making sure that the key is not set in redis r_client.delete('maintenance') response = self.get('/auth/reset/') self.assertEqual(response.code, 200) self.assertIn(('<label for="newpass2" class="col-sm-2 ' 'control-label">Repeat New Password' '</label>'), response.body) # not displaying due to maintenance r_client.set('maintenance', 'This is my error message') response = self.get('/auth/reset/') self.assertEqual(response.code, 200) self.assertNotIn(('<label for="newpass2" class="col-sm-2 ' 'control-label">Repeat New Password' '</label>'), response.body) r_client.delete('maintenance')
def post(self): username = self.get_argument("username", "").strip().lower() passwd = self.get_argument("password", "") nextpage = self.get_argument("next", None) if nextpage is None: if "auth/" not in self.request.headers['Referer']: nextpage = self.request.headers['Referer'] else: nextpage = "%s/" % qiita_config.portal_dir msg = "" # check the user level try: if User(username).level == "unverified": # email not verified so dont log in msg = ("Email not verified. Please check your email and click " "the verify link. You may need to check your spam " "folder to find the email.<br/>If a verification email" " has not arrived in 15 minutes, please email <a href='" "mailto:[email protected]'>[email protected]</a>") except QiitaDBUnknownIDError: msg = "Unknown user" except RuntimeError: # means DB not available, so set maintenance mode and failover r_client.set("maintenance", "Database connection unavailable, " "please try again later.") self.redirect("%s/" % qiita_config.portal_dir) return # Check the login information login = None try: login = User.login(username, passwd) except IncorrectEmailError: msg = "Unknown user" except IncorrectPasswordError: msg = "Incorrect password" except UnverifiedEmailError: msg = "You have not verified your email address" if login: # everything good so log in self.set_current_user(username) self.redirect(nextpage) else: self.render("index.html", message=msg, level='danger')
def test_post_select_samples(self): # just making sure that the key is not set in redis r_client.delete('maintenance') response = self.get('/auth/reset/') self.assertEqual(response.code, 200) self.assertIn(('<label for="newpass2" class="col-sm-2 ' 'control-label">Repeat New Password' '</label>'), response.body) # not displaying due to maintenance r_client.set('maintenance', 'This is my error message') response = self.get('/auth/reset/') self.assertEqual(response.code, 200) self.assertNotIn(('<label for="newpass2" class="col-sm-2 ' 'control-label">Repeat New Password' '</label>'), response.body) r_client.delete('maintenance')
def post(self, study_id): method = self.get_argument('remote-request-type') url = self.get_argument('inputURL') ssh_key = self.request.files['ssh-key'][0]['body'] status = 'success' message = '' try: study = Study(int(study_id)) except QiitaDBUnknownIDError: raise HTTPError(404, reason="Study %s does not exist" % study_id) check_access( self.current_user, study, no_public=True, raise_error=True) _, upload_folder = get_mountpoint("uploads")[0] upload_folder = join(upload_folder, study_id) ssh_key_fp = join(upload_folder, '.key.txt') create_nested_path(upload_folder) with open(ssh_key_fp, 'w') as f: f.write(ssh_key) qiita_plugin = Software.from_name_and_version('Qiita', 'alpha') if method == 'list': cmd = qiita_plugin.get_command('list_remote_files') params = Parameters.load(cmd, values_dict={ 'url': url, 'private_key': ssh_key_fp, 'study_id': study_id}) elif method == 'transfer': cmd = qiita_plugin.get_command('download_remote_files') params = Parameters.load(cmd, values_dict={ 'url': url, 'private_key': ssh_key_fp, 'destination': upload_folder}) else: status = 'error' message = 'Not a valid method' if status == 'success': job = ProcessingJob.create(self.current_user, params, True) job.submit() r_client.set( UPLOAD_STUDY_FORMAT % study_id, dumps({'job_id': job.id})) self.write({'status': status, 'message': message})
def test_analysis_description_handler_get_request(self): obs = analysis_description_handler_get_request(1, User('*****@*****.**')) exp = {'analysis_name': 'SomeAnalysis', 'analysis_id': 1, 'analysis_description': 'A test analysis', 'alert_type': 'info', 'alert_msg': ''} self.assertEqual(obs, exp) r_client.set('analysis_1', dumps({'job_id': 'job_id'})) r_client.set('job_id', dumps({'status_msg': 'running'})) obs = analysis_description_handler_get_request(1, User('*****@*****.**')) exp = {'analysis_name': 'SomeAnalysis', 'analysis_id': 1, 'analysis_description': 'A test analysis', 'alert_type': 'info', 'alert_msg': 'An artifact is being deleted from this analysis'} self.assertEqual(obs, exp) r_client.set('job_id', dumps( {'status_msg': 'Success', 'return': {'status': 'danger', 'message': 'Error deleting artifact'}})) obs = analysis_description_handler_get_request(1, User('*****@*****.**')) exp = {'analysis_name': 'SomeAnalysis', 'analysis_id': 1, 'analysis_description': 'A test analysis', 'alert_type': 'danger', 'alert_msg': 'Error deleting artifact'} self.assertEqual(obs, exp)
def sample_template_delete_req(study_id, user_id): """Deletes the sample template attached to the study Parameters ---------- study_id : int The current study object id user_id : str The current user object id Returns ------- dict results dictonary in the format {'status': status, 'message': msg} status can be success, warning, or error depending on result message has the warnings or errors """ exists = _check_sample_template_exists(int(study_id)) if exists['status'] != 'success': return exists access_error = check_access(int(study_id), user_id) if access_error: return access_error qiita_plugin = Software.from_name_and_version('Qiita', 'alpha') cmd = qiita_plugin.get_command('delete_sample_template') params = Parameters.load(cmd, values_dict={'study': int(study_id)}) job = ProcessingJob.create(User(user_id), params) # Store the job id attaching it to the sample template id r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id, dumps({'job_id': job.id})) job.submit() return {'status': 'success', 'message': ''}
def list_remote_files(job): """Lists valid study files on a remote server Parameters ---------- job : qiita_db.processing_job.ProcessingJob The processing job performing the task """ with qdb.sql_connection.TRN: url = job.parameters.values['url'] private_key = job.parameters.values['private_key'] study_id = job.parameters.values['study_id'] try: files = list_remote(url, private_key) r_client.set("upload_study_%s" % study_id, dumps({'job_id': job.id, 'url': url, 'files': files})) except Exception: job._set_error(traceback.format_exception(*exc_info())) else: job._set_status('success') finally: # making sure to always delete the key so Qiita never keeps it remove(private_key)
def sample_template_handler_delete_request(study_id, user): """Deletes the sample template Parameters ---------- study_id: int The study to delete the sample information user: qiita_db.user The user performing the request Returns ------- dict of {'job': str} job: the id of the job deleting the sample information to the study Raises ------ HTTPError 404 If the sample template doesn't exist """ # Check if the current user has access to the study and if the sample # template exists sample_template_checks(study_id, user, check_exists=True) qiita_plugin = Software.from_name_and_version('Qiita', 'alpha') cmd = qiita_plugin.get_command('delete_sample_template') params = Parameters.load(cmd, values_dict={'study': int(study_id)}) job = ProcessingJob.create(user, params, True) # Store the job if deleteing the sample template r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id, dumps({'job_id': job.id})) job.submit() return {'job': job.id}
def sample_template_handler_delete_request(study_id, user): """Deletes the sample template Parameters ---------- study_id: int The study to delete the sample information user: qiita_db.user The user performing the request Returns ------- dict of {'job': str} job: the id of the job deleting the sample information to the study Raises ------ HTTPError 404 If the sample template doesn't exist """ # Check if the current user has access to the study and if the sample # template exists sample_template_checks(study_id, user, check_exists=True) qiita_plugin = Software.from_name_and_version('Qiita', 'alpha') cmd = qiita_plugin.get_command('delete_sample_template') params = Parameters.load(cmd, values_dict={'study': int(study_id)}) job = ProcessingJob.create(user, params, True) # Store the job if deleteing the sample template r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id, dumps({'job_id': job.id})) job.submit() return {'job': job.id}
def test_get_sample_template_processing_status(self): key = SAMPLE_TEMPLATE_KEY_FORMAT % 1 obs_proc, obs_at, obs_am = get_sample_template_processing_status(1) self.assertFalse(obs_proc) self.assertEqual(obs_at, "") self.assertEqual(obs_am, "") # With job id and processing qiita_plugin = qdb.software.Software.from_name_and_version( 'Qiita', 'alpha') cmd = qiita_plugin.get_command('update_sample_template') params = qdb.software.Parameters.load(cmd, values_dict={ 'study': 1, 'template_fp': 'ignored' }) job = qdb.processing_job.ProcessingJob.create( qdb.user.User('*****@*****.**'), params) job._set_status('running') r_client.set(key, dumps({'job_id': job.id})) obs_proc, obs_at, obs_am = get_sample_template_processing_status(1) self.assertTrue(obs_proc) self.assertEqual(obs_at, "info") self.assertEqual(obs_am, "This sample template is currently being processed") # With job id and success job._set_status('success') r_client.set( key, dumps({ 'job_id': job.id, 'alert_type': 'warning', 'alert_msg': 'Some\nwarning' })) obs_proc, obs_at, obs_am = get_sample_template_processing_status(1) self.assertFalse(obs_proc) self.assertEqual(obs_at, "warning") self.assertEqual(obs_am, "Some</br>warning") # With job and not success job = qdb.processing_job.ProcessingJob.create( qdb.user.User('*****@*****.**'), params) job._set_status('running') job._set_error('Some\nerror') r_client.set(key, dumps({'job_id': job.id})) obs_proc, obs_at, obs_am = get_sample_template_processing_status(1) self.assertFalse(obs_proc) self.assertEqual(obs_at, "danger") self.assertEqual(obs_am, "Some</br>error")
def test_get_sample_template_processing_status(self): key = SAMPLE_TEMPLATE_KEY_FORMAT % 1 obs_proc, obs_at, obs_am = get_sample_template_processing_status(1) self.assertFalse(obs_proc) self.assertEqual(obs_at, "") self.assertEqual(obs_am, "") # With job id and processing qiita_plugin = qdb.software.Software.from_name_and_version('Qiita', 'alpha') cmd = qiita_plugin.get_command('update_sample_template') params = qdb.software.Parameters.load( cmd, values_dict={'study': 1, 'template_fp': 'ignored'}) job = qdb.processing_job.ProcessingJob.create( qdb.user.User('*****@*****.**'), params, True) job._set_status('running') r_client.set(key, dumps({'job_id': job.id})) obs_proc, obs_at, obs_am = get_sample_template_processing_status(1) self.assertTrue(obs_proc) self.assertEqual(obs_at, "info") self.assertEqual( obs_am, "This sample template is currently being processed") # With job id and success job._set_status('success') r_client.set(key, dumps({'job_id': job.id, 'alert_type': 'warning', 'alert_msg': 'Some\nwarning'})) obs_proc, obs_at, obs_am = get_sample_template_processing_status(1) self.assertFalse(obs_proc) self.assertEqual(obs_at, "warning") self.assertEqual(obs_am, "Some</br>warning") # With job and not success job = qdb.processing_job.ProcessingJob.create( qdb.user.User('*****@*****.**'), params, True) job._set_status('running') job._set_error('Some\nerror') r_client.set(key, dumps({'job_id': job.id})) obs_proc, obs_at, obs_am = get_sample_template_processing_status(1) self.assertFalse(obs_proc) self.assertEqual(obs_at, "danger") self.assertEqual(obs_am, "Some</br>error")
def test_analysis_description_handler_get_request(self): obs = analysis_description_handler_get_request(1, User('*****@*****.**')) exp = {'analysis_name': 'SomeAnalysis', 'analysis_id': 1, 'analysis_description': 'A test analysis', 'analysis_mapping_id': 16, 'analysis_is_public': False, 'alert_type': 'info', 'alert_msg': ''} self.assertEqual(obs, exp) r_client.set('analysis_1', dumps({'job_id': 'job_id'})) r_client.set('job_id', dumps({'status_msg': 'running'})) obs = analysis_description_handler_get_request(1, User('*****@*****.**')) exp = {'analysis_name': 'SomeAnalysis', 'analysis_id': 1, 'analysis_description': 'A test analysis', 'analysis_mapping_id': 16, 'analysis_is_public': False, 'alert_type': 'info', 'alert_msg': 'An artifact is being deleted from this analysis'} self.assertEqual(obs, exp) r_client.set('job_id', dumps( {'status_msg': 'Success', 'return': {'status': 'danger', 'message': 'Error deleting artifact'}})) obs = analysis_description_handler_get_request(1, User('*****@*****.**')) exp = {'analysis_name': 'SomeAnalysis', 'analysis_id': 1, 'analysis_description': 'A test analysis', 'analysis_mapping_id': 16, 'analysis_is_public': False, 'alert_type': 'danger', 'alert_msg': 'Error deleting artifact'} self.assertEqual(obs, exp)
def artifact_post_req(user_id, filepaths, artifact_type, name, prep_template_id, artifact_id=None): """Creates the initial artifact for the prep template Parameters ---------- user_id : str User adding the atrifact filepaths : dict of str Comma-separated list of files to attach to the artifact, keyed by file type artifact_type : str The type of the artifact name : str Name to give the artifact prep_template_id : int or str castable to int Prep template to attach the artifact to artifact_id : int or str castable to int, optional The id of the imported artifact Returns ------- dict of objects A dictionary containing the new artifact ID {'status': status, 'message': message, 'artifact': id} """ prep_template_id = int(prep_template_id) prep = PrepTemplate(prep_template_id) study_id = prep.study_id # First check if the user has access to the study access_error = check_access(study_id, user_id) if access_error: return access_error user = User(user_id) if artifact_id: # if the artifact id has been provided, import the artifact qiita_plugin = Software.from_name_and_version('Qiita', 'alpha') cmd = qiita_plugin.get_command('copy_artifact') params = Parameters.load(cmd, values_dict={'artifact': artifact_id, 'prep_template': prep.id}) job = ProcessingJob.create(user, params, True) else: uploads_path = get_mountpoint('uploads')[0][1] path_builder = partial(join, uploads_path, str(study_id)) cleaned_filepaths = {} for ftype, file_list in viewitems(filepaths): # JavaScript sends us this list as a comma-separated list for fp in file_list.split(','): # JavaScript will send this value as an empty string if the # list of files was empty. In such case, the split will # generate a single element containing the empty string. Check # for that case here and, if fp is not the empty string, # proceed to check if the file exists if fp: # Check if filepath being passed exists for study full_fp = path_builder(fp) exists = check_fp(study_id, full_fp) if exists['status'] != 'success': return {'status': 'error', 'message': 'File does not exist: %s' % fp} if ftype not in cleaned_filepaths: cleaned_filepaths[ftype] = [] cleaned_filepaths[ftype].append(full_fp) # This should never happen, but it doesn't hurt to actually have # a explicit check, in case there is something odd with the JS if not cleaned_filepaths: return {'status': 'error', 'message': "Can't create artifact, no files provided."} # This try/except will catch the case when the plugins are not # activated so there is no Validate for the given artifact_type try: command = Command.get_validator(artifact_type) except QiitaDBError as e: return {'status': 'error', 'message': str(e)} job = ProcessingJob.create( user, Parameters.load(command, values_dict={ 'template': prep_template_id, 'files': dumps(cleaned_filepaths), 'artifact_type': artifact_type, 'name': name, 'analysis': None, }), True) # Submit the job job.submit() r_client.set(PREP_TEMPLATE_KEY_FORMAT % prep.id, dumps({'job_id': job.id, 'is_qiita_job': True})) return {'status': 'success', 'message': ''}
def sample_template_handler_patch_request(user, req_op, req_path, req_value=None, req_from=None, direct_upload=False): """Patches the sample template Parameters ---------- user: qiita_db.user.User The user performing the request req_op : str The operation to perform on the sample template req_path : str The path to the attribute to patch req_value : str, optional The new value req_from : str, optional The original path of the element direct_upload : boolean, optional If the file being uploaded comes from a direct upload (True) Returns ------- Raises ------ HTTPError 400 If the path parameter doens't follow the expected format 400 If the given operation is not supported """ req_path = [v for v in req_path.split('/') if v] # At this point we know the path should be at least length 2 if len(req_path) < 2: raise HTTPError(400, reason='Incorrect path parameter') study_id = int(req_path[0]) # Check if the current user has access to the study and if the sample # template exists sample_template_checks(study_id, user, check_exists=True) if req_op == 'remove': # Path format # column: study_id/columns/column_name # sample: study_id/samples/sample_id if len(req_path) != 3: raise HTTPError(400, reason='Incorrect path parameter') attribute = req_path[1] attr_id = req_path[2] qiita_plugin = Software.from_name_and_version('Qiita', 'alpha') cmd = qiita_plugin.get_command('delete_sample_or_column') params = Parameters.load(cmd, values_dict={ 'obj_class': 'SampleTemplate', 'obj_id': study_id, 'sample_or_col': attribute, 'name': attr_id }) job = ProcessingJob.create(user, params, True) # Store the job id attaching it to the sample template id r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id, dumps({'job_id': job.id})) job.submit() return {'job': job.id} elif req_op == 'replace': # WARNING: Although the patch operation is a replace, is not a full # true replace. A replace is in theory equivalent to a remove + add. # In this case, the replace operation doesn't necessarily removes # anything (e.g. when only new columns/samples are being added to the) # sample information. # Path format: study_id/data # Forcing to specify data for extensibility. In the future we may want # to use this function to replace other elements of the sample # information if len(req_path) != 2: raise HTTPError(400, reason='Incorrect path parameter') attribute = req_path[1] if attribute == 'data': # Update the sample information if req_value is None: raise HTTPError(400, reason="Value is required when updating " "sample information") if direct_upload: # We can assume that the file exist as it was generated by # the system filepath = req_value if not exists(filepath): reason = ('Upload file not found (%s), please report to ' '*****@*****.**' % filepath) raise HTTPError(404, reason=reason) else: # Check if the file exists fp_rsp = check_fp(study_id, req_value) if fp_rsp['status'] != 'success': raise HTTPError(404, reason='Filepath not found') filepath = fp_rsp['file'] qiita_plugin = Software.from_name_and_version('Qiita', 'alpha') cmd = qiita_plugin.get_command('update_sample_template') params = Parameters.load(cmd, values_dict={ 'study': study_id, 'template_fp': filepath }) job = ProcessingJob.create(user, params, True) # Store the job id attaching it to the sample template id r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id, dumps({'job_id': job.id})) job.submit() return {'job': job.id} else: raise HTTPError(404, reason='Attribute %s not found' % attribute) else: raise HTTPError(400, reason='Operation %s not supported. Current ' 'supported operations: remove, replace' % req_op)
<<<<<<< HEAD msg = '' msg_level = 'success' study_id = Artifact(preprocessed_data_id).study study = Study(study_id) ======= study = Artifact(preprocessed_data_id).study >>>>>>> 405cbef0c9f71c620da95a0c1ba6c7d3d588b3ed state = study.ebi_submission_status if state == 'submitting': message = "Cannot resubmit! Current state is: %s" % state self.display_template(preprocessed_data_id, message, 'danger') else: qiita_plugin = Software.from_name_and_version('Qiita', 'alpha') cmd = qiita_plugin.get_command('submit_to_EBI') params = Parameters.load( cmd, values_dict={'artifact': preprocessed_data_id, 'submission_type': submission_type}) job = ProcessingJob.create(user, params, True) r_client.set('ebi_submission_%s' % preprocessed_data_id, dumps({'job_id': job.id, 'is_qiita_job': True})) job.submit() level = 'success' message = 'EBI submission started. Job id: %s' % job.id self.redirect("%s/study/description/%d?level=%s&message=%s" % ( qiita_config.portal_dir, study.id, level, url_escape(message)))
def correct_redis_data(key, cmd, values_dict, user): """Corrects the data stored in the redis DB Parameters ---------- key: str The redis key to fix cmd : qiita_db.software.Command Command to use to create the processing job values_dict : dict Dictionary used to instantiate the parameters of the command user : qiita_db.user. User The user that will own the job """ info = r_client.get(key) if info: info = loads(info) if info['job_id'] is not None: if 'is_qiita_job' in info: if info['is_qiita_job']: try: job = ProcessingJob(info['job_id']) payload = { 'job_id': info['job_id'], 'alert_type': info['status'], 'alert_msg': info['alert_msg'] } r_client.set(key, dumps(payload)) except (QiitaDBUnknownIDError, KeyError): # We shomehow lost the information of this job # Simply delete the key r_client.delete(key) else: # These jobs don't contain any information on the live # dump. We can safely delete the key r_client.delete(key) else: # These jobs don't contain any information on the live # dump. We can safely delete the key r_client.delete(key) else: # Job is null, we have the information here if info['status'] == 'success': # In the success case no information is stored. We can # safely delete the key r_client.delete(key) elif info['status'] == 'warning': # In case of warning the key message stores the warning # message. We need to create a new job, mark it as # successful and store the error message as expected by # the new structure params = Parameters.load(cmd, values_dict=values_dict) job = ProcessingJob.create(user, params) job._set_status('success') payload = { 'job_id': job.id, 'alert_type': 'warning', 'alert_msg': info['message'] } r_client.set(key, dumps(payload)) else: # The status is error. The key message stores the error # message. We need to create a new job and mark it as # failed with the given error message params = Parameters.load(cmd, values_dict=values_dict) job = ProcessingJob.create(user, params) job._set_error(info['message']) payload = {'job_id': job.id} r_client.set(key, dumps(payload)) else: # The key doesn't contain any information. Delete the key r_client.delete(key)
def sample_template_handler_patch_request(user, req_op, req_path, req_value=None, req_from=None): """Patches the sample template Parameters ---------- user: qiita_db.user.User The user performing the request req_op : str The operation to perform on the sample template req_path : str The path to the attribute to patch req_value : str, optional The new value req_from : str, optional The original path of the element Returns ------- Raises ------ HTTPError 400 If the path parameter doens't follow the expected format 400 If the given operation is not supported """ req_path = [v for v in req_path.split('/') if v] # At this point we know the path should be at least length 2 if len(req_path) < 2: raise HTTPError(400, reason='Incorrect path parameter') study_id = int(req_path[0]) # Check if the current user has access to the study and if the sample # template exists sample_template_checks(study_id, user, check_exists=True) if req_op == 'remove': # Path format # column: study_id/columns/column_name # sample: study_id/samples/sample_id if len(req_path) != 3: raise HTTPError(400, reason='Incorrect path parameter') attribute = req_path[1] attr_id = req_path[2] qiita_plugin = Software.from_name_and_version('Qiita', 'alpha') cmd = qiita_plugin.get_command('delete_sample_or_column') params = Parameters.load( cmd, values_dict={'obj_class': 'SampleTemplate', 'obj_id': study_id, 'sample_or_col': attribute, 'name': attr_id}) job = ProcessingJob.create(user, params, True) # Store the job id attaching it to the sample template id r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id, dumps({'job_id': job.id})) job.submit() return {'job': job.id} elif req_op == 'replace': # WARNING: Although the patch operation is a replace, is not a full # true replace. A replace is in theory equivalent to a remove + add. # In this case, the replace operation doesn't necessarily removes # anything (e.g. when only new columns/samples are being added to the) # sample information. # Path format: study_id/data # Forcing to specify data for extensibility. In the future we may want # to use this function to replace other elements of the sample # information if len(req_path) != 2: raise HTTPError(400, reason='Incorrect path parameter') attribute = req_path[1] if attribute == 'data': # Update the sample information if req_value is None: raise HTTPError(400, reason="Value is required when updating " "sample information") # Check if the file exists fp_rsp = check_fp(study_id, req_value) if fp_rsp['status'] != 'success': raise HTTPError(404, reason='Filepath not found') filepath = fp_rsp['file'] qiita_plugin = Software.from_name_and_version('Qiita', 'alpha') cmd = qiita_plugin.get_command('update_sample_template') params = Parameters.load( cmd, values_dict={'study': study_id, 'template_fp': filepath}) job = ProcessingJob.create(user, params, True) # Store the job id attaching it to the sample template id r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id, dumps({'job_id': job.id})) job.submit() return {'job': job.id} else: raise HTTPError(404, reason='Attribute %s not found' % attribute) else: raise HTTPError(400, reason='Operation %s not supported. Current ' 'supported operations: remove, replace' % req_op)
def test_analysis_description_handler_get_request(self): obs = analysis_description_handler_get_request(1, User('*****@*****.**')) exp = { 'analysis_name': 'SomeAnalysis', 'analysis_id': 1, 'analysis_description': 'A test analysis', 'analysis_mapping_id': 16, 'analysis_is_public': False, 'alert_type': 'info', 'artifacts': { 4: (1, 'Identification of the Microbiomes for Cannabis ' 'Soils', ('Pick closed-reference OTUs | Split ' 'libraries FASTQ', 'QIIME v1.9.1'), [ '1.SKB7.640196', '1.SKB8.640193', '1.SKD8.640184', '1.SKM4.640180', '1.SKM9.640192' ]), 5: (1, 'Identification of the Microbiomes for Cannabis ' 'Soils', ('Pick closed-reference OTUs | Split ' 'libraries FASTQ', 'QIIME v1.9.1'), [ '1.SKB7.640196', '1.SKB8.640193', '1.SKD8.640184', '1.SKM4.640180', '1.SKM9.640192' ]), 6: (1, 'Identification of the Microbiomes for Cannabis ' 'Soils', ('Pick closed-reference OTUs | Split ' 'libraries FASTQ', 'QIIME v1.9.1'), [ '1.SKB7.640196', '1.SKB8.640193', '1.SKD8.640184', '1.SKM4.640180', '1.SKM9.640192' ]) }, 'alert_msg': '' } self.assertEqual(obs, exp) r_client.set('analysis_1', dumps({'job_id': 'job_id'})) r_client.set('job_id', dumps({'status_msg': 'running'})) obs = analysis_description_handler_get_request(1, User('*****@*****.**')) exp = { 'analysis_name': 'SomeAnalysis', 'analysis_id': 1, 'analysis_description': 'A test analysis', 'analysis_mapping_id': 16, 'analysis_is_public': False, 'alert_type': 'info', 'artifacts': { 4: (1, 'Identification of the Microbiomes for Cannabis ' 'Soils', ('Pick closed-reference OTUs | Split ' 'libraries FASTQ', 'QIIME v1.9.1'), [ '1.SKB7.640196', '1.SKB8.640193', '1.SKD8.640184', '1.SKM4.640180', '1.SKM9.640192' ]), 5: (1, 'Identification of the Microbiomes for Cannabis ' 'Soils', ('Pick closed-reference OTUs | Split ' 'libraries FASTQ', 'QIIME v1.9.1'), [ '1.SKB7.640196', '1.SKB8.640193', '1.SKD8.640184', '1.SKM4.640180', '1.SKM9.640192' ]), 6: (1, 'Identification of the Microbiomes for Cannabis ' 'Soils', ('Pick closed-reference OTUs | Split ' 'libraries FASTQ', 'QIIME v1.9.1'), [ '1.SKB7.640196', '1.SKB8.640193', '1.SKD8.640184', '1.SKM4.640180', '1.SKM9.640192' ]) }, 'alert_msg': 'An artifact is being deleted from this analysis' } self.assertEqual(obs, exp) r_client.set( 'job_id', dumps({ 'status_msg': 'Success', 'return': { 'status': 'danger', 'message': 'Error deleting artifact' } })) obs = analysis_description_handler_get_request(1, User('*****@*****.**')) exp = { 'analysis_name': 'SomeAnalysis', 'analysis_id': 1, 'analysis_description': 'A test analysis', 'analysis_mapping_id': 16, 'analysis_is_public': False, 'alert_type': 'danger', 'artifacts': { 4: (1, 'Identification of the Microbiomes for Cannabis ' 'Soils', ('Pick closed-reference OTUs | Split ' 'libraries FASTQ', 'QIIME v1.9.1'), [ '1.SKB7.640196', '1.SKB8.640193', '1.SKD8.640184', '1.SKM4.640180', '1.SKM9.640192' ]), 5: (1, 'Identification of the Microbiomes for Cannabis ' 'Soils', ('Pick closed-reference OTUs | Split ' 'libraries FASTQ', 'QIIME v1.9.1'), [ '1.SKB7.640196', '1.SKB8.640193', '1.SKD8.640184', '1.SKM4.640180', '1.SKM9.640192' ]), 6: (1, 'Identification of the Microbiomes for Cannabis ' 'Soils', ('Pick closed-reference OTUs | Split ' 'libraries FASTQ', 'QIIME v1.9.1'), [ '1.SKB7.640196', '1.SKB8.640193', '1.SKD8.640184', '1.SKM4.640180', '1.SKM9.640192' ]) }, 'alert_msg': 'Error deleting artifact' } self.assertEqual(obs, exp)
def post(self): user_email = self.get_argument('user_email') job_id = self.get_argument('job_id', None) prep_id = self.get_argument('prep_id', None) atype = self.get_argument('artifact_type') aname = self.get_argument('command_artifact_name', 'Name') files = self.get_argument('files') if job_id is None and prep_id is None: raise HTTPError(400, reason='You need to specify a job_id or a prep_id') if job_id is not None and prep_id is not None: raise HTTPError( 400, reason='You need to specify only a job_id or a prep_id') user = qdb.user.User(user_email) values = { 'files': files, 'artifact_type': atype, 'name': aname, # leaving here in case we need to add a way to add an artifact # directly to an analysis, for more information see # ProcessingJob._complete_artifact_transformation 'analysis': None } PJ = qdb.processing_job.ProcessingJob if job_id is not None: TN = qdb.sql_connection.TRN job = PJ(job_id) with TN: sql = """SELECT command_output_id FROM qiita.command_output WHERE name = %s AND command_id = %s""" TN.add(sql, [aname, job.command.id]) results = TN.execute_fetchflatten() if len(results) < 1: raise HTTPError( 400, 'The command_artifact_name does not ' 'exist in the command') cmd_out_id = results[0] provenance = { 'job': job_id, 'cmd_out_id': cmd_out_id, # direct_creation is a flag to avoid having to wait # for the complete job to create the new artifact, # which is normally ran during regular processing. # Skipping is fine because we are adding an artifact # to an existing job outside of regular processing 'direct_creation': True, 'name': aname } values['provenance'] = dumps(provenance) # inherint the first prep info file from the first input artifact prep_id = job.input_artifacts[0].prep_templates[0].id else: prep_id = int(prep_id) values['template'] = prep_id cmd = qdb.software.Command.get_validator(atype) params = qdb.software.Parameters.load(cmd, values_dict=values) new_job = PJ.create(user, params, True) new_job.submit() r_client.set('prep_template_%d' % prep_id, dumps({ 'job_id': new_job.id, 'is_qiita_job': True })) self.write(new_job.id) self.finish()
def artifact_post_req(user_id, filepaths, artifact_type, name, prep_template_id, artifact_id=None): """Creates the initial artifact for the prep template Parameters ---------- user_id : str User adding the atrifact filepaths : dict of str Comma-separated list of files to attach to the artifact, keyed by file type artifact_type : str The type of the artifact name : str Name to give the artifact prep_template_id : int or str castable to int Prep template to attach the artifact to artifact_id : int or str castable to int, optional The id of the imported artifact Returns ------- dict of objects A dictionary containing the new artifact ID {'status': status, 'message': message, 'artifact': id} """ prep_template_id = int(prep_template_id) prep = PrepTemplate(prep_template_id) study_id = prep.study_id # First check if the user has access to the study access_error = check_access(study_id, user_id) if access_error: return access_error user = User(user_id) if artifact_id: # if the artifact id has been provided, import the artifact qiita_plugin = Software.from_name_and_version('Qiita', 'alpha') cmd = qiita_plugin.get_command('copy_artifact') params = Parameters.load(cmd, values_dict={ 'artifact': artifact_id, 'prep_template': prep.id }) job = ProcessingJob.create(user, params, True) else: uploads_path = get_mountpoint('uploads')[0][1] path_builder = partial(join, uploads_path, str(study_id)) cleaned_filepaths = {} for ftype, file_list in filepaths.items(): # JavaScript sends us this list as a comma-separated list for fp in file_list.split(','): # JavaScript will send this value as an empty string if the # list of files was empty. In such case, the split will # generate a single element containing the empty string. Check # for that case here and, if fp is not the empty string, # proceed to check if the file exists if fp: # Check if filepath being passed exists for study full_fp = path_builder(fp) exists = check_fp(study_id, full_fp) if exists['status'] != 'success': return { 'status': 'error', 'message': 'File does not exist: %s' % fp } if ftype not in cleaned_filepaths: cleaned_filepaths[ftype] = [] cleaned_filepaths[ftype].append(full_fp) # This should never happen, but it doesn't hurt to actually have # a explicit check, in case there is something odd with the JS if not cleaned_filepaths: return { 'status': 'error', 'message': "Can't create artifact, no files provided." } # This try/except will catch the case when the plugins are not # activated so there is no Validate for the given artifact_type try: command = Command.get_validator(artifact_type) except QiitaDBError as e: return {'status': 'error', 'message': str(e)} job = ProcessingJob.create( user, Parameters.load(command, values_dict={ 'template': prep_template_id, 'files': dumps(cleaned_filepaths), 'artifact_type': artifact_type, 'name': name, 'analysis': None, }), True) # Submit the job job.submit() r_client.set(PREP_TEMPLATE_KEY_FORMAT % prep.id, dumps({ 'job_id': job.id, 'is_qiita_job': True })) return {'status': 'success', 'message': ''}
def sample_template_patch_request(user_id, req_op, req_path, req_value=None, req_from=None): """Modifies an attribute of the artifact Parameters ---------- user_id : str The id of the user performing the patch operation req_op : str The operation to perform on the artifact req_path : str The prep information and attribute to patch req_value : str, optional The value that needs to be modified req_from : str, optional The original path of the element Returns ------- dict of {str, str} A dictionary with the following keys: - status: str, whether if the request is successful or not - message: str, if the request is unsuccessful, a human readable error """ if req_op == 'remove': req_path = [v for v in req_path.split('/') if v] # format # column: study_id/row_id/columns/column_name # sample: study_id/row_id/samples/sample_id if len(req_path) != 4: return {'status': 'error', 'message': 'Incorrect path parameter'} st_id = req_path[0] row_id = req_path[1] attribute = req_path[2] attr_id = req_path[3] # Check if the user actually has access to the template st = SampleTemplate(st_id) access_error = check_access(st.study_id, user_id) if access_error: return access_error qiita_plugin = Software.from_name_and_version('Qiita', 'alpha') cmd = qiita_plugin.get_command('delete_sample_or_column') params = Parameters.load(cmd, values_dict={ 'obj_class': 'SampleTemplate', 'obj_id': int(st_id), 'sample_or_col': attribute, 'name': attr_id }) job = ProcessingJob.create(User(user_id), params) # Store the job id attaching it to the sample template id r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % st_id, dumps({'job_id': job.id})) job.submit() return {'status': 'success', 'message': '', 'row_id': row_id} else: return { 'status': 'error', 'message': 'Operation "%s" not supported. ' 'Current supported operations: remove' % req_op, 'row_id': 0 }
def prep_template_patch_req(user_id, req_op, req_path, req_value=None, req_from=None): """Modifies an attribute of the prep template Parameters ---------- user_id : str The id of the user performing the patch operation req_op : str The operation to perform on the prep information req_path : str The prep information and attribute to patch req_value : str, optional The value that needs to be modified req_from : str, optional The original path of the element Returns ------- dict of {str, str, str} A dictionary with the following keys: - status: str, whether if the request is successful or not - message: str, if the request is unsuccessful, a human readable error - row_id: str, the row_id that we tried to delete """ req_path = [v for v in req_path.split('/') if v] if req_op == 'replace': # The structure of the path should be /prep_id/attribute_to_modify/ # so if we don't have those 2 elements, we should return an error if len(req_path) != 2: return {'status': 'error', 'message': 'Incorrect path parameter'} prep_id = int(req_path[0]) attribute = req_path[1] # Check if the user actually has access to the prep template prep = PrepTemplate(prep_id) access_error = check_access(prep.study_id, user_id) if access_error: return access_error status = 'success' msg = '' if attribute == 'investigation_type': prep.investigation_type = req_value elif attribute == 'data': fp = check_fp(prep.study_id, req_value) if fp['status'] != 'success': return fp fp = fp['file'] qiita_plugin = Software.from_name_and_version('Qiita', 'alpha') cmd = qiita_plugin.get_command('update_prep_template') params = Parameters.load( cmd, values_dict={'prep_template': prep_id, 'template_fp': fp}) job = ProcessingJob.create(User(user_id), params, True) r_client.set(PREP_TEMPLATE_KEY_FORMAT % prep_id, dumps({'job_id': job.id})) job.submit() elif attribute == 'name': prep.name = req_value.strip() else: # We don't understand the attribute so return an error return {'status': 'error', 'message': 'Attribute "%s" not found. ' 'Please, check the path parameter' % attribute} return {'status': status, 'message': msg} elif req_op == 'remove': # The structure of the path should be: # /prep_id/row_id/{columns|samples}/name if len(req_path) != 4: return {'status': 'error', 'message': 'Incorrect path parameter'} prep_id = int(req_path[0]) row_id = req_path[1] attribute = req_path[2] attr_id = req_path[3] # Check if the user actually has access to the study pt = PrepTemplate(prep_id) access_error = check_access(pt.study_id, user_id) if access_error: return access_error qiita_plugin = Software.from_name_and_version('Qiita', 'alpha') cmd = qiita_plugin.get_command('delete_sample_or_column') params = Parameters.load( cmd, values_dict={'obj_class': 'PrepTemplate', 'obj_id': prep_id, 'sample_or_col': attribute, 'name': attr_id}) job = ProcessingJob.create(User(user_id), params, True) # Store the job id attaching it to the sample template id r_client.set(PREP_TEMPLATE_KEY_FORMAT % prep_id, dumps({'job_id': job.id})) job.submit() return {'status': 'success', 'message': '', 'row_id': row_id} else: return {'status': 'error', 'message': 'Operation "%s" not supported. ' 'Current supported operations: replace, remove' % req_op, 'row_id': '0'}
def sample_template_post_req(study_id, user_id, data_type, sample_template): """Creates the sample template from the given file Parameters ---------- study_id : int The current study object id user_id : str The current user object id data_type : str Data type for the sample template sample_template : str filename to use for creation Returns ------- dict results dictonary in the format {'status': status, 'message': msg, 'file': sample_template} status can be success, warning, or error depending on result message has the warnings or errors file has the file name """ study_id = int(study_id) access_error = check_access(study_id, user_id) if access_error: return access_error fp_rsp = check_fp(study_id, sample_template) if fp_rsp['status'] != 'success': # Unknown filepath, so return the error message return fp_rsp fp_rsp = fp_rsp['file'] # Define here the message and message level in case of success is_mapping_file = looks_like_qiime_mapping_file(fp_rsp) if is_mapping_file and not data_type: return { 'status': 'error', 'message': 'Please, choose a data type if uploading a ' 'QIIME mapping file', 'file': sample_template } qiita_plugin = Software.from_name_and_version('Qiita', 'alpha') cmd = qiita_plugin.get_command('create_sample_template') params = Parameters.load(cmd, values_dict={ 'fp': fp_rsp, 'study_id': study_id, 'is_mapping_file': is_mapping_file, 'data_type': data_type }) job = ProcessingJob.create(User(user_id), params) r_client.set(SAMPLE_TEMPLATE_KEY_FORMAT % study_id, dumps({'job_id': job.id})) # Store the job id attaching it to the sample template id job.submit() return {'status': 'success', 'message': '', 'file': sample_template}
def correct_redis_data(key, cmd, values_dict, user): """Corrects the data stored in the redis DB Parameters ---------- key: str The redis key to fix cmd : qiita_db.software.Command Command to use to create the processing job values_dict : dict Dictionary used to instantiate the parameters of the command user : qiita_db.user. User The user that will own the job """ info = r_client.get(key) if info: info = loads(info) if info['job_id'] is not None: if 'is_qiita_job' in info: if info['is_qiita_job']: try: job = ProcessingJob(info['job_id']) payload = {'job_id': info['job_id'], 'alert_type': info['status'], 'alert_msg': info['alert_msg']} r_client.set(key, dumps(payload)) except (QiitaDBUnknownIDError, KeyError): # We shomehow lost the information of this job # Simply delete the key r_client.delete(key) else: # These jobs don't contain any information on the live # dump. We can safely delete the key r_client.delete(key) else: # These jobs don't contain any information on the live # dump. We can safely delete the key r_client.delete(key) else: # Job is null, we have the information here if info['status'] == 'success': # In the success case no information is stored. We can # safely delete the key r_client.delete(key) elif info['status'] == 'warning': # In case of warning the key message stores the warning # message. We need to create a new job, mark it as # successful and store the error message as expected by # the new structure params = Parameters.load(cmd, values_dict=values_dict) job = ProcessingJob.create(user, params) job._set_status('success') payload = {'job_id': job.id, 'alert_type': 'warning', 'alert_msg': info['message']} r_client.set(key, dumps(payload)) else: # The status is error. The key message stores the error # message. We need to create a new job and mark it as # failed with the given error message params = Parameters.load(cmd, values_dict=values_dict) job = ProcessingJob.create(user, params) job._set_error(info['message']) payload = {'job_id': job.id} r_client.set(key, dumps(payload)) else: # The key doesn't contain any information. Delete the key r_client.delete(key)