def test_get_data_types(self): """Tests that get_data_types works with valid arguments""" obs = get_data_types() exp = {"16S": 1, "18S": 2, "ITS": 3, "Proteomic": 4, "Metabolomic": 5, "Metagenomic": 6} self.assertEqual(obs, exp) obs = get_data_types(key="data_type_id") exp = {v: k for k, v in exp.items()} self.assertEqual(obs, exp)
def test_get_data_types(self): """Tests that get_data_types works with valid arguments""" obs = get_data_types() exp = {'16S': 1, '18S': 2, 'ITS': 3, 'Proteomic': 4, 'Metabolomic': 5, 'Metagenomic': 6} self.assertEqual(obs, exp) obs = get_data_types(key='data_type_id') exp = {v: k for k, v in exp.items()} self.assertEqual(obs, exp)
def test_get_data_types(self): """Tests that get_data_types works with valid arguments""" obs = get_data_types() exp = { '16S': 1, '18S': 2, 'ITS': 3, 'Proteomic': 4, 'Metabolomic': 5, 'Metagenomic': 6 } self.assertEqual(obs, exp) obs = get_data_types(key='data_type_id') exp = {v: k for k, v in exp.items()} self.assertEqual(obs, exp)
def render(self, study, full_access): files = [f for _, f in get_files_from_uploads_folders(str(study.id)) if f.endswith(('txt', 'tsv'))] data_types = sorted(viewitems(get_data_types()), key=itemgetter(1)) prep_templates_info = [ res for res in _template_generator(study, full_access)] # Get all the ENA terms for the investigation type ontology = Ontology(convert_to_id('ENA', 'ontology')) # make "Other" show at the bottom of the drop down menu ena_terms = [] for v in sorted(ontology.terms): if v != 'Other': ena_terms.append('<option value="%s">%s</option>' % (v, v)) ena_terms.append('<option value="Other">Other</option>') # New Type is for users to add a new user-defined investigation type user_defined_terms = ontology.user_defined_terms + ['New Type'] return self.render_string( "study_description_templates/prep_template_tab.html", files=files, data_types=data_types, available_prep_templates=prep_templates_info, ena_terms=ena_terms, user_defined_terms=user_defined_terms, study=study, full_access=full_access)
def render(self, study, full_access): files = [ f for _, f in get_files_from_uploads_folders(str(study.id)) if f.endswith(('txt', 'tsv')) ] data_types = sorted(viewitems(get_data_types()), key=itemgetter(1)) prep_templates_info = [ res for res in _template_generator(study, full_access) ] # Get all the ENA terms for the investigation type ontology = Ontology(convert_to_id('ENA', 'ontology')) # make "Other" show at the bottom of the drop down menu ena_terms = [] for v in sorted(ontology.terms): if v != 'Other': ena_terms.append('<option value="%s">%s</option>' % (v, v)) ena_terms.append('<option value="Other">Other</option>') # New Type is for users to add a new user-defined investigation type user_defined_terms = ontology.user_defined_terms + ['New Type'] return self.render_string( "study_description_templates/prep_template_tab.html", files=files, data_types=data_types, available_prep_templates=prep_templates_info, ena_terms=ena_terms, user_defined_terms=user_defined_terms, study=study, full_access=full_access)
def render(self, study): study_info = study.info id = study.id abstract = study_info['study_abstract'] description = study_info['study_description'] pmids = ", ".join([pubmed_linkifier([pmid]) for pmid in study.pmids]) princ_inv = StudyPerson(study_info['principal_investigator_id']) pi_link = study_person_linkifier((princ_inv.email, princ_inv.name)) number_samples_promised = study_info['number_samples_promised'] number_samples_collected = study_info['number_samples_collected'] metadata_complete = study_info['metadata_complete'] data_types = sorted(viewitems(get_data_types()), key=itemgetter(1)) # Retrieve the files from the uploads folder, so the user can choose # the sample template of the study. Filter them to only include the # ones that ends with 'txt' or 'tsv'. files = [f for _, f in get_files_from_uploads_folders(str(study.id)) if f.endswith(('txt', 'tsv'))] # If the sample template exists, retrieve all its filepaths if SampleTemplate.exists(study.id): sample_templates = SampleTemplate(study.id).get_filepaths() else: # If the sample template does not exist, just pass an empty list sample_templates = [] # Check if the request came from a local source is_local_request = is_localhost(self.request.headers['host']) # The user can choose the sample template only if the study is # sandboxed or the current user is an admin show_select_sample = ( study.status == 'sandbox' or self.current_user.level == 'admin') # Ebi information ebi_status = study.ebi_submission_status ebi_accession = study.ebi_study_accession if ebi_accession: ebi_accession = (EBI_LINKIFIER.format(ebi_accession)) return self.render_string( "study_description_templates/study_information_tab.html", abstract=abstract, description=description, id=id, pmids=pmids, principal_investigator=pi_link, number_samples_promised=number_samples_promised, number_samples_collected=number_samples_collected, metadata_complete=metadata_complete, show_select_sample=show_select_sample, files=files, study_id=study.id, sample_templates=sample_templates, is_local_request=is_local_request, data_types=data_types, ebi_status=ebi_status, ebi_accession=ebi_accession)
def get(self): data = self.get_argument("data", None) study_id = self.get_argument("study_id", None) data_type = self.get_argument("data_type", None) dtypes = get_data_types().keys() if data is None or study_id is None or data not in ('raw', 'biom'): raise HTTPError(422, reason='You need to specify both data (the ' 'data type you want to download - raw/biom) and ' 'study_id') elif data_type is not None and data_type not in dtypes: raise HTTPError(422, reason='Not a valid data_type. Valid types ' 'are: %s' % ', '.join(dtypes)) else: study_id = int(study_id) try: study = Study(study_id) except QiitaDBUnknownIDError: raise HTTPError(422, reason='Study does not exist') else: public_raw_download = study.public_raw_download if study.status != 'public': raise HTTPError(422, reason='Study is not public. If this ' 'is a mistake contact: ' '*****@*****.**') elif data == 'raw' and not public_raw_download: raise HTTPError(422, reason='No raw data access. If this ' 'is a mistake contact: ' '*****@*****.**') else: to_download = [] for a in study.artifacts( dtype=data_type, artifact_type='BIOM' if data == 'biom' else None): if a.visibility != 'public': continue to_download.extend(self._list_artifact_files_nginx(a)) if not to_download: raise HTTPError(422, reason='Nothing to download. If ' 'this is a mistake contact: ' '*****@*****.**') else: self._write_nginx_file_list(to_download) zip_fn = 'study_%d_%s_%s.zip' % ( study_id, data, datetime.now().strftime('%m%d%y-%H%M%S')) self._set_nginx_headers(zip_fn) self.finish()
def get(self): data = self.get_argument("data", None) study_id = self.get_argument("study_id", None) prep_id = self.get_argument("prep_id", None) data_type = self.get_argument("data_type", None) dtypes = get_data_types().keys() templates = ['sample_information', 'prep_information'] valid_data = ['raw', 'biom'] + templates to_download = [] if data is None or (study_id is None and prep_id is None) or \ data not in valid_data: raise HTTPError(422, reason='You need to specify both data (the ' 'data type you want to download - %s) and ' 'study_id or prep_id' % '/'.join(valid_data)) elif data_type is not None and data_type not in dtypes: raise HTTPError(422, reason='Not a valid data_type. Valid types ' 'are: %s' % ', '.join(dtypes)) elif data in templates and prep_id is None and study_id is None: raise HTTPError(422, reason='If downloading a sample or ' 'preparation file you need to define study_id or' ' prep_id') elif data in templates: if data_type is not None: raise HTTPError(422, reason='If requesting an information ' 'file you cannot specify the data_type') elif prep_id is not None and data == 'prep_information': fname = 'preparation_information_%s' % prep_id prep_id = int(prep_id) try: infofile = PrepTemplate(prep_id) except QiitaDBUnknownIDError: raise HTTPError( 422, reason='Preparation information does not exist') elif study_id is not None and data == 'sample_information': fname = 'sample_information_%s' % study_id study_id = int(study_id) try: infofile = SampleTemplate(study_id) except QiitaDBUnknownIDError: raise HTTPError( 422, reason='Sample information does not exist') else: raise HTTPError(422, reason='Review your parameters, not a ' 'valid combination') x = retrieve_filepaths( infofile._filepath_table, infofile._id_column, infofile.id, sort='descending')[0] basedir_len = len(get_db_files_base_dir()) + 1 fp = x['fp'][basedir_len:] to_download.append((fp, fp, str(x['checksum']), str(x['fp_size']))) self._write_nginx_file_list(to_download) zip_fn = '%s_%s.zip' % ( fname, datetime.now().strftime('%m%d%y-%H%M%S')) self._set_nginx_headers(zip_fn) else: study_id = int(study_id) try: study = Study(study_id) except QiitaDBUnknownIDError: raise HTTPError(422, reason='Study does not exist') else: public_raw_download = study.public_raw_download if study.status != 'public': raise HTTPError(404, reason='Study is not public. If this ' 'is a mistake contact: ' '*****@*****.**') elif data == 'raw' and not public_raw_download: raise HTTPError(422, reason='No raw data access. If this ' 'is a mistake contact: ' '*****@*****.**') else: # raw data artifacts = [a for a in study.artifacts(dtype=data_type) if not a.parents] # bioms if data == 'biom': artifacts = study.artifacts( dtype=data_type, artifact_type='BIOM') for a in artifacts: if a.visibility != 'public': continue to_download.extend(self._list_artifact_files_nginx(a)) if not to_download: raise HTTPError(422, reason='Nothing to download. If ' 'this is a mistake contact: ' '*****@*****.**') else: self._write_nginx_file_list(to_download) zip_fn = 'study_%d_%s_%s.zip' % ( study_id, data, datetime.now().strftime( '%m%d%y-%H%M%S')) self._set_nginx_headers(zip_fn) self.finish()
def render(self, study): study_info = study.info id = study.id abstract = study_info['study_abstract'] description = study_info['study_description'] publications = [] for doi, pmid in study.publications: if doi is not None: publications.append(doi_linkifier([doi])) if pmid is not None: publications.append(pubmed_linkifier([pmid])) publications = ", ".join(publications) princ_inv = StudyPerson(study_info['principal_investigator_id']) pi_link = study_person_linkifier((princ_inv.email, princ_inv.name)) number_samples_promised = study_info['number_samples_promised'] number_samples_collected = study_info['number_samples_collected'] metadata_complete = study_info['metadata_complete'] data_types = sorted(viewitems(get_data_types()), key=itemgetter(1)) # Retrieve the files from the uploads folder, so the user can choose # the sample template of the study. Filter them to only include the # ones that ends with 'txt' or 'tsv'. files = [ f for _, f in get_files_from_uploads_folders(str(study.id)) if f.endswith(('txt', 'tsv')) ] # If the sample template exists, retrieve all its filepaths if SampleTemplate.exists(study.id): sample_templates = SampleTemplate(study.id).get_filepaths() else: # If the sample template does not exist, just pass an empty list sample_templates = [] # Check if the request came from a local source is_local_request = is_localhost(self.request.headers['host']) # The user can choose the sample template only if the study is # sandboxed or the current user is an admin show_select_sample = (study.status == 'sandbox' or self.current_user.level == 'admin') # EBI information ebi_status = study.ebi_submission_status ebi_accession = study.ebi_study_accession if ebi_accession: ebi_accession = (EBI_LINKIFIER.format(ebi_accession)) return self.render_string( "study_description_templates/study_information_tab.html", abstract=abstract, description=description, id=id, publications=publications, principal_investigator=pi_link, number_samples_promised=number_samples_promised, number_samples_collected=number_samples_collected, metadata_complete=metadata_complete, show_select_sample=show_select_sample, files=files, study_id=study.id, sample_templates=sample_templates, is_local_request=is_local_request, data_types=data_types, ebi_status=ebi_status, ebi_accession=ebi_accession)
def render(self, study, raw_data, full_access): user = self.current_user study_status = study.status user_level = user.level raw_data_id = raw_data.id files = [f for _, f in get_files_from_uploads_folders(str(study.id))] # Get the available prep template data types data_types = sorted(viewitems(get_data_types()), key=itemgetter(1)) # Get all the ENA terms for the investigation type ontology = Ontology(convert_to_id('ENA', 'ontology')) # make "Other" show at the bottom of the drop down menu ena_terms = [] for v in sorted(ontology.terms): if v != 'Other': ena_terms.append('<option value="%s">%s</option>' % (v, v)) ena_terms.append('<option value="Other">Other</option>') # New Type is for users to add a new user-defined investigation type user_defined_terms = ontology.user_defined_terms + ['New Type'] # Get all the information about the prep templates available_prep_templates = [] for p in sorted(raw_data.prep_templates): if PrepTemplate.exists(p): pt = PrepTemplate(p) # if the prep template doesn't belong to this study, skip if (study.id == pt.study_id and (full_access or pt.status == 'public')): available_prep_templates.append(pt) # getting filepath_types if raw_data.filetype == 'SFF': fts = ['sff'] elif raw_data.filetype == 'FASTA': fts = ['fasta', 'qual'] elif raw_data.filetype == 'FASTQ': fts = ['barcodes', 'forward seqs', 'reverse seqs'] else: fts = [k.split('_', 1)[1].replace('_', ' ') for k in get_filepath_types() if k.startswith('raw_')] # The raw data can be edited (e.i. adding prep templates and files) # only if the study is sandboxed or the current user is an admin is_editable = study_status == 'sandbox' or user_level == 'admin' # Get the files linked with the raw_data raw_data_files = raw_data.get_filepaths() # Get the status of the data linking raw_data_link_status = raw_data.link_filepaths_status # By default don't show the unlink button show_unlink_btn = False # By default disable the the link file button disable_link_btn = True # Define the message for the link status if raw_data_link_status == 'linking': link_msg = "Linking files..." elif raw_data_link_status == 'unlinking': link_msg = "Unlinking files..." else: # The link button is only disable if raw data link status is # linking or unlinking, so we can enable it here disable_link_btn = False # The unlink button is only shown if the study is editable, the raw # data linking status is not in linking or unlinking, and there are # files attached to the raw data. At this point, we are sure that # the raw data linking status is not in linking or unlinking so we # still need to check if it is editable or there are files attached show_unlink_btn = is_editable and raw_data_files if raw_data_link_status.startswith('failed'): link_msg = "Error (un)linking files: %s" % raw_data_link_status else: link_msg = "" # Get the raw_data filetype raw_data_filetype = raw_data.filetype return self.render_string( "study_description_templates/raw_data_editor_tab.html", study_id=study.id, study_status=study_status, user_level=user_level, raw_data_id=raw_data_id, files=files, data_types=data_types, ena_terms=ena_terms, user_defined_terms=user_defined_terms, available_prep_templates=available_prep_templates, filepath_types=fts, is_editable=is_editable, show_unlink_btn=show_unlink_btn, link_msg=link_msg, raw_data_files=raw_data_files, raw_data_filetype=raw_data_filetype, disable_link_btn=disable_link_btn)
def render(self, study, raw_data): user = self.current_user study_status = study.status user_level = user.level raw_data_id = raw_data.id files = [f for _, f in get_files_from_uploads_folders(str(study.id))] # Get the available prep template data types data_types = sorted(viewitems(get_data_types()), key=itemgetter(1)) # Get all the ENA terms for the investigation type ontology = Ontology(convert_to_id('ENA', 'ontology')) # make "Other" show at the bottom of the drop down menu ena_terms = [] for v in sorted(ontology.terms): if v != 'Other': ena_terms.append('<option value="%s">%s</option>' % (v, v)) ena_terms.append('<option value="Other">Other</option>') # New Type is for users to add a new user-defined investigation type user_defined_terms = ontology.user_defined_terms + ['New Type'] # Get all the information about the prep templates available_prep_templates = [] for p in sorted(raw_data.prep_templates): if PrepTemplate.exists(p): pt = PrepTemplate(p) # if the prep template doesn't belong to this study, skip if study.id == pt.study_id: available_prep_templates.append(pt) # getting filepath_types if raw_data.filetype == 'SFF': fts = ['sff'] elif raw_data.filetype == 'FASTA': fts = ['fasta', 'qual'] elif raw_data.filetype == 'FASTQ': fts = ['barcodes', 'forward seqs', 'reverse seqs'] else: fts = [ k.split('_', 1)[1].replace('_', ' ') for k in get_filepath_types() if k.startswith('raw_') ] # The raw data can be edited (e.i. adding prep templates and files) # only if the study is sandboxed or the current user is an admin is_editable = study_status == 'sandbox' or user_level == 'admin' # Get the files linked with the raw_data raw_data_files = raw_data.get_filepaths() # Get the status of the data linking raw_data_link_status = raw_data.link_filepaths_status # By default don't show the unlink button show_unlink_btn = False # By default disable the the link file button disable_link_btn = True # Define the message for the link status if raw_data_link_status == 'linking': link_msg = "Linking files..." elif raw_data_link_status == 'unlinking': link_msg = "Unlinking files..." else: # The link button is only disable if raw data link status is # linking or unlinking, so we can enable it here disable_link_btn = False # The unlink button is only shown if the study is editable, the raw # data linking status is not in linking or unlinking, and there are # files attached to the raw data. At this point, we are sure that # the raw data linking status is not in linking or unlinking so we # still need to check if it is editable or there are files attached show_unlink_btn = is_editable and raw_data_files if raw_data_link_status.startswith('failed'): link_msg = "Error (un)linking files: %s" % raw_data_link_status else: link_msg = "" # Get the raw_data filetype raw_data_filetype = raw_data.filetype return self.render_string( "study_description_templates/raw_data_editor_tab.html", study_id=study.id, study_status=study_status, user_level=user_level, raw_data_id=raw_data_id, files=files, data_types=data_types, ena_terms=ena_terms, user_defined_terms=user_defined_terms, available_prep_templates=available_prep_templates, filepath_types=fts, is_editable=is_editable, show_unlink_btn=show_unlink_btn, link_msg=link_msg, raw_data_files=raw_data_files, raw_data_filetype=raw_data_filetype, disable_link_btn=disable_link_btn)
def display_template(self, study, msg, msg_level, tab_to_display=""): """Simple function to avoid duplication of code""" # Check if the request came from a local source is_local_request = ('localhost' in self.request.headers['host'] or '127.0.0.1' in self.request.headers['host']) # getting raw filepath_ types fts = [k.split('_', 1)[1].replace('_', ' ') for k in get_filepath_types() if k.startswith('raw_')] fts = ['<option value="%s">%s</option>' % (f, f) for f in fts] user = User(self.current_user) # getting the RawData and its prep templates available_raw_data = yield Task(self.get_raw_data, study.raw_data()) available_prep_templates = yield Task(self.get_prep_templates, available_raw_data) # set variable holding if we have files attached to all raw data or not raw_files = True if available_raw_data else False for r in available_raw_data: if not r.get_filepaths(): raw_files = False # set variable holding if we have all prep templates or not prep_templates = True if available_prep_templates else False for key, val in viewitems(available_prep_templates): if not val: prep_templates = False # other general vars, note that we create the select options here # so we do not have to loop several times over them in the template data_types = sorted(viewitems(get_data_types()), key=itemgetter(1)) data_types = ['<option value="%s">%s</option>' % (v, k) for k, v in data_types] filetypes = sorted(viewitems(get_filetypes()), key=itemgetter(1)) filetypes = ['<option value="%s">%s</option>' % (v, k) for k, v in filetypes] other_studies_rd = yield Task(self.get_raw_data_from_other_studies, user, study) other_studies_rd = ['<option value="%s">%s</option>' % (k, "id: %d, study: %s" % (k, v)) for k, v in viewitems(other_studies_rd)] ontology = Ontology(convert_to_id('ENA', 'ontology')) # make "Other" show at the bottom of the drop down menu ena_terms = [] for v in sorted(ontology.terms): if v != 'Other': ena_terms.append('<option value="%s">%s</option>' % (v, v)) ena_terms.append('<option value="Other">Other</option>') # New Type is for users to add a new user-defined investigation type user_defined_terms = ontology.user_defined_terms + ['New Type'] princ_inv = StudyPerson(study.info['principal_investigator_id']) pi_link = study_person_linkifier((princ_inv.email, princ_inv.name)) if SampleTemplate.exists(study.id): sample_templates = SampleTemplate(study.id).get_filepaths() else: sample_templates = [] self.render('study_description.html', user=self.current_user, study_title=study.title, study_info=study.info, study_id=study.id, filetypes=''.join(filetypes), user_level=user.level, data_types=''.join(data_types), available_raw_data=available_raw_data, available_prep_templates=available_prep_templates, ste=SampleTemplate.exists(study.id), study_status=study.status, filepath_types=''.join(fts), ena_terms=''.join(ena_terms), tab_to_display=tab_to_display, level=msg_level, message=msg, prep_templates=prep_templates, raw_files=raw_files, can_upload=check_access(user, study, no_public=True), other_studies_rd=''.join(other_studies_rd), user_defined_terms=user_defined_terms, files=get_files_from_uploads_folders(str(study.id)), is_public=study.status == 'public', pmids=", ".join([pubmed_linkifier([pmid]) for pmid in study.pmids]), principal_investigator=pi_link, is_local_request=is_local_request, sample_templates=sample_templates)