Esempio n. 1
0
    def test_get_data_types(self):
        """Tests that get_data_types works with valid arguments"""
        obs = get_data_types()
        exp = {"16S": 1, "18S": 2, "ITS": 3, "Proteomic": 4, "Metabolomic": 5, "Metagenomic": 6}
        self.assertEqual(obs, exp)

        obs = get_data_types(key="data_type_id")
        exp = {v: k for k, v in exp.items()}
        self.assertEqual(obs, exp)
Esempio n. 2
0
    def test_get_data_types(self):
        """Tests that get_data_types works with valid arguments"""
        obs = get_data_types()
        exp = {'16S': 1, '18S': 2, 'ITS': 3, 'Proteomic': 4, 'Metabolomic': 5,
               'Metagenomic': 6}
        self.assertEqual(obs, exp)

        obs = get_data_types(key='data_type_id')
        exp = {v: k for k, v in exp.items()}
        self.assertEqual(obs, exp)
Esempio n. 3
0
    def test_get_data_types(self):
        """Tests that get_data_types works with valid arguments"""
        obs = get_data_types()
        exp = {
            '16S': 1,
            '18S': 2,
            'ITS': 3,
            'Proteomic': 4,
            'Metabolomic': 5,
            'Metagenomic': 6
        }
        self.assertEqual(obs, exp)

        obs = get_data_types(key='data_type_id')
        exp = {v: k for k, v in exp.items()}
        self.assertEqual(obs, exp)
Esempio n. 4
0
    def render(self, study, full_access):
        files = [f for _, f in get_files_from_uploads_folders(str(study.id))
                 if f.endswith(('txt', 'tsv'))]
        data_types = sorted(viewitems(get_data_types()), key=itemgetter(1))
        prep_templates_info = [
            res for res in _template_generator(study, full_access)]
        # Get all the ENA terms for the investigation type
        ontology = Ontology(convert_to_id('ENA', 'ontology'))
        # make "Other" show at the bottom of the drop down menu
        ena_terms = []
        for v in sorted(ontology.terms):
            if v != 'Other':
                ena_terms.append('<option value="%s">%s</option>' % (v, v))
        ena_terms.append('<option value="Other">Other</option>')

        # New Type is for users to add a new user-defined investigation type
        user_defined_terms = ontology.user_defined_terms + ['New Type']

        return self.render_string(
            "study_description_templates/prep_template_tab.html",
            files=files,
            data_types=data_types,
            available_prep_templates=prep_templates_info,
            ena_terms=ena_terms,
            user_defined_terms=user_defined_terms,
            study=study,
            full_access=full_access)
Esempio n. 5
0
    def render(self, study, full_access):
        files = [
            f for _, f in get_files_from_uploads_folders(str(study.id))
            if f.endswith(('txt', 'tsv'))
        ]
        data_types = sorted(viewitems(get_data_types()), key=itemgetter(1))

        prep_templates_info = [
            res for res in _template_generator(study, full_access)
        ]
        # Get all the ENA terms for the investigation type
        ontology = Ontology(convert_to_id('ENA', 'ontology'))
        # make "Other" show at the bottom of the drop down menu
        ena_terms = []
        for v in sorted(ontology.terms):
            if v != 'Other':
                ena_terms.append('<option value="%s">%s</option>' % (v, v))
        ena_terms.append('<option value="Other">Other</option>')

        # New Type is for users to add a new user-defined investigation type
        user_defined_terms = ontology.user_defined_terms + ['New Type']

        return self.render_string(
            "study_description_templates/prep_template_tab.html",
            files=files,
            data_types=data_types,
            available_prep_templates=prep_templates_info,
            ena_terms=ena_terms,
            user_defined_terms=user_defined_terms,
            study=study,
            full_access=full_access)
Esempio n. 6
0
    def render(self, study):
        study_info = study.info
        id = study.id
        abstract = study_info['study_abstract']
        description = study_info['study_description']
        pmids = ", ".join([pubmed_linkifier([pmid]) for pmid in study.pmids])
        princ_inv = StudyPerson(study_info['principal_investigator_id'])
        pi_link = study_person_linkifier((princ_inv.email, princ_inv.name))
        number_samples_promised = study_info['number_samples_promised']
        number_samples_collected = study_info['number_samples_collected']
        metadata_complete = study_info['metadata_complete']
        data_types = sorted(viewitems(get_data_types()), key=itemgetter(1))

        # Retrieve the files from the uploads folder, so the user can choose
        # the sample template of the study. Filter them to only include the
        # ones that ends with 'txt' or 'tsv'.
        files = [f for _, f in get_files_from_uploads_folders(str(study.id))
                 if f.endswith(('txt', 'tsv'))]

        # If the sample template exists, retrieve all its filepaths
        if SampleTemplate.exists(study.id):
            sample_templates = SampleTemplate(study.id).get_filepaths()
        else:
            # If the sample template does not exist, just pass an empty list
            sample_templates = []

        # Check if the request came from a local source
        is_local_request = is_localhost(self.request.headers['host'])

        # The user can choose the sample template only if the study is
        # sandboxed or the current user is an admin
        show_select_sample = (
            study.status == 'sandbox' or self.current_user.level == 'admin')

        # Ebi information
        ebi_status = study.ebi_submission_status
        ebi_accession = study.ebi_study_accession
        if ebi_accession:
            ebi_accession = (EBI_LINKIFIER.format(ebi_accession))

        return self.render_string(
            "study_description_templates/study_information_tab.html",
            abstract=abstract,
            description=description,
            id=id,
            pmids=pmids,
            principal_investigator=pi_link,
            number_samples_promised=number_samples_promised,
            number_samples_collected=number_samples_collected,
            metadata_complete=metadata_complete,
            show_select_sample=show_select_sample,
            files=files,
            study_id=study.id,
            sample_templates=sample_templates,
            is_local_request=is_local_request,
            data_types=data_types,
            ebi_status=ebi_status,
            ebi_accession=ebi_accession)
Esempio n. 7
0
    def get(self):
        data = self.get_argument("data", None)
        study_id = self.get_argument("study_id", None)
        data_type = self.get_argument("data_type", None)
        dtypes = get_data_types().keys()

        if data is None or study_id is None or data not in ('raw', 'biom'):
            raise HTTPError(422,
                            reason='You need to specify both data (the '
                            'data type you want to download - raw/biom) and '
                            'study_id')
        elif data_type is not None and data_type not in dtypes:
            raise HTTPError(422,
                            reason='Not a valid data_type. Valid types '
                            'are: %s' % ', '.join(dtypes))
        else:
            study_id = int(study_id)
            try:
                study = Study(study_id)
            except QiitaDBUnknownIDError:
                raise HTTPError(422, reason='Study does not exist')
            else:
                public_raw_download = study.public_raw_download
                if study.status != 'public':
                    raise HTTPError(422,
                                    reason='Study is not public. If this '
                                    'is a mistake contact: '
                                    '*****@*****.**')
                elif data == 'raw' and not public_raw_download:
                    raise HTTPError(422,
                                    reason='No raw data access. If this '
                                    'is a mistake contact: '
                                    '*****@*****.**')
                else:
                    to_download = []
                    for a in study.artifacts(
                            dtype=data_type,
                            artifact_type='BIOM' if data == 'biom' else None):
                        if a.visibility != 'public':
                            continue
                        to_download.extend(self._list_artifact_files_nginx(a))

                    if not to_download:
                        raise HTTPError(422,
                                        reason='Nothing to download. If '
                                        'this is a mistake contact: '
                                        '*****@*****.**')
                    else:
                        self._write_nginx_file_list(to_download)

                        zip_fn = 'study_%d_%s_%s.zip' % (
                            study_id, data,
                            datetime.now().strftime('%m%d%y-%H%M%S'))

                        self._set_nginx_headers(zip_fn)
        self.finish()
Esempio n. 8
0
    def get(self):
        data = self.get_argument("data", None)
        study_id = self.get_argument("study_id",  None)
        prep_id = self.get_argument("prep_id",  None)
        data_type = self.get_argument("data_type",  None)
        dtypes = get_data_types().keys()

        templates = ['sample_information', 'prep_information']
        valid_data = ['raw', 'biom'] + templates

        to_download = []
        if data is None or (study_id is None and prep_id is None) or \
                data not in valid_data:
            raise HTTPError(422, reason='You need to specify both data (the '
                            'data type you want to download - %s) and '
                            'study_id or prep_id' % '/'.join(valid_data))
        elif data_type is not None and data_type not in dtypes:
            raise HTTPError(422, reason='Not a valid data_type. Valid types '
                            'are: %s' % ', '.join(dtypes))
        elif data in templates and prep_id is None and study_id is None:
            raise HTTPError(422, reason='If downloading a sample or '
                            'preparation file you need to define study_id or'
                            ' prep_id')
        elif data in templates:
            if data_type is not None:
                raise HTTPError(422, reason='If requesting an information '
                                'file you cannot specify the data_type')
            elif prep_id is not None and data == 'prep_information':
                fname = 'preparation_information_%s' % prep_id
                prep_id = int(prep_id)
                try:
                    infofile = PrepTemplate(prep_id)
                except QiitaDBUnknownIDError:
                    raise HTTPError(
                        422, reason='Preparation information does not exist')
            elif study_id is not None and data == 'sample_information':
                fname = 'sample_information_%s' % study_id
                study_id = int(study_id)
                try:
                    infofile = SampleTemplate(study_id)
                except QiitaDBUnknownIDError:
                    raise HTTPError(
                        422, reason='Sample information does not exist')
            else:
                raise HTTPError(422, reason='Review your parameters, not a '
                                'valid combination')
            x = retrieve_filepaths(
                infofile._filepath_table, infofile._id_column, infofile.id,
                sort='descending')[0]

            basedir_len = len(get_db_files_base_dir()) + 1
            fp = x['fp'][basedir_len:]
            to_download.append((fp, fp, str(x['checksum']), str(x['fp_size'])))
            self._write_nginx_file_list(to_download)

            zip_fn = '%s_%s.zip' % (
                fname, datetime.now().strftime('%m%d%y-%H%M%S'))
            self._set_nginx_headers(zip_fn)
        else:
            study_id = int(study_id)
            try:
                study = Study(study_id)
            except QiitaDBUnknownIDError:
                raise HTTPError(422, reason='Study does not exist')
            else:
                public_raw_download = study.public_raw_download
                if study.status != 'public':
                    raise HTTPError(404, reason='Study is not public. If this '
                                    'is a mistake contact: '
                                    '*****@*****.**')
                elif data == 'raw' and not public_raw_download:
                    raise HTTPError(422, reason='No raw data access. If this '
                                    'is a mistake contact: '
                                    '*****@*****.**')
                else:
                    # raw data
                    artifacts = [a for a in study.artifacts(dtype=data_type)
                                 if not a.parents]
                    # bioms
                    if data == 'biom':
                        artifacts = study.artifacts(
                            dtype=data_type, artifact_type='BIOM')
                    for a in artifacts:
                        if a.visibility != 'public':
                            continue
                        to_download.extend(self._list_artifact_files_nginx(a))

                if not to_download:
                    raise HTTPError(422, reason='Nothing to download. If '
                                    'this is a mistake contact: '
                                    '*****@*****.**')
                else:
                    self._write_nginx_file_list(to_download)

                    zip_fn = 'study_%d_%s_%s.zip' % (
                        study_id, data, datetime.now().strftime(
                            '%m%d%y-%H%M%S'))

                    self._set_nginx_headers(zip_fn)

        self.finish()
Esempio n. 9
0
    def render(self, study):
        study_info = study.info
        id = study.id
        abstract = study_info['study_abstract']
        description = study_info['study_description']
        publications = []
        for doi, pmid in study.publications:
            if doi is not None:
                publications.append(doi_linkifier([doi]))
            if pmid is not None:
                publications.append(pubmed_linkifier([pmid]))
        publications = ", ".join(publications)
        princ_inv = StudyPerson(study_info['principal_investigator_id'])
        pi_link = study_person_linkifier((princ_inv.email, princ_inv.name))
        number_samples_promised = study_info['number_samples_promised']
        number_samples_collected = study_info['number_samples_collected']
        metadata_complete = study_info['metadata_complete']

        data_types = sorted(viewitems(get_data_types()), key=itemgetter(1))

        # Retrieve the files from the uploads folder, so the user can choose
        # the sample template of the study. Filter them to only include the
        # ones that ends with 'txt' or 'tsv'.
        files = [
            f for _, f in get_files_from_uploads_folders(str(study.id))
            if f.endswith(('txt', 'tsv'))
        ]

        # If the sample template exists, retrieve all its filepaths
        if SampleTemplate.exists(study.id):
            sample_templates = SampleTemplate(study.id).get_filepaths()
        else:
            # If the sample template does not exist, just pass an empty list
            sample_templates = []

        # Check if the request came from a local source
        is_local_request = is_localhost(self.request.headers['host'])

        # The user can choose the sample template only if the study is
        # sandboxed or the current user is an admin
        show_select_sample = (study.status == 'sandbox'
                              or self.current_user.level == 'admin')

        # EBI information
        ebi_status = study.ebi_submission_status
        ebi_accession = study.ebi_study_accession
        if ebi_accession:
            ebi_accession = (EBI_LINKIFIER.format(ebi_accession))

        return self.render_string(
            "study_description_templates/study_information_tab.html",
            abstract=abstract,
            description=description,
            id=id,
            publications=publications,
            principal_investigator=pi_link,
            number_samples_promised=number_samples_promised,
            number_samples_collected=number_samples_collected,
            metadata_complete=metadata_complete,
            show_select_sample=show_select_sample,
            files=files,
            study_id=study.id,
            sample_templates=sample_templates,
            is_local_request=is_local_request,
            data_types=data_types,
            ebi_status=ebi_status,
            ebi_accession=ebi_accession)
Esempio n. 10
0
    def render(self, study, raw_data, full_access):
        user = self.current_user
        study_status = study.status
        user_level = user.level
        raw_data_id = raw_data.id
        files = [f for _, f in get_files_from_uploads_folders(str(study.id))]

        # Get the available prep template data types
        data_types = sorted(viewitems(get_data_types()), key=itemgetter(1))

        # Get all the ENA terms for the investigation type
        ontology = Ontology(convert_to_id('ENA', 'ontology'))
        # make "Other" show at the bottom of the drop down menu
        ena_terms = []
        for v in sorted(ontology.terms):
            if v != 'Other':
                ena_terms.append('<option value="%s">%s</option>' % (v, v))
        ena_terms.append('<option value="Other">Other</option>')

        # New Type is for users to add a new user-defined investigation type
        user_defined_terms = ontology.user_defined_terms + ['New Type']

        # Get all the information about the prep templates
        available_prep_templates = []
        for p in sorted(raw_data.prep_templates):
            if PrepTemplate.exists(p):
                pt = PrepTemplate(p)
                # if the prep template doesn't belong to this study, skip
                if (study.id == pt.study_id and
                        (full_access or pt.status == 'public')):
                    available_prep_templates.append(pt)

        # getting filepath_types
        if raw_data.filetype == 'SFF':
            fts = ['sff']
        elif raw_data.filetype == 'FASTA':
            fts = ['fasta', 'qual']
        elif raw_data.filetype == 'FASTQ':
            fts = ['barcodes', 'forward seqs', 'reverse seqs']
        else:
            fts = [k.split('_', 1)[1].replace('_', ' ')
                   for k in get_filepath_types() if k.startswith('raw_')]

        # The raw data can be edited (e.i. adding prep templates and files)
        # only if the study is sandboxed or the current user is an admin
        is_editable = study_status == 'sandbox' or user_level == 'admin'

        # Get the files linked with the raw_data
        raw_data_files = raw_data.get_filepaths()

        # Get the status of the data linking
        raw_data_link_status = raw_data.link_filepaths_status

        # By default don't show the unlink button
        show_unlink_btn = False
        # By default disable the the link file button
        disable_link_btn = True
        # Define the message for the link status
        if raw_data_link_status == 'linking':
            link_msg = "Linking files..."
        elif raw_data_link_status == 'unlinking':
            link_msg = "Unlinking files..."
        else:
            # The link button is only disable if raw data link status is
            # linking or unlinking, so we can enable it here
            disable_link_btn = False
            # The unlink button is only shown if the study is editable, the raw
            # data linking status is not in linking or unlinking, and there are
            # files attached to the raw data. At this  point, we are sure that
            # the raw data linking status is not in linking or unlinking so we
            # still need to check if it is editable or there are files attached
            show_unlink_btn = is_editable and raw_data_files
            if raw_data_link_status.startswith('failed'):
                link_msg = "Error (un)linking files: %s" % raw_data_link_status
            else:
                link_msg = ""

        # Get the raw_data filetype
        raw_data_filetype = raw_data.filetype

        return self.render_string(
            "study_description_templates/raw_data_editor_tab.html",
            study_id=study.id,
            study_status=study_status,
            user_level=user_level,
            raw_data_id=raw_data_id,
            files=files,
            data_types=data_types,
            ena_terms=ena_terms,
            user_defined_terms=user_defined_terms,
            available_prep_templates=available_prep_templates,
            filepath_types=fts,
            is_editable=is_editable,
            show_unlink_btn=show_unlink_btn,
            link_msg=link_msg,
            raw_data_files=raw_data_files,
            raw_data_filetype=raw_data_filetype,
            disable_link_btn=disable_link_btn)
Esempio n. 11
0
    def render(self, study, raw_data):
        user = self.current_user
        study_status = study.status
        user_level = user.level
        raw_data_id = raw_data.id
        files = [f for _, f in get_files_from_uploads_folders(str(study.id))]

        # Get the available prep template data types
        data_types = sorted(viewitems(get_data_types()), key=itemgetter(1))

        # Get all the ENA terms for the investigation type
        ontology = Ontology(convert_to_id('ENA', 'ontology'))
        # make "Other" show at the bottom of the drop down menu
        ena_terms = []
        for v in sorted(ontology.terms):
            if v != 'Other':
                ena_terms.append('<option value="%s">%s</option>' % (v, v))
        ena_terms.append('<option value="Other">Other</option>')

        # New Type is for users to add a new user-defined investigation type
        user_defined_terms = ontology.user_defined_terms + ['New Type']

        # Get all the information about the prep templates
        available_prep_templates = []
        for p in sorted(raw_data.prep_templates):
            if PrepTemplate.exists(p):
                pt = PrepTemplate(p)
                # if the prep template doesn't belong to this study, skip
                if study.id == pt.study_id:
                    available_prep_templates.append(pt)

        # getting filepath_types
        if raw_data.filetype == 'SFF':
            fts = ['sff']
        elif raw_data.filetype == 'FASTA':
            fts = ['fasta', 'qual']
        elif raw_data.filetype == 'FASTQ':
            fts = ['barcodes', 'forward seqs', 'reverse seqs']
        else:
            fts = [
                k.split('_', 1)[1].replace('_', ' ')
                for k in get_filepath_types() if k.startswith('raw_')
            ]

        # The raw data can be edited (e.i. adding prep templates and files)
        # only if the study is sandboxed or the current user is an admin
        is_editable = study_status == 'sandbox' or user_level == 'admin'

        # Get the files linked with the raw_data
        raw_data_files = raw_data.get_filepaths()

        # Get the status of the data linking
        raw_data_link_status = raw_data.link_filepaths_status

        # By default don't show the unlink button
        show_unlink_btn = False
        # By default disable the the link file button
        disable_link_btn = True
        # Define the message for the link status
        if raw_data_link_status == 'linking':
            link_msg = "Linking files..."
        elif raw_data_link_status == 'unlinking':
            link_msg = "Unlinking files..."
        else:
            # The link button is only disable if raw data link status is
            # linking or unlinking, so we can enable it here
            disable_link_btn = False
            # The unlink button is only shown if the study is editable, the raw
            # data linking status is not in linking or unlinking, and there are
            # files attached to the raw data. At this  point, we are sure that
            # the raw data linking status is not in linking or unlinking so we
            # still need to check if it is editable or there are files attached
            show_unlink_btn = is_editable and raw_data_files
            if raw_data_link_status.startswith('failed'):
                link_msg = "Error (un)linking files: %s" % raw_data_link_status
            else:
                link_msg = ""

        # Get the raw_data filetype
        raw_data_filetype = raw_data.filetype

        return self.render_string(
            "study_description_templates/raw_data_editor_tab.html",
            study_id=study.id,
            study_status=study_status,
            user_level=user_level,
            raw_data_id=raw_data_id,
            files=files,
            data_types=data_types,
            ena_terms=ena_terms,
            user_defined_terms=user_defined_terms,
            available_prep_templates=available_prep_templates,
            filepath_types=fts,
            is_editable=is_editable,
            show_unlink_btn=show_unlink_btn,
            link_msg=link_msg,
            raw_data_files=raw_data_files,
            raw_data_filetype=raw_data_filetype,
            disable_link_btn=disable_link_btn)
Esempio n. 12
0
    def display_template(self, study, msg, msg_level, tab_to_display=""):
        """Simple function to avoid duplication of code"""
        # Check if the request came from a local source
        is_local_request = ('localhost' in self.request.headers['host'] or
                            '127.0.0.1' in self.request.headers['host'])

        # getting raw filepath_ types
        fts = [k.split('_', 1)[1].replace('_', ' ')
               for k in get_filepath_types() if k.startswith('raw_')]
        fts = ['<option value="%s">%s</option>' % (f, f) for f in fts]

        user = User(self.current_user)
        # getting the RawData and its prep templates
        available_raw_data = yield Task(self.get_raw_data, study.raw_data())
        available_prep_templates = yield Task(self.get_prep_templates,
                                              available_raw_data)
        # set variable holding if we have files attached to all raw data or not
        raw_files = True if available_raw_data else False
        for r in available_raw_data:
            if not r.get_filepaths():
                raw_files = False

        # set variable holding if we have all prep templates or not
        prep_templates = True if available_prep_templates else False
        for key, val in viewitems(available_prep_templates):
            if not val:
                prep_templates = False
        # other general vars, note that we create the select options here
        # so we do not have to loop several times over them in the template
        data_types = sorted(viewitems(get_data_types()), key=itemgetter(1))
        data_types = ['<option value="%s">%s</option>' % (v, k)
                      for k, v in data_types]
        filetypes = sorted(viewitems(get_filetypes()), key=itemgetter(1))
        filetypes = ['<option value="%s">%s</option>' % (v, k)
                     for k, v in filetypes]
        other_studies_rd = yield Task(self.get_raw_data_from_other_studies,
                                      user, study)
        other_studies_rd = ['<option value="%s">%s</option>' % (k,
                            "id: %d, study: %s" % (k, v))
                            for k, v in viewitems(other_studies_rd)]

        ontology = Ontology(convert_to_id('ENA', 'ontology'))

        # make "Other" show at the bottom of the drop down menu
        ena_terms = []
        for v in sorted(ontology.terms):
            if v != 'Other':
                ena_terms.append('<option value="%s">%s</option>' % (v, v))
        ena_terms.append('<option value="Other">Other</option>')

        # New Type is for users to add a new user-defined investigation type
        user_defined_terms = ontology.user_defined_terms + ['New Type']
        princ_inv = StudyPerson(study.info['principal_investigator_id'])
        pi_link = study_person_linkifier((princ_inv.email, princ_inv.name))

        if SampleTemplate.exists(study.id):
            sample_templates = SampleTemplate(study.id).get_filepaths()
        else:
            sample_templates = []

        self.render('study_description.html', user=self.current_user,
                    study_title=study.title, study_info=study.info,
                    study_id=study.id, filetypes=''.join(filetypes),
                    user_level=user.level, data_types=''.join(data_types),
                    available_raw_data=available_raw_data,
                    available_prep_templates=available_prep_templates,
                    ste=SampleTemplate.exists(study.id),
                    study_status=study.status,
                    filepath_types=''.join(fts), ena_terms=''.join(ena_terms),
                    tab_to_display=tab_to_display, level=msg_level,
                    message=msg, prep_templates=prep_templates,
                    raw_files=raw_files,
                    can_upload=check_access(user, study, no_public=True),
                    other_studies_rd=''.join(other_studies_rd),
                    user_defined_terms=user_defined_terms,
                    files=get_files_from_uploads_folders(str(study.id)),
                    is_public=study.status == 'public',
                    pmids=", ".join([pubmed_linkifier([pmid])
                                     for pmid in study.pmids]),
                    principal_investigator=pi_link,
                    is_local_request=is_local_request,
                    sample_templates=sample_templates)