def test_get_filepath_types(self):
"""Tests that get_filepath_types works with valid arguments"""
obs = get_filepath_types()
exp = {
'raw_forward_seqs': 1,
'raw_reverse_seqs': 2,
'raw_barcodes': 3,
'preprocessed_fasta': 4,
'preprocessed_fastq': 5,
'preprocessed_demux': 6,
'biom': 7,
'directory': 8,
'plain_text': 9,
'reference_seqs': 10,
'reference_tax': 11,
'reference_tree': 12,
'log': 13,
'sample_template': 14,
'prep_template': 15,
'qiime_map': 16,
}
exp = dict(
self.conn_handler.execute_fetchall(
"SELECT filepath_type,filepath_type_id FROM qiita.filepath_type"
))
self.assertEqual(obs, exp)
obs = get_filepath_types(key='filepath_type_id')
exp = {v: k for k, v in exp.items()}
self.assertEqual(obs, exp)
def test_get_filepath_types(self):
"""Tests that get_filepath_types works with valid arguments"""
obs = get_filepath_types()
exp = {
"raw_forward_seqs": 1,
"raw_reverse_seqs": 2,
"raw_barcodes": 3,
"preprocessed_fasta": 4,
"preprocessed_fastq": 5,
"preprocessed_demux": 6,
"biom": 7,
"directory": 8,
"plain_text": 9,
"reference_seqs": 10,
"reference_tax": 11,
"reference_tree": 12,
"log": 13,
"sample_template": 14,
"prep_template": 15,
"qiime_map": 16,
}
exp = dict(self.conn_handler.execute_fetchall("SELECT filepath_type,filepath_type_id FROM qiita.filepath_type"))
self.assertEqual(obs, exp)
obs = get_filepath_types(key="filepath_type_id")
exp = {v: k for k, v in exp.items()}
self.assertEqual(obs, exp)
def test_get_filepath_types(self):
"""Tests that get_filepath_types works with valid arguments"""
obs = get_filepath_types()
exp = {'raw_sequences': 1, 'raw_barcodes': 2, 'raw_spectra': 3,
'preprocessed_sequences': 4, 'preprocessed_sequences_qual': 5,
'biom': 6, 'directory': 7, 'plain_text': 8}
self.assertEqual(obs, exp)
obs = get_filepath_types(key='filepath_type_id')
exp = {v: k for k, v in exp.items()}
self.assertEqual(obs, exp)
def test_get_filepath_types(self):
"""Tests that get_filepath_types works with valid arguments"""
obs = get_filepath_types()
exp = {'raw_forward_seqs': 1, 'raw_reverse_seqs': 2,
'raw_barcodes': 3, 'preprocessed_fasta': 4,
'preprocessed_fastq': 5, 'preprocessed_demux': 6, 'biom': 7,
'directory': 8, 'plain_text': 9, 'reference_seqs': 10,
'reference_tax': 11, 'reference_tree': 12, 'log': 13,
'sample_template': 14, 'prep_template': 15, 'qiime_map': 16}
self.assertEqual(obs, exp)
obs = get_filepath_types(key='filepath_type_id')
exp = {v: k for k, v in exp.items()}
self.assertEqual(obs, exp)
def test_get_filepath_types(self):
"""Tests that get_filepath_types works with valid arguments"""
obs = get_filepath_types()
exp = {
'raw_sequences': 1,
'raw_barcodes': 2,
'raw_spectra': 3,
'preprocessed_sequences': 4,
'preprocessed_sequences_qual': 5,
'biom': 6,
'directory': 7,
'plain_text': 8,
'reference_seqs': 9,
'reference_tax': 10,
'reference_tree': 11
}
self.assertEqual(obs, exp)
obs = get_filepath_types(key='filepath_type_id')
exp = {v: k for k, v in exp.items()}
self.assertEqual(obs, exp)
def test_get_filepath_types_fail(self):
"""Tests that get_Filetypes fails with invalid argument"""
with self.assertRaises(QiitaDBColumnError):
get_filepath_types(key='invalid')
def test_get_filepath_types_fail(self):
"""Tests that get_Filetypes fails with invalid argument"""
with self.assertRaises(QiitaDBColumnError):
get_filepath_types(key='invalid')
from qiita_pet.util import convert_text_html
from qiita_db.study import Study
from qiita_db.data import RawData
from qiita_db.ontology import Ontology
from qiita_db.metadata_template import (PrepTemplate, TARGET_GENE_DATA_TYPES,
PREP_TEMPLATE_COLUMNS_TARGET_GENE)
from qiita_db.parameters import (Preprocessed454Params,
PreprocessedIlluminaParams)
from qiita_pet.util import STATUS_STYLER, is_localhost
from qiita_pet.handlers.util import download_link_or_path
from .base_uimodule import BaseUIModule
from qiita_core.util import execute_as_transaction
filepath_types = [k.split('_', 1)[1].replace('_', ' ')
for k in get_filepath_types()
if k.startswith('raw_')]
fp_type_by_ft = defaultdict(
lambda: filepath_types, SFF=['sff'], FASTA=['fasta', 'qual'],
FASTQ=['barcodes', 'forward seqs', 'reverse seqs'],
FASTA_Sanger=['fasta'],
per_sample_FASTQ=['forward seqs', 'reverse seqs'])
@execute_as_transaction
def _get_accessible_raw_data(user):
"""Retrieves a tuple of raw_data_id and one study title for that
raw_data
"""
d = {}
accessible_studies = user.user_studies.union(user.shared_studies)
# Distributed under the terms of the BSD 3-clause License.
#
# The full license is in the file LICENSE, distributed with this software.
# -----------------------------------------------------------------------------
from os.path import basename
from collections import defaultdict
from qiita_db.util import get_filepath_types
from qiita_pet.util import STATUS_STYLER
from .base_uimodule import BaseUIModule
from qiita_core.util import execute_as_transaction
filepath_types = [k.split('_', 1)[1].replace('_', ' ')
for k in get_filepath_types()
if k.startswith('raw_')]
fp_type_by_ft = defaultdict(
lambda: filepath_types, SFF=['sff'], FASTA=['fasta', 'qual'],
FASTQ=['barcodes', 'forward seqs', 'reverse seqs'],
FASTA_Sanger=['fasta'],
per_sample_FASTQ=['forward seqs', 'reverse seqs'])
@execute_as_transaction
def _get_accessible_raw_data(user):
"""Retrieves a tuple of raw_data_id and one study title for that
raw_data
"""
d = {}
accessible_studies = user.user_studies.union(user.shared_studies)
def render(self, study, raw_data, full_access):
user = self.current_user
study_status = study.status
user_level = user.level
raw_data_id = raw_data.id
files = [f for _, f in get_files_from_uploads_folders(str(study.id))]
# Get the available prep template data types
data_types = sorted(viewitems(get_data_types()), key=itemgetter(1))
# Get all the ENA terms for the investigation type
ontology = Ontology(convert_to_id('ENA', 'ontology'))
# make "Other" show at the bottom of the drop down menu
ena_terms = []
for v in sorted(ontology.terms):
if v != 'Other':
ena_terms.append('<option value="%s">%s</option>' % (v, v))
ena_terms.append('<option value="Other">Other</option>')
# New Type is for users to add a new user-defined investigation type
user_defined_terms = ontology.user_defined_terms + ['New Type']
# Get all the information about the prep templates
available_prep_templates = []
for p in sorted(raw_data.prep_templates):
if PrepTemplate.exists(p):
pt = PrepTemplate(p)
# if the prep template doesn't belong to this study, skip
if (study.id == pt.study_id and
(full_access or pt.status == 'public')):
available_prep_templates.append(pt)
# getting filepath_types
if raw_data.filetype == 'SFF':
fts = ['sff']
elif raw_data.filetype == 'FASTA':
fts = ['fasta', 'qual']
elif raw_data.filetype == 'FASTQ':
fts = ['barcodes', 'forward seqs', 'reverse seqs']
else:
fts = [k.split('_', 1)[1].replace('_', ' ')
for k in get_filepath_types() if k.startswith('raw_')]
# The raw data can be edited (e.i. adding prep templates and files)
# only if the study is sandboxed or the current user is an admin
is_editable = study_status == 'sandbox' or user_level == 'admin'
# Get the files linked with the raw_data
raw_data_files = raw_data.get_filepaths()
# Get the status of the data linking
raw_data_link_status = raw_data.link_filepaths_status
# By default don't show the unlink button
show_unlink_btn = False
# By default disable the the link file button
disable_link_btn = True
# Define the message for the link status
if raw_data_link_status == 'linking':
link_msg = "Linking files..."
elif raw_data_link_status == 'unlinking':
link_msg = "Unlinking files..."
else:
# The link button is only disable if raw data link status is
# linking or unlinking, so we can enable it here
disable_link_btn = False
# The unlink button is only shown if the study is editable, the raw
# data linking status is not in linking or unlinking, and there are
# files attached to the raw data. At this point, we are sure that
# the raw data linking status is not in linking or unlinking so we
# still need to check if it is editable or there are files attached
show_unlink_btn = is_editable and raw_data_files
if raw_data_link_status.startswith('failed'):
link_msg = "Error (un)linking files: %s" % raw_data_link_status
else:
link_msg = ""
# Get the raw_data filetype
raw_data_filetype = raw_data.filetype
return self.render_string(
"study_description_templates/raw_data_editor_tab.html",
study_id=study.id,
study_status=study_status,
user_level=user_level,
raw_data_id=raw_data_id,
files=files,
data_types=data_types,
ena_terms=ena_terms,
user_defined_terms=user_defined_terms,
available_prep_templates=available_prep_templates,
filepath_types=fts,
is_editable=is_editable,
show_unlink_btn=show_unlink_btn,
link_msg=link_msg,
raw_data_files=raw_data_files,
raw_data_filetype=raw_data_filetype,
disable_link_btn=disable_link_btn)
def render(self, study, raw_data):
user = self.current_user
study_status = study.status
user_level = user.level
raw_data_id = raw_data.id
files = [f for _, f in get_files_from_uploads_folders(str(study.id))]
# Get the available prep template data types
data_types = sorted(viewitems(get_data_types()), key=itemgetter(1))
# Get all the ENA terms for the investigation type
ontology = Ontology(convert_to_id('ENA', 'ontology'))
# make "Other" show at the bottom of the drop down menu
ena_terms = []
for v in sorted(ontology.terms):
if v != 'Other':
ena_terms.append('<option value="%s">%s</option>' % (v, v))
ena_terms.append('<option value="Other">Other</option>')
# New Type is for users to add a new user-defined investigation type
user_defined_terms = ontology.user_defined_terms + ['New Type']
# Get all the information about the prep templates
available_prep_templates = []
for p in sorted(raw_data.prep_templates):
if PrepTemplate.exists(p):
pt = PrepTemplate(p)
# if the prep template doesn't belong to this study, skip
if study.id == pt.study_id:
available_prep_templates.append(pt)
# getting filepath_types
if raw_data.filetype == 'SFF':
fts = ['sff']
elif raw_data.filetype == 'FASTA':
fts = ['fasta', 'qual']
elif raw_data.filetype == 'FASTQ':
fts = ['barcodes', 'forward seqs', 'reverse seqs']
else:
fts = [
k.split('_', 1)[1].replace('_', ' ')
for k in get_filepath_types() if k.startswith('raw_')
]
# The raw data can be edited (e.i. adding prep templates and files)
# only if the study is sandboxed or the current user is an admin
is_editable = study_status == 'sandbox' or user_level == 'admin'
# Get the files linked with the raw_data
raw_data_files = raw_data.get_filepaths()
# Get the status of the data linking
raw_data_link_status = raw_data.link_filepaths_status
# By default don't show the unlink button
show_unlink_btn = False
# By default disable the the link file button
disable_link_btn = True
# Define the message for the link status
if raw_data_link_status == 'linking':
link_msg = "Linking files..."
elif raw_data_link_status == 'unlinking':
link_msg = "Unlinking files..."
else:
# The link button is only disable if raw data link status is
# linking or unlinking, so we can enable it here
disable_link_btn = False
# The unlink button is only shown if the study is editable, the raw
# data linking status is not in linking or unlinking, and there are
# files attached to the raw data. At this point, we are sure that
# the raw data linking status is not in linking or unlinking so we
# still need to check if it is editable or there are files attached
show_unlink_btn = is_editable and raw_data_files
if raw_data_link_status.startswith('failed'):
link_msg = "Error (un)linking files: %s" % raw_data_link_status
else:
link_msg = ""
# Get the raw_data filetype
raw_data_filetype = raw_data.filetype
return self.render_string(
"study_description_templates/raw_data_editor_tab.html",
study_id=study.id,
study_status=study_status,
user_level=user_level,
raw_data_id=raw_data_id,
files=files,
data_types=data_types,
ena_terms=ena_terms,
user_defined_terms=user_defined_terms,
available_prep_templates=available_prep_templates,
filepath_types=fts,
is_editable=is_editable,
show_unlink_btn=show_unlink_btn,
link_msg=link_msg,
raw_data_files=raw_data_files,
raw_data_filetype=raw_data_filetype,
disable_link_btn=disable_link_btn)
def display_template(self, study, msg, msg_level, tab_to_display=""):
"""Simple function to avoid duplication of code"""
# Check if the request came from a local source
is_local_request = ('localhost' in self.request.headers['host'] or
'127.0.0.1' in self.request.headers['host'])
# getting raw filepath_ types
fts = [k.split('_', 1)[1].replace('_', ' ')
for k in get_filepath_types() if k.startswith('raw_')]
fts = ['<option value="%s">%s</option>' % (f, f) for f in fts]
user = User(self.current_user)
# getting the RawData and its prep templates
available_raw_data = yield Task(self.get_raw_data, study.raw_data())
available_prep_templates = yield Task(self.get_prep_templates,
available_raw_data)
# set variable holding if we have files attached to all raw data or not
raw_files = True if available_raw_data else False
for r in available_raw_data:
if not r.get_filepaths():
raw_files = False
# set variable holding if we have all prep templates or not
prep_templates = True if available_prep_templates else False
for key, val in viewitems(available_prep_templates):
if not val:
prep_templates = False
# other general vars, note that we create the select options here
# so we do not have to loop several times over them in the template
data_types = sorted(viewitems(get_data_types()), key=itemgetter(1))
data_types = ['<option value="%s">%s</option>' % (v, k)
for k, v in data_types]
filetypes = sorted(viewitems(get_filetypes()), key=itemgetter(1))
filetypes = ['<option value="%s">%s</option>' % (v, k)
for k, v in filetypes]
other_studies_rd = yield Task(self.get_raw_data_from_other_studies,
user, study)
other_studies_rd = ['<option value="%s">%s</option>' % (k,
"id: %d, study: %s" % (k, v))
for k, v in viewitems(other_studies_rd)]
ontology = Ontology(convert_to_id('ENA', 'ontology'))
# make "Other" show at the bottom of the drop down menu
ena_terms = []
for v in sorted(ontology.terms):
if v != 'Other':
ena_terms.append('<option value="%s">%s</option>' % (v, v))
ena_terms.append('<option value="Other">Other</option>')
# New Type is for users to add a new user-defined investigation type
user_defined_terms = ontology.user_defined_terms + ['New Type']
princ_inv = StudyPerson(study.info['principal_investigator_id'])
pi_link = study_person_linkifier((princ_inv.email, princ_inv.name))
if SampleTemplate.exists(study.id):
sample_templates = SampleTemplate(study.id).get_filepaths()
else:
sample_templates = []
self.render('study_description.html', user=self.current_user,
study_title=study.title, study_info=study.info,
study_id=study.id, filetypes=''.join(filetypes),
user_level=user.level, data_types=''.join(data_types),
available_raw_data=available_raw_data,
available_prep_templates=available_prep_templates,
ste=SampleTemplate.exists(study.id),
study_status=study.status,
filepath_types=''.join(fts), ena_terms=''.join(ena_terms),
tab_to_display=tab_to_display, level=msg_level,
message=msg, prep_templates=prep_templates,
raw_files=raw_files,
can_upload=check_access(user, study, no_public=True),
other_studies_rd=''.join(other_studies_rd),
user_defined_terms=user_defined_terms,
files=get_files_from_uploads_folders(str(study.id)),
is_public=study.status == 'public',
pmids=", ".join([pubmed_linkifier([pmid])
for pmid in study.pmids]),
principal_investigator=pi_link,
is_local_request=is_local_request,
sample_templates=sample_templates)
from future.utils import viewitems
from qiita_db.util import (get_artifact_types, get_files_from_uploads_folders,
get_data_types, convert_to_id, get_filepath_types)
from qiita_db.software import Command
from qiita_db.ontology import Ontology
from qiita_db.metadata_template.constants import (
TARGET_GENE_DATA_TYPES, PREP_TEMPLATE_COLUMNS_TARGET_GENE)
from qiita_pet.util import (STATUS_STYLER, is_localhost, EBI_LINKIFIER,
get_artifact_processing_status)
from qiita_pet.handlers.util import download_link_or_path
from .base_uimodule import BaseUIModule
from qiita_core.util import execute_as_transaction
filepath_types = [
k.split('_', 1)[1].replace('_', ' ') for k in get_filepath_types()
if k.startswith('raw_')
]
fp_type_by_ft = defaultdict(lambda: filepath_types,
SFF=['sff'],
FASTA=['fasta', 'qual'],
FASTQ=['barcodes', 'forward seqs', 'reverse seqs'],
FASTA_Sanger=['fasta'],
per_sample_FASTQ=['forward seqs', 'reverse seqs'])
@execute_as_transaction
def _get_accessible_raw_data(user):
"""Retrieves a tuple of raw_data_id and one study title for that
raw_data
"""
