def render(self, raw_data_id, prep_template, study, files):
rd = RawData(raw_data_id)
raw_data_files = [(basename(fp), fp_type[4:])
for _, fp, fp_type in rd.get_filepaths()]
filetype = rd.filetype
fp_types = fp_type_by_ft[filetype]
raw_data_link_status = rd.link_filepaths_status
show_buttons = rd.status(study) == 'sandbox'
link_msg = ""
if show_buttons:
# Define the message for the link status
if raw_data_link_status == 'linking':
link_msg = "Linking files..."
show_buttons = False
elif raw_data_link_status == 'unlinking':
link_msg = "Unlinking files..."
show_buttons = False
elif raw_data_link_status.startswith('failed'):
link_msg = "Error (un)linking files: %s" % raw_data_link_status
link_msg = convert_text_html(link_msg)
return self.render_string(
"study_description_templates/raw_data_info.html",
rd_id=raw_data_id,
rd_filetype=rd.filetype,
raw_data_files=raw_data_files,
prep_template_id=prep_template.id,
files=files,
filepath_types=fp_types,
filetype=filetype,
link_msg=link_msg,
show_buttons=show_buttons)
def test_get_filepaths(self):
"""Correctly returns the filepaths to the raw files"""
rd = RawData(1)
obs = rd.get_filepaths()
exp = [
(join(self.db_test_raw_dir, '1_s_G1_L001_sequences.fastq.gz'), 1),
(join(self.db_test_raw_dir,
'1_s_G1_L001_sequences_barcodes.fastq.gz'), 2)]
self.assertEqual(obs, exp)
def test_get_filepaths(self):
"""Correctly returns the filepaths to the raw files"""
rd = RawData(1)
obs = rd.get_filepaths()
exp = [(join(self.db_test_raw_dir,
'1_s_G1_L001_sequences.fastq.gz'), "raw_sequences"),
(join(self.db_test_raw_dir,
'1_s_G1_L001_sequences_barcodes.fastq.gz'),
"raw_barcodes")]
self.assertEqual(obs, exp)
WHERE raw_data_id NOT IN (
SELECT DISTINCT raw_data_id FROM qiita.prep_template);"""
rd_ids = [x[0] for x in conn_handler.execute_fetchall(sql)]
# We will delete those RawData. However, if they have files attached, we should
# move them to the uploads folder of the study
sql_detach = """DELETE FROM qiita.study_raw_data
WHERE raw_data_id = %s AND study_id = %s"""
sql_unlink = "DELETE FROM qiita.raw_filepath WHERE raw_data_id = %s"
sql_delete = "DELETE FROM qiita.raw_data WHERE raw_data_id = %s"
sql_studies = """SELECT study_id FROM qiita.study_raw_data
WHERE raw_data_id = %s"""
move_files = []
for rd_id in rd_ids:
rd = RawData(rd_id)
filepaths = rd.get_filepaths()
studies = [s[0] for s in conn_handler.execute_fetchall(sql_studies,
(rd_id,))]
if filepaths:
# we need to move the files to a study. We chose the one with lower
# study id. Currently there is no case in the live database in which a
# RawData with no prep templates is attached to more than one study,
# but I think it is better to normalize this just in case
move_files.append((min(studies), filepaths))
# To delete the RawData we first need to unlink all the files
conn_handler.add_to_queue(queue, sql_unlink, (rd_id,))
# Then, remove the raw data from all the studies
for st_id in studies:
conn_handler.add_to_queue(queue, sql_detach, (rd_id, st_id))
def render(self, study, prep_template, full_access, ena_terms,
user_defined_terms):
user = self.current_user
is_local_request = is_localhost(self.request.headers['host'])
template_fps = []
qiime_fps = []
# Unfortunately, both the prep template and the qiime mapping files
# have the sample type. The way to differentiate them is if we have
# the substring 'qiime' in the basename
for id_, fp in prep_template.get_filepaths():
if 'qiime' in basename(fp):
qiime_fps.append(
download_link_or_path(
is_local_request, fp, id_, 'Qiime mapping'))
else:
template_fps.append(
download_link_or_path(
is_local_request, fp, id_, 'Prep template'))
# Since get_filepaths returns the paths sorted from newest to oldest,
# the first in both list is the latest one
current_template_fp = template_fps[0]
current_qiime_fp = qiime_fps[0]
if len(template_fps) > 1:
show_old_templates = True
old_templates = template_fps[1:]
else:
show_old_templates = False
old_templates = None
if len(qiime_fps) > 1:
show_old_qiime_fps = True
old_qiime_fps = qiime_fps[1:]
else:
show_old_qiime_fps = False
old_qiime_fps = None
filetypes = sorted(
((ft, ft_id, fp_type_by_ft[ft])
for ft, ft_id in viewitems(get_filetypes())),
key=itemgetter(1))
files = [f for _, f in get_files_from_uploads_folders(str(study.id))]
other_studies_rd = sorted(viewitems(
_get_accessible_raw_data(user)))
# A prep template can be modified if its status is sandbox
is_editable = prep_template.status == 'sandbox'
raw_data_id = prep_template.raw_data
preprocess_options = []
preprocessed_data = None
show_preprocess_btn = True
no_preprocess_msg = None
if raw_data_id:
rd = RawData(raw_data_id)
rd_ft = rd.filetype
# If the prep template has a raw data associated, it can be
# preprocessed. Retrieve the pre-processing parameters
if rd_ft in ('SFF', 'FASTA'):
param_iter = Preprocessed454Params.iter()
elif rd_ft == 'FASTQ':
param_iter = [pip for pip in PreprocessedIlluminaParams.iter()
if pip.values['barcode_type'] != 'not-barcoded']
elif rd_ft == 'per_sample_FASTQ':
param_iter = [pip for pip in PreprocessedIlluminaParams.iter()
if pip.values['barcode_type'] == 'not-barcoded']
else:
raise NotImplementedError(
"Pre-processing of %s files currently not supported."
% rd_ft)
preprocess_options = []
for param in param_iter:
text = ("<b>%s:</b> %s" % (k, v)
for k, v in viewitems(param.values))
preprocess_options.append((param.id,
param.name,
'<br>'.join(text)))
preprocessed_data = prep_template.preprocessed_data
# Check if the template have all the required columns for
# preprocessing
raw_data_files = rd.get_filepaths()
if len(raw_data_files) == 0:
show_preprocess_btn = False
no_preprocess_msg = (
"Preprocessing disabled because there are no files "
"linked with the Raw Data")
else:
if prep_template.data_type() in TARGET_GENE_DATA_TYPES:
raw_forward_fps = [fp for _, fp, ftype in raw_data_files
if ftype == 'raw_forward_seqs']
key = ('demultiplex_multiple' if len(raw_forward_fps) > 1
else 'demultiplex')
missing_cols = prep_template.check_restrictions(
[PREP_TEMPLATE_COLUMNS_TARGET_GENE[key]])
if rd_ft == 'per_sample_FASTQ':
show_preprocess_btn = 'run_prefix' not in missing_cols
else:
show_preprocess_btn = len(missing_cols) == 0
no_preprocess_msg = None
if not show_preprocess_btn:
no_preprocess_msg = (
"Preprocessing disabled due to missing columns in "
"the prep template: %s" % ', '.join(missing_cols))
preprocessing_status = prep_template.preprocessing_status
return self.render_string(
"study_description_templates/prep_template_info_tab.html",
pt_id=prep_template.id,
study_id=study.id,
raw_data=raw_data_id,
current_template_fp=current_template_fp,
current_qiime_fp=current_qiime_fp,
show_old_templates=show_old_templates,
old_templates=old_templates,
show_old_qiime_fps=show_old_qiime_fps,
old_qiime_fps=old_qiime_fps,
filetypes=filetypes,
files=files,
other_studies_rd=other_studies_rd,
prep_template=prep_template,
study=study,
ena_terms=ena_terms,
user_defined_terms=user_defined_terms,
investigation_type=prep_template.investigation_type,
is_editable=is_editable,
preprocess_options=preprocess_options,
preprocessed_data=preprocessed_data,
preprocessing_status=preprocessing_status,
show_preprocess_btn=show_preprocess_btn,
no_preprocess_msg=no_preprocess_msg)
