def add_raw_data(self, study, user, callback):
"""Adds an existing raw data to the study
Parameters
----------
study : Study
The current study object
user : User
The current user object
callback : function
The callback function to call with the results once the processing
is done
"""
msg = "Raw data successfully added"
msg_level = "success"
# Get the arguments to add the raw data
pt_id = self.get_argument('prep_template_id')
raw_data_id = self.get_argument('raw_data_id')
prep_template = PrepTemplate(pt_id)
raw_data = RawData(raw_data_id)
try:
prep_template.raw_data = raw_data
except QiitaDBError as e:
msg = html_error_message % ("adding the raw data",
str(raw_data_id), str(e))
msg = convert_text_html(msg)
callback((msg, msg_level, 'prep_template_tab', pt_id, None))
def test_convert_text_html(self):
test = ('<b>This is line</b>\nThis is another\n'
'This is a link: http://test.com')
exp = ('<b>This is line</b><br/>This is another<br/>'
'This is a link: <a href="http://test.com">http://test.com</a>')
obs = convert_text_html(test)
self.assertEqual(obs, exp)
def test_convert_text_html(self):
test = ('<b>This is line</b>\nThis is another\n'
'This is a link: http://test.com')
exp = ('<b>This is line</b><br/>This is another<br/>'
'This is a link: <a href="http://test.com">http://test.com</a>')
obs = convert_text_html(test)
self.assertEqual(obs, exp)
def render(self, raw_data_id, prep_template, study, files):
rd = RawData(raw_data_id)
raw_data_files = [(basename(fp), fp_type[4:])
for _, fp, fp_type in rd.get_filepaths()]
filetype = rd.filetype
fp_types = fp_type_by_ft[filetype]
raw_data_link_status = rd.link_filepaths_status
show_buttons = rd.status(study) == 'sandbox'
link_msg = ""
if show_buttons:
# Define the message for the link status
if raw_data_link_status == 'linking':
link_msg = "Linking files..."
show_buttons = False
elif raw_data_link_status == 'unlinking':
link_msg = "Unlinking files..."
show_buttons = False
elif raw_data_link_status.startswith('failed'):
link_msg = "Error (un)linking files: %s" % raw_data_link_status
link_msg = convert_text_html(link_msg)
return self.render_string(
"study_description_templates/raw_data_info.html",
rd_id=raw_data_id,
rd_filetype=rd.filetype,
raw_data_files=raw_data_files,
prep_template_id=prep_template.id,
files=files,
filepath_types=fp_types,
filetype=filetype,
link_msg=link_msg,
show_buttons=show_buttons)
def update_prep_template(self, study, user, callback):
"""Update a prep template from the POST method
Parameters
----------
study : Study
The current study object
user : User
The current user object
callback : function
The callback function to call with the results once the processing
is done
Raises
------
HTTPError
If the prep template file does not exists
"""
# If we are on this function, the arguments "prep_template_id",
# "update_prep_template_file" must defined. If not, let tornado
# raise its error
pt_id = int(self.get_argument('prep_template_id'))
prep_template = self.get_argument('update_prep_template_file')
# Define here the message and message level in case of success
msg = "The prep template '%s' has been updated" % prep_template
msg_level = "success"
# Get the uploads folder
_, base_fp = get_mountpoint("uploads")[0]
# Get the path of the prep template in the uploads folder
fp = join(base_fp, str(study.id), prep_template)
if not exists(fp):
# The file does not exist, fail nicely
# Using 400 because we want the user to get the error in the GUI
raise HTTPError(400, "This file doesn't exist: %s" % fp)
try:
with warnings.catch_warnings(record=True) as warns:
pt = PrepTemplate(pt_id)
pt.update(load_template_to_dataframe(fp))
remove(fp)
# join all the warning messages into one. Note that this info
# will be ignored if an exception is raised
if warns:
msg = '; '.join([str(w.message) for w in warns])
msg_level = 'warning'
except (TypeError, QiitaDBColumnError, QiitaDBExecutionError,
QiitaDBDuplicateError, IOError, ValueError, KeyError,
CParserError, QiitaDBDuplicateHeaderError, QiitaDBError) as e:
# Some error occurred while processing the sample template
# Show the error to the user so they can fix the template
msg = html_error_message % ('updating the prep template:',
basename(fp), str(e))
msg = convert_text_html(msg)
msg_level = "danger"
callback((msg, msg_level, 'prep_template_tab', pt_id, None))
def update_sample_template(self, study, user, callback):
"""Update a sample template from the POST method
Parameters
----------
study : Study
The current study object
user : User
The current user object
callback : function
The callback function to call with the results once the processing
is done
Raises
------
HTTPError
If the sample template file does not exists
"""
# If we are on this function, the argument "sample_template" must
# defined. If not, let tornado raise its error
sample_template = self.get_argument('sample_template')
# Define here the message and message level in case of success
msg = "The sample template '%s' has been updated" % sample_template
msg_level = "success"
# Get the uploads folder
_, base_fp = get_mountpoint("uploads")[0]
# Get the path of the sample template in the uploads folder
fp_rsp = join(base_fp, str(study.id), sample_template)
if not exists(fp_rsp):
# The file does not exist, fail nicely
raise HTTPError(400, "This file doesn't exist: %s" % fp_rsp)
try:
with warnings.catch_warnings(record=True) as warns:
# deleting previous uploads and inserting new one
st = SampleTemplate(study.id)
df = load_template_to_dataframe(fp_rsp)
st.extend(df)
st.update(df)
remove(fp_rsp)
# join all the warning messages into one. Note that this info
# will be ignored if an exception is raised
if warns:
msg = '\n'.join(set(str(w.message) for w in warns))
msg_level = 'warning'
except (TypeError, QiitaDBColumnError, QiitaDBExecutionError,
QiitaDBDuplicateError, IOError, ValueError, KeyError,
CParserError, QiitaDBDuplicateHeaderError, QiitaDBError) as e:
# Some error occurred while processing the sample template
# Show the error to the user so they can fix the template
msg = html_error_message % ('updating the sample template:',
basename(fp_rsp), str(e))
msg = convert_text_html(msg)
msg_level = "danger"
callback((msg, msg_level, None, None, None))
def update_sample_template(self, study, user, callback):
"""Update a sample template from the POST method
Parameters
----------
study : Study
The current study object
user : User
The current user object
callback : function
The callback function to call with the results once the processing
is done
Raises
------
HTTPError
If the sample template file does not exists
"""
# If we are on this function, the argument "sample_template" must
# defined. If not, let tornado raise its error
sample_template = self.get_argument('sample_template')
# Define here the message and message level in case of success
msg = "The sample template '%s' has been updated" % sample_template
msg_level = "success"
# Get the uploads folder
_, base_fp = get_mountpoint("uploads")[0]
# Get the path of the sample template in the uploads folder
fp_rsp = join(base_fp, str(study.id), sample_template)
if not exists(fp_rsp):
# The file does not exist, fail nicely
raise HTTPError(400, "This file doesn't exist: %s" % fp_rsp)
try:
with warnings.catch_warnings(record=True) as warns:
# deleting previous uploads and inserting new one
st = SampleTemplate(study.id)
df = load_template_to_dataframe(fp_rsp)
st.extend(df)
st.update(df)
remove(fp_rsp)
# join all the warning messages into one. Note that this info
# will be ignored if an exception is raised
if warns:
msg = '\n'.join(set(str(w.message) for w in warns))
msg_level = 'warning'
except (TypeError, QiitaDBColumnError, QiitaDBExecutionError,
QiitaDBDuplicateError, IOError, ValueError, KeyError,
CParserError, QiitaDBDuplicateHeaderError, QiitaDBError) as e:
# Some error occurred while processing the sample template
# Show the error to the user so they can fix the template
msg = html_error_message % ('updating the sample template:',
basename(fp_rsp), str(e))
msg = convert_text_html(msg)
msg_level = "danger"
callback((msg, msg_level, None, None, None))
def update_investigation_type(self, study, user, callback):
"""Updates the investigation type of a prep template
Parameters
----------
study : Study
The current study object
user : User
The current user object
callback : function
The callback function to call with the results once the processing
is done
"""
msg = "investigation type successfully updated"
msg_level = "success"
ppd_id = int(self.get_argument('ppd_id'))
prep_id = self.get_argument('prep_id')
edit_investigation_type = self.get_argument('edit-investigation-type',
None)
edit_user_defined_investigation_type = self.get_argument(
'edit-user-defined-investigation-type', None)
edit_new_investigation_type = self.get_argument(
'edit-new-investigation-type', None)
pt = PrepTemplate(prep_id)
investigation_type = self._process_investigation_type(
edit_investigation_type, edit_user_defined_investigation_type,
edit_new_investigation_type)
try:
pt.investigation_type = investigation_type
except QiitaDBColumnError as e:
msg = html_error_message % (", invalid investigation type: ",
investigation_type, str(e))
msg = convert_text_html(msg)
msg_level = "danger"
if ppd_id == 0:
top_tab = "prep_template_tab"
sub_tab = prep_id
prep_tab = None
else:
top_tab = "preprocessed_data_tab"
sub_tab = ppd_id
prep_tab = None
callback((msg, msg_level, top_tab, sub_tab, prep_tab))
def update_investigation_type(self, study, user, callback):
"""Updates the investigation type of a prep template
Parameters
----------
study : Study
The current study object
user : User
The current user object
callback : function
The callback function to call with the results once the processing
is done
"""
msg = "investigation type successfully updated"
msg_level = "success"
ppd_id = int(self.get_argument('ppd_id'))
prep_id = self.get_argument('prep_id')
edit_investigation_type = self.get_argument('edit-investigation-type',
None)
edit_user_defined_investigation_type = self.get_argument(
'edit-user-defined-investigation-type', None)
edit_new_investigation_type = self.get_argument(
'edit-new-investigation-type', None)
pt = PrepTemplate(prep_id)
investigation_type = self._process_investigation_type(
edit_investigation_type, edit_user_defined_investigation_type,
edit_new_investigation_type)
try:
pt.investigation_type = investigation_type
except QiitaDBColumnError as e:
msg = html_error_message % (", invalid investigation type: ",
investigation_type, str(e))
msg = convert_text_html(msg)
msg_level = "danger"
if ppd_id == 0:
top_tab = "prep_template_tab"
sub_tab = prep_id
prep_tab = None
else:
top_tab = "preprocessed_data_tab"
sub_tab = ppd_id
prep_tab = None
callback((msg, msg_level, top_tab, sub_tab, prep_tab))
def process_sample_template(self, study, user, callback):
"""Process a sample template from the POST method
Parameters
----------
study : Study
The current study object
user : User
The current user object
callback : function
The callback function to call with the results once the processing
is done
Raises
------
HTTPError
If the sample template file does not exists
"""
# If we are on this function, the arguments "sample_template" and
# "data_type" must be defined. If not, let tornado raise its error
sample_template = self.get_argument('sample_template')
data_type = self.get_argument('data_type')
# Get the uploads folder
_, base_fp = get_mountpoint("uploads")[0]
# Get the path of the sample template in the uploads folder
fp_rsp = join(base_fp, str(study.id), sample_template)
if not exists(fp_rsp):
# The file does not exist, fail nicely
raise HTTPError(404, "This file doesn't exist: %s" % fp_rsp)
# Define here the message and message level in case of success
msg = "The sample template '%s' has been added" % sample_template
msg_level = "success"
is_mapping_file = looks_like_qiime_mapping_file(fp_rsp)
try:
if is_mapping_file and not data_type:
raise ValueError("Please, choose a data type if uploading a "
"QIIME mapping file")
with warnings.catch_warnings(record=True) as warns:
if is_mapping_file:
create_templates_from_qiime_mapping_file(fp_rsp, study,
int(data_type))
else:
SampleTemplate.create(load_template_to_dataframe(fp_rsp),
study)
remove(fp_rsp)
# join all the warning messages into one. Note that this
# info will be ignored if an exception is raised
if warns:
msg = '; '.join([convert_text_html(str(w.message))
for w in warns])
msg_level = 'warning'
except (TypeError, QiitaDBColumnError, QiitaDBExecutionError,
QiitaDBDuplicateError, IOError, ValueError, KeyError,
CParserError, QiitaDBDuplicateHeaderError,
QiitaDBError, QiitaWareError) as e:
# Some error occurred while processing the sample template
# Show the error to the user so they can fix the template
error_msg = ('parsing the QIIME mapping file'
if is_mapping_file
else 'parsing the sample template')
msg = html_error_message % (error_msg, basename(fp_rsp),
str(e))
msg = convert_text_html(msg)
msg_level = "danger"
callback((msg, msg_level, None, None, None))
def get(self):
msg = self.get_argument('message', '')
msg = convert_text_html(msg)
lvl = self.get_argument('level', '')
iframe = self.get_argument('iframe', '')
self.render("iframe.html", iframe=iframe, message=msg, level=lvl)
def get(self):
msg = self.get_argument('message', '')
msg = convert_text_html(msg)
lvl = self.get_argument('level', '')
self.render("index.html", message=msg, level=lvl)
def get(self):
msg = self.get_argument('message', '')
msg = convert_text_html(msg)
lvl = self.get_argument('level', '')
iframe = self.get_argument('iframe', '')
self.render("iframe.html", iframe=iframe, message=msg, level=lvl)
def render(self, study_id, preprocessed_data):
user = self.current_user
ppd_id = preprocessed_data.id
ebi_status = preprocessed_data.submitted_to_insdc_status()
ebi_study_accession = preprocessed_data.ebi_study_accession
ebi_submission_accession = preprocessed_data.ebi_submission_accession
vamps_status = preprocessed_data.submitted_to_vamps_status()
filepaths = preprocessed_data.get_filepaths()
is_local_request = is_localhost(self.request.headers['host'])
show_ebi_btn = user.level == "admin"
processing_status = convert_text_html(
preprocessed_data.processing_status)
processed_data = sorted(preprocessed_data.processed_data)
# Get all the ENA terms for the investigation type
ontology = Ontology(convert_to_id('ENA', 'ontology'))
# make "Other" show at the bottom of the drop down menu
ena_terms = []
for v in sorted(ontology.terms):
if v != 'Other':
ena_terms.append('<option value="%s">%s</option>' % (v, v))
ena_terms.append('<option value="Other">Other</option>')
# New Type is for users to add a new user-defined investigation type
user_defined_terms = ontology.user_defined_terms + ['New Type']
if PrepTemplate.exists(preprocessed_data.prep_template):
prep_template_id = preprocessed_data.prep_template
prep_template = PrepTemplate(prep_template_id)
raw_data_id = prep_template.raw_data
inv_type = prep_template.investigation_type or "None Selected"
else:
prep_template_id = None
raw_data_id = None
inv_type = "None Selected"
process_params = {param.id: (generate_param_str(param), param.name)
for param in ProcessedSortmernaParams.iter()}
# We just need to provide an ID for the default parameters,
# so we can initialize the interface
default_params = 1
return self.render_string(
"study_description_templates/preprocessed_data_info_tab.html",
ppd_id=ppd_id,
show_ebi_btn=show_ebi_btn,
ebi_status=ebi_status,
ebi_study_accession=ebi_study_accession,
ebi_submission_accession=ebi_submission_accession,
filepaths=filepaths,
is_local_request=is_local_request,
prep_template_id=prep_template_id,
raw_data_id=raw_data_id,
inv_type=inv_type,
ena_terms=ena_terms,
vamps_status=vamps_status,
user_defined_terms=user_defined_terms,
process_params=process_params,
default_params=default_params,
study_id=preprocessed_data.study,
processing_status=processing_status,
processed_data=processed_data)
def add_prep_template(self, study, user, callback):
"""Adds a prep template to the system
Parameters
----------
study : Study
The current study object
user : User
The current user object
callback : function
The callback function to call with the results once the processing
is done
"""
msg = "Your prep template was added"
msg_level = "success"
# If we are on this function, the arguments "prep_template" and
# "data_type_id" must be defined. If not,
# let tornado raise its error
prep_template = self.get_argument('prep_template')
data_type_id = self.get_argument('data_type_id')
# These parameters are optional
investigation_type = self.get_argument('investigation-type', None)
user_defined_investigation_type = self.get_argument(
'user-defined-investigation-type', None)
new_investigation_type = self.get_argument('new-investigation-type',
None)
investigation_type = self._process_investigation_type(
investigation_type, user_defined_investigation_type,
new_investigation_type)
# Get the upload base directory
_, base_path = get_mountpoint("uploads")[0]
# Get the path to the prep template
fp_rpt = join(base_path, str(study.id), prep_template)
if not exists(fp_rpt):
# The file does not exists, fail nicely
raise HTTPError(400, "This file doesn't exist: %s" % fp_rpt)
try:
with warnings.catch_warnings(record=True) as warns:
# force all warnings to always be triggered
warnings.simplefilter("always")
# deleting previous uploads and inserting new one
pt_id = self.remove_add_prep_template(fp_rpt, study,
data_type_id,
investigation_type)
# join all the warning messages into one. Note that this info
# will be ignored if an exception is raised
if warns:
msg = '; '.join([str(w.message) for w in warns])
msg_level = 'warning'
except (TypeError, QiitaDBError, QiitaDBColumnError,
QiitaDBExecutionError, QiitaDBDuplicateError, IOError,
ValueError, CParserError) as e:
pt_id = None
# Some error occurred while processing the prep template
# Show the error to the user so he can fix the template
msg = html_error_message % ("parsing the prep template: ",
basename(fp_rpt), str(e))
msg = convert_text_html(msg)
msg_level = "danger"
callback((msg, msg_level, 'prep_template_tab', pt_id, None))
def add_prep_template(self, study, user, callback):
"""Adds a prep template to the system
Parameters
----------
study : Study
The current study object
user : User
The current user object
callback : function
The callback function to call with the results once the processing
is done
"""
msg = "Your prep template was added"
msg_level = "success"
# If we are on this function, the arguments "prep_template" and
# "data_type_id" must be defined. If not,
# let tornado raise its error
prep_template = self.get_argument('prep_template')
data_type_id = self.get_argument('data_type_id')
# These parameters are optional
investigation_type = self.get_argument('investigation-type', None)
user_defined_investigation_type = self.get_argument(
'user-defined-investigation-type', None)
new_investigation_type = self.get_argument('new-investigation-type',
None)
investigation_type = self._process_investigation_type(
investigation_type, user_defined_investigation_type,
new_investigation_type)
# Get the upload base directory
_, base_path = get_mountpoint("uploads")[0]
# Get the path to the prep template
fp_rpt = join(base_path, str(study.id), prep_template)
if not exists(fp_rpt):
# The file does not exists, fail nicely
raise HTTPError(400, "This file doesn't exist: %s" % fp_rpt)
try:
with warnings.catch_warnings(record=True) as warns:
# force all warnings to always be triggered
warnings.simplefilter("always")
# deleting previous uploads and inserting new one
pt_id = self.remove_add_prep_template(fp_rpt, study,
data_type_id,
investigation_type)
# join all the warning messages into one. Note that this info
# will be ignored if an exception is raised
if warns:
msg = '; '.join([str(w.message) for w in warns])
msg_level = 'warning'
except (TypeError, QiitaDBError, QiitaDBColumnError,
QiitaDBExecutionError, QiitaDBDuplicateError, IOError,
ValueError, CParserError) as e:
pt_id = None
# Some error occurred while processing the prep template
# Show the error to the user so he can fix the template
msg = html_error_message % ("parsing the prep template: ",
basename(fp_rpt), str(e))
msg = convert_text_html(msg)
msg_level = "danger"
callback((msg, msg_level, 'prep_template_tab', pt_id, None))
def get(self):
msg = self.get_argument('message', '')
msg = convert_text_html(msg)
lvl = self.get_argument('level', '')
self.render("index.html", message=msg, level=lvl)
def process_sample_template(self, study, user, callback):
"""Process a sample template from the POST method
Parameters
----------
study : Study
The current study object
user : User
The current user object
callback : function
The callback function to call with the results once the processing
is done
Raises
------
HTTPError
If the sample template file does not exists
"""
# If we are on this function, the arguments "sample_template" and
# "data_type" must be defined. If not, let tornado raise its error
sample_template = self.get_argument('sample_template')
data_type = self.get_argument('data_type')
# Get the uploads folder
_, base_fp = get_mountpoint("uploads")[0]
# Get the path of the sample template in the uploads folder
fp_rsp = join(base_fp, str(study.id), sample_template)
if not exists(fp_rsp):
# The file does not exist, fail nicely
raise HTTPError(404, "This file doesn't exist: %s" % fp_rsp)
# Define here the message and message level in case of success
msg = "The sample template '%s' has been added" % sample_template
msg_level = "success"
is_mapping_file = looks_like_qiime_mapping_file(fp_rsp)
try:
if is_mapping_file and not data_type:
raise ValueError("Please, choose a data type if uploading a "
"QIIME mapping file")
with warnings.catch_warnings(record=True) as warns:
if is_mapping_file:
create_templates_from_qiime_mapping_file(
fp_rsp, study, int(data_type))
else:
SampleTemplate.create(load_template_to_dataframe(fp_rsp),
study)
remove(fp_rsp)
# join all the warning messages into one. Note that this
# info will be ignored if an exception is raised
if warns:
msg = '; '.join(
[convert_text_html(str(w.message)) for w in warns])
msg_level = 'warning'
except (TypeError, QiitaDBColumnError, QiitaDBExecutionError,
QiitaDBDuplicateError, IOError, ValueError, KeyError,
CParserError, QiitaDBDuplicateHeaderError, QiitaDBError,
QiitaWareError) as e:
# Some error occurred while processing the sample template
# Show the error to the user so they can fix the template
error_msg = ('parsing the QIIME mapping file'
if is_mapping_file else 'parsing the sample template')
msg = html_error_message % (error_msg, basename(fp_rsp), str(e))
msg = convert_text_html(msg)
msg_level = "danger"
callback((msg, msg_level, None, None, None))