def _process_investigation_type(self, inv_type, user_def_type, new_type):
"""Return the investigation_type and add it to the ontology if needed
Parameters
----------
inv_type : str
The investigation type
user_def_type : str
The user-defined investigation type
new_type : str
The new user-defined investigation_type
Returns
-------
str
The investigation type chosen by the user
"""
if inv_type == 'None Selected':
inv_type = None
elif inv_type == 'Other' and user_def_type == 'New Type':
# This is a nre user defined investigation type so store it
inv_type = new_type
ontology = Ontology(convert_to_id('ENA', 'ontology'))
ontology.add_user_defined_term(inv_type)
elif inv_type == 'Other' and user_def_type != 'New Type':
inv_type = user_def_type
return inv_type
def _process_investigation_type(inv_type, user_def_type, new_type):
"""Return the investigation_type and add it to the ontology if needed
Parameters
----------
inv_type : str
The investigation type
user_def_type : str
The user-defined investigation type
new_type : str
The new user-defined investigation_type
Returns
-------
str
The investigation type chosen by the user
"""
if inv_type == '':
inv_type = None
elif inv_type == 'Other' and user_def_type == 'New Type':
# This is a new user defined investigation type so store it
inv_type = new_type
ontology = Ontology(convert_to_id('ENA', 'ontology'))
ontology.add_user_defined_term(inv_type)
elif inv_type == 'Other' and user_def_type != 'New Type':
inv_type = user_def_type
return inv_type
class TestOntology(TestCase):
def setUp(self):
self.ontology = Ontology(999999999)
def testConvertToID(self):
self.assertEqual(convert_to_id('ENA', 'ontology'), 999999999)
def testConvertFromID(self):
self.assertEqual(convert_from_id(999999999, 'ontology'), 'ENA')
def testShortNameProperty(self):
self.assertEqual(self.ontology.shortname, 'ENA')
def testTerms(self):
obs = self.ontology.terms
self.assertEqual(obs, [
'Whole Genome Sequencing',
'Metagenomics',
'Transcriptome Analysis',
'Resequencing',
'Epigenetics',
'Synthetic Genomics',
'Forensic or Paleo-genomics',
'Gene Regulation Study',
'Cancer Genomics',
'Population Genomics',
'RNASeq',
'Exome Sequencing',
'Pooled Clone Sequencing',
'Other'])
def test_user_defined_terms(self):
obs = self.ontology.user_defined_terms
self.assertEqual(obs, [])
def test_term_type(self):
obs = self.ontology.term_type('RNASeq')
self.assertEqual('ontology', obs)
obs = self.ontology.term_type('Sasquatch')
self.assertEqual('not_ontology', obs)
self.ontology.add_user_defined_term('Test Term')
obs = self.ontology.term_type('Test Term')
self.assertEqual('user_defined', obs)
def test_add_user_defined_term(self):
self.assertFalse('Test Term' in self.ontology.user_defined_terms)
pre = len(self.ontology.user_defined_terms)
self.ontology.add_user_defined_term('Test Term')
post = len(self.ontology.user_defined_terms)
self.assertTrue('Test Term' in self.ontology.user_defined_terms)
self.assertEqual(post-pre, 1)
def testContains(self):
self.assertTrue('Metagenomics' in self.ontology)
self.assertFalse('NotATerm' in self.ontology)
class TestOntology(TestCase):
def setUp(self):
self.ontology = Ontology(999999999)
def testConvertToID(self):
self.assertEqual(convert_to_id('ENA', 'ontology'), 999999999)
def testConvertFromID(self):
self.assertEqual(convert_from_id(999999999, 'ontology'), 'ENA')
def testShortNameProperty(self):
self.assertEqual(self.ontology.shortname, 'ENA')
def testTerms(self):
obs = self.ontology.terms
self.assertEqual(obs, [
'Whole Genome Sequencing', 'Metagenomics',
'Transcriptome Analysis', 'Resequencing', 'Epigenetics',
'Synthetic Genomics', 'Forensic or Paleo-genomics',
'Gene Regulation Study', 'Cancer Genomics', 'Population Genomics',
'RNASeq', 'Exome Sequencing', 'Pooled Clone Sequencing', 'Other'
])
def test_user_defined_terms(self):
obs = self.ontology.user_defined_terms
self.assertEqual(obs, [])
def test_term_type(self):
obs = self.ontology.term_type('RNASeq')
self.assertEqual('ontology', obs)
obs = self.ontology.term_type('Sasquatch')
self.assertEqual('not_ontology', obs)
self.ontology.add_user_defined_term('Test Term')
obs = self.ontology.term_type('Test Term')
self.assertEqual('user_defined', obs)
def test_add_user_defined_term(self):
self.assertFalse('Test Term' in self.ontology.user_defined_terms)
pre = len(self.ontology.user_defined_terms)
self.ontology.add_user_defined_term('Test Term')
post = len(self.ontology.user_defined_terms)
self.assertTrue('Test Term' in self.ontology.user_defined_terms)
self.assertEqual(post - pre, 1)
def testContains(self):
self.assertTrue('Metagenomics' in self.ontology)
self.assertFalse('NotATerm' in self.ontology)
def ontology_patch_handler(req_op, req_path, req_value=None, req_from=None):
"""Patches an ontology
Parameters
----------
req_op : str
The operation to perform on the ontology
req_path : str
The ontology to patch
req_value : str, optional
The value that needs to be modified
req_from : str, optional
The original path of the element
Returns
-------
dict of {str: str}
A dictionary of the form: {'status': str, 'message': str} in which
status is the status of the request ('error' or 'success') and message
is a human readable string with the error message in case that status
is 'error'.
"""
if req_op == 'add':
req_path = [v for v in req_path.split('/') if v]
if len(req_path) != 1:
return {'status': 'error', 'message': 'Incorrect path parameter'}
req_path = req_path[0]
try:
o_id = convert_to_id(req_path, 'ontology')
except QiitaDBLookupError:
return {
'status': 'error',
'message': 'Ontology "%s" does not exist' % req_path
}
ontology = Ontology(o_id)
ontology.add_user_defined_term(req_value)
return {'status': 'success', 'message': ''}
else:
return {
'status':
'error',
'message':
'Operation "%s" not supported. '
'Current supported operations: add' % req_op
}
def ontology_patch_handler(req_op, req_path, req_value=None, req_from=None):
"""Patches an ontology
Parameters
----------
req_op : str
The operation to perform on the ontology
req_path : str
The ontology to patch
req_value : str, optional
The value that needs to be modified
req_from : str, optional
The original path of the element
Returns
-------
dict of {str: str}
A dictionary of the form: {'status': str, 'message': str} in which
status is the status of the request ('error' or 'success') and message
is a human readable string with the error message in case that status
is 'error'.
"""
if req_op == "add":
req_path = [v for v in req_path.split("/") if v]
if len(req_path) != 1:
return {"status": "error", "message": "Incorrect path parameter"}
req_path = req_path[0]
try:
o_id = convert_to_id(req_path, "ontology")
except QiitaDBLookupError:
return {"status": "error", "message": 'Ontology "%s" does not exist' % req_path}
ontology = Ontology(o_id)
ontology.add_user_defined_term(req_value)
return {"status": "success", "message": ""}
else:
return {
"status": "error",
"message": 'Operation "%s" not supported. ' "Current supported operations: add" % req_op,
}
def make_environment(load_ontologies, download_reference, add_demo_user):
r"""Creates the new environment specified in the configuration
Parameters
----------
load_ontologies : bool
Whether or not to retrieve and unpack ontology information
download_reference : bool
Whether or not to download greengenes reference files
add_demo_user : bool
Whether or not to add a demo user to the database with username
[email protected] and password "password"
Raises
------
IOError
If `download_reference` is true but one of the files cannot be
retrieved
QiitaEnvironmentError
If the environment already exists
"""
if load_ontologies and qiita_config.test_environment:
raise EnvironmentError("Cannot load ontologies in a test environment! "
"Pass --no-load-ontologies, or set "
"TEST_ENVIRONMENT = FALSE in your "
"configuration")
# Connect to the postgres server
admin_conn = SQLConnectionHandler(admin='admin_without_database')
# Check that it does not already exists
if _check_db_exists(qiita_config.database, admin_conn):
raise QiitaEnvironmentError(
"Database {0} already present on the system. You can drop it "
"by running 'qiita_env drop'".format(qiita_config.database))
# Create the database
print('Creating database')
admin_conn.set_autocommit('on')
admin_conn.execute('CREATE DATABASE %s' % qiita_config.database)
admin_conn.set_autocommit('off')
del admin_conn
# Connect to the postgres server, but this time to the just created db
conn = SQLConnectionHandler()
print('Inserting database metadata')
# Build the SQL layout into the database
with open(SETTINGS_FP, 'U') as f:
conn.execute(f.read())
# Insert the settings values to the database
conn.execute("INSERT INTO settings (test, base_data_dir, base_work_dir) "
"VALUES (%s, %s, %s)",
(qiita_config.test_environment, qiita_config.base_data_dir,
qiita_config.working_dir))
create_layout_and_patch(conn, verbose=True)
if load_ontologies:
_add_ontology_data(conn)
# these values can only be added if the environment is being loaded
# with the ontologies, thus this cannot exist inside intialize.sql
# because otherwise loading the ontologies would be a requirement
ontology = Ontology(convert_to_id('ENA', 'ontology'))
ontology.add_user_defined_term('Amplicon Sequencing')
if download_reference:
_download_reference_files(conn)
# we don't do this if it's a test environment because populate.sql
# already adds this user...
if add_demo_user and not qiita_config.test_environment:
conn.execute("""
INSERT INTO qiita.qiita_user (email, user_level_id, password,
name, affiliation, address, phone)
VALUES
('*****@*****.**', 4,
'$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe',
'Demo', 'Qitta Dev', '1345 Colorado Avenue', '303-492-1984')""")
conn.execute("""
INSERT INTO qiita.analysis (email, name, description, dflt,
analysis_status_id)
VALUES
('*****@*****.**', '[email protected]', 'dflt', 't', 1)""")
print('Demo user successfully created')
if qiita_config.test_environment:
_populate_test_db(conn)
print('Test environment successfully created')
else:
print('Production environment successfully created')
def make_environment(load_ontologies, download_reference, add_demo_user):
r"""Creates the new environment specified in the configuration
Parameters
----------
load_ontologies : bool
Whether or not to retrieve and unpack ontology information
download_reference : bool
Whether or not to download greengenes reference files
add_demo_user : bool
Whether or not to add a demo user to the database with username
[email protected] and password "password"
Raises
------
IOError
If `download_reference` is true but one of the files cannot be
retrieved
QiitaEnvironmentError
If the environment already exists
"""
if load_ontologies and qiita_config.test_environment:
raise EnvironmentError("Cannot load ontologies in a test environment! "
"Pass --no-load-ontologies, or set "
"TEST_ENVIRONMENT = FALSE in your "
"configuration")
# Connect to the postgres server
admin_conn = SQLConnectionHandler(admin='admin_without_database')
# Check that it does not already exists
if _check_db_exists(qiita_config.database, admin_conn):
raise QiitaEnvironmentError(
"Database {0} already present on the system. You can drop it "
"by running 'qiita_env drop'".format(qiita_config.database))
# Create the database
print('Creating database')
admin_conn.set_autocommit('on')
admin_conn.execute('CREATE DATABASE %s' % qiita_config.database)
admin_conn.set_autocommit('off')
del admin_conn
# Connect to the postgres server, but this time to the just created db
conn = SQLConnectionHandler()
print('Inserting database metadata')
# Build the SQL layout into the database
with open(SETTINGS_FP, 'U') as f:
conn.execute(f.read())
# Insert the settings values to the database
conn.execute("INSERT INTO settings (test, base_data_dir, base_work_dir) "
"VALUES (%s, %s, %s)",
(qiita_config.test_environment, qiita_config.base_data_dir,
qiita_config.working_dir))
create_layout_and_patch(conn, verbose=True)
if load_ontologies:
_add_ontology_data(conn)
# these values can only be added if the environment is being loaded
# with the ontologies, thus this cannot exist inside intialize.sql
# because otherwise loading the ontologies would be a requirement
ontology = Ontology(convert_to_id('ENA', 'ontology'))
ontology.add_user_defined_term('Amplicon Sequencing')
if download_reference:
_download_reference_files(conn)
# we don't do this if it's a test environment because populate.sql
# already adds this user...
if add_demo_user and not qiita_config.test_environment:
conn.execute("""
INSERT INTO qiita.qiita_user (email, user_level_id, password,
name, affiliation, address, phone)
VALUES
('*****@*****.**', 4,
'$2a$12$gnUi8Qg.0tvW243v889BhOBhWLIHyIJjjgaG6dxuRJkUM8nXG9Efe',
'Demo', 'Qitta Dev', '1345 Colorado Avenue', '303-492-1984')""")
print('Demo user successfully created')
if qiita_config.test_environment:
_populate_test_db(conn)
print('Test environment successfully created')
else:
print('Production environment successfully created')
def post(self, study_id):
study_id = int(study_id)
user = User(self.current_user)
try:
study = Study(study_id)
except QiitaDBUnknownIDError:
# Study not in database so fail nicely
raise HTTPError(404, "Study %d does not exist" % study_id)
else:
check_access(User(self.current_user), study,
raise_error=True)
# vars to add sample template
msg = ''
msg_level = ''
tab_to_display = ''
sample_template = self.get_argument('sample_template', None)
# vars to add raw data
filetype = self.get_argument('filetype', None)
previous_raw_data = self.get_argument('previous_raw_data', None)
# vars to add prep template
add_prep_template = self.get_argument('add_prep_template', None)
raw_data_id = self.get_argument('raw_data_id', None)
data_type_id = self.get_argument('data_type_id', None)
make_public = self.get_argument('make_public', False)
make_sandbox = self.get_argument('make_sandbox', False)
approve_study = self.get_argument('approve_study', False)
request_approval = self.get_argument('request_approval', False)
investigation_type = self.get_argument('investigation-type', None)
user_defined_investigation_type = self.get_argument(
'user-defined-investigation-type', None)
new_investigation_type = self.get_argument('new-investigation-type',
None)
# None Selected is the equivalent to the user not specifying the info
# thus we should make the investigation_type None
if investigation_type == "" or investigation_type == "None Selected":
investigation_type = None
# to update investigation type
update_investigation_type = self.get_argument(
'update_investigation_type', None)
edit_investigation_type = self.get_argument('edit-investigation-type',
None)
edit_user_defined_investigation_type = self.get_argument(
'edit-user-defined-investigation-type', None)
edit_new_investigation_type = self.get_argument(
'edit-new-investigation-type', None)
# None Selected is the equivalent to the user not specifying the info
# thus we should make the investigation_type None
if edit_investigation_type == "" or \
edit_investigation_type == "None Selected":
edit_investigation_type = None
msg_level = 'success'
if sample_template:
# processing sample templates
_, base_fp = get_mountpoint("uploads")[0]
fp_rsp = join(base_fp, str(study_id), sample_template)
if not exists(fp_rsp):
raise HTTPError(400, "This file doesn't exist: %s" % fp_rsp)
try:
# deleting previous uploads and inserting new one
yield Task(self.remove_add_study_template,
study.raw_data,
study_id, fp_rsp)
except (TypeError, QiitaDBColumnError, QiitaDBExecutionError,
QiitaDBDuplicateError, IOError, ValueError, KeyError,
CParserError, QiitaDBDuplicateHeaderError) as e:
msg = html_error_message % ('parsing the sample template:',
basename(fp_rsp), str(e))
self.display_template(study, msg, "danger")
return
msg = ("The sample template '%s' has been added" %
sample_template)
tab_to_display = ""
elif request_approval:
study.status = 'awaiting_approval'
msg = "Study sent to admin for approval"
tab_to_display = ""
elif make_public:
msg = ''
study.status = 'public'
msg = "Study set to public"
tab_to_display = ""
elif make_sandbox:
msg = ''
study.status = 'sandbox'
msg = "Study reverted to sandbox"
tab_to_display = ""
elif approve_study:
# make sure user is admin, then make full private study
if user.level == 'admin' or not \
qiita_config.require_approval:
study.status = 'private'
msg = "Study approved"
tab_to_display = ""
elif filetype or previous_raw_data:
# adding blank raw data
if filetype and previous_raw_data:
msg = ("You can not specify both a new raw data and a "
"previouly used one")
elif filetype:
try:
RawData.create(filetype, [study])
except (TypeError, QiitaDBColumnError, QiitaDBExecutionError,
QiitaDBDuplicateError, IOError, ValueError, KeyError,
CParserError) as e:
msg = html_error_message % ("creating a new raw data "
"object for study:",
str(study.id), str(e))
self.display_template(study, msg, "danger")
return
msg = ""
else:
raw_data = [RawData(rd) for rd in previous_raw_data]
study.add_raw_data(raw_data)
msg = ""
tab_to_display = ""
elif add_prep_template and raw_data_id and data_type_id:
# adding prep templates
if investigation_type == 'Other' and \
user_defined_investigation_type == 'New Type':
investigation_type = new_investigation_type
# this is a new user defined investigation type so store it
ontology = Ontology(convert_to_id('ENA', 'ontology'))
ontology.add_user_defined_term(investigation_type)
elif investigation_type == 'Other' and \
user_defined_investigation_type != 'New Type':
investigation_type = user_defined_investigation_type
raw_data_id = int(raw_data_id)
_, base_path = get_mountpoint("uploads")[0]
fp_rpt = join(base_path, str(study_id), add_prep_template)
if not exists(fp_rpt):
raise HTTPError(400, "This file doesn't exist: %s" % fp_rpt)
try:
# inserting prep templates
yield Task(self.remove_add_prep_template, fp_rpt, raw_data_id,
study, data_type_id, investigation_type)
except (TypeError, QiitaDBColumnError, QiitaDBExecutionError,
QiitaDBDuplicateError, IOError, ValueError,
CParserError) as e:
msg = html_error_message % ("parsing the prep template: ",
basename(fp_rpt), str(e))
self.display_template(study, msg, "danger",
str(raw_data_id))
return
msg = "Your prep template was added"
tab_to_display = str(raw_data_id)
elif update_investigation_type:
# updating the prep template investigation type
pt = PrepTemplate(update_investigation_type)
investigation_type = edit_investigation_type
# figure out whether to add it as a user defined term or not
if edit_investigation_type == 'Other' and \
edit_user_defined_investigation_type == 'New Type':
investigation_type = edit_new_investigation_type
# this is a new user defined investigation type so store it
ontology = Ontology(convert_to_id('ENA', 'ontology'))
ontology.add_user_defined_term(investigation_type)
elif investigation_type == 'Other' and \
user_defined_investigation_type != 'New Type':
investigation_type = edit_user_defined_investigation_type
try:
pt.investigation_type = investigation_type
except QiitaDBColumnError as e:
msg = html_error_message % (", invalid investigation type: ",
investigation_type, str(e))
self.display_template(study, msg, "danger",
str(pt.raw_data))
return
msg = "The prep template has been updated!"
tab_to_display = str(pt.raw_data)
else:
msg = ("Error, did you select a valid uploaded file or are "
"passing the correct parameters?")
msg_level = 'danger'
tab_to_display = ""
self.display_template(study, msg, msg_level, tab_to_display)