def test_repr(self):
jq = JobQuery('y', script='z', prestaged=[123],
status_callback_url='foo', restricted=True)
s = "JobQuery('y', script='z', prestaged=[123], "
s += "status_callback_url='foo',"
s += "restricted=True, ncpus=1)"
self.assertEqual(jq.__repr__(), s)
def test_submitQuery_with_tarball(self):
self.factory.tarball = 'Magma-1.1.tar.gz'
self.factory.submitJob2Launcher = Mock()
job = self.factory.fromScratch(u'bob')
jobquery = JobQuery(job.dir, "", [])
status_cb_url = 'http://example.com/status/{}.json'.format(job.id)
jobquery.status_callback_url = status_cb_url
self.factory.submitQuery(jobquery, job)
jobmanager_query = {'jobdir': jobquery.dir + '/',
'executable': "/bin/sh",
'prestaged': [self.factory.script_fn,
'results.db',
# tarball is staged as well
'Magma-1.1.tar.gz'
],
"poststaged": ['results.db'],
"stderr": "stderr.txt",
"stdout": "stdout.txt",
'arguments': [self.factory.script_fn],
'status_callback_url': status_cb_url
}
self.factory.submitJob2Launcher.assert_called_with(jobmanager_query)
self.assertEqual(job.state, u'PENDING')
def test_submitQuery_invalid_joblauncher(self):
from requests.exceptions import HTTPError
from magmaweb.job import JobSubmissionError
exc = HTTPError('422 Client Error: Unprocessable Entity')
self.factory.submitJob2Launcher = Mock(side_effect=exc)
job = self.factory.fromScratch(u'bob')
jobquery = JobQuery(job.dir, "", [])
status_cb_url = 'http://example.com/status/{}.json'.format(job.id)
jobquery.status_callback_url = status_cb_url
with self.assertRaises(JobSubmissionError):
self.factory.submitQuery(jobquery, job)
self.assertEqual(job.state, 'SUBMISSION_ERROR')
def test_submitQuery_no_joblauncher(self):
from requests.exceptions import ConnectionError
from magmaweb.job import JobSubmissionError
exc = ConnectionError('[Errno 111] Connection refused')
self.factory.submitJob2Launcher = Mock(side_effect=exc)
job = self.factory.fromScratch(u'bob')
jobquery = JobQuery(job.dir, "", [])
status_cb_url = 'http://example.com/status/{}.json'.format(job.id)
jobquery.status_callback_url = status_cb_url
with self.assertRaises(JobSubmissionError):
self.factory.submitQuery(jobquery, job)
self.assertEqual(job.state, 'SUBMISSION_ERROR')
def test_with_metabolize(self):
params = MultiDict(structure_format='smiles',
structures='CCO Ethanol',
metabolize='on',
scenario=[
{'type': 'phase1', 'steps': '2'},
{'type': 'phase2', 'steps': '1'}
],
)
query = self.jobquery.add_structures(params)
sf = 'structures.dat'
scen = 'scenario.csv'
script = "{magma} add_structures -g -t 'smiles' structures.dat {db} |"
script += "{magma} metabolize -g --scenario scenario.csv"
script += " --call_back_url '/' -j - {db}\n"
expected_query = JobQuery(directory=self.jobdir,
prestaged=[sf, scen],
script=script,
status_callback_url='/',
)
self.assertEqual(query, expected_query)
self.assertMultiLineEqual(
'CCO Ethanol', self.fetch_file('structures.dat'))
self.assertMultiLineEqual(
'phase1,2\nphase2,1\n', self.fetch_file('scenario.csv'))
def test_with_structure_database_exclude_halo(self):
params = {'ms_intensity_cutoff': 200000,
'msms_intensity_cutoff': 10,
'max_broken_bonds': 4,
'max_water_losses': 1,
'structure_database': 'pubchem',
'min_refscore': 1,
'max_mz': 1200,
'excl_halo': True,
}
query = self.jobquery.annotate(params, False)
script = "{magma} annotate -c '200000.0' -d '10.0'"
script += " -b '4'"
script += " --max_water_losses '1' --ncpus '1' --call_back_url '/'"
script += " --structure_database 'pubchem'"
script += " --db_options ',1200,False,False,1'"
script += " --fast {db}\n"
expected_query = JobQuery(directory=self.jobdir,
prestaged=[],
script=script,
status_callback_url='/',
)
self.assertEqual(query, expected_query)
def test_resticted(self):
self.jobquery.restricted = True
params = {'ms_data_format': 'mzxml',
'ms_data': 'foo',
'max_ms_level': 3,
'abs_peak_cutoff': 1000,
'scan': 5,
'precursor_mz_precision': 0.005,
'mz_precision': 5.0,
'mz_precision_abs': 0.001,
'ionisation_mode': 1,
}
query = self.jobquery.add_ms_data(params)
script = "{magma} read_ms_data --ms_data_format 'mzxml'"
script += " -i '1' -m '3' -a '1000.0'"
script += " -p '5.0' -q '0.001' --precursor_mz_precision '0.005'"
script += " --scan '5'"
script += " --time_limit 1 --call_back_url '/' ms_data.dat {db}\n"
expected_query = JobQuery(directory=self.jobdir,
prestaged=['ms_data.dat'],
script=script,
status_callback_url='/',
restricted=True,
)
self.assertEqual(query, expected_query)
self.assertMultiLineEqual('foo', self.fetch_file('ms_data.dat'))
def test_ms_data_as_file(self):
import tempfile
from cgi import FieldStorage
msfile = tempfile.TemporaryFile()
msfile.write('foo')
msfile.flush()
msfield = FieldStorage()
msfield.file = msfile
params = {'ms_data_format': 'mzxml',
'ms_data_file': msfield,
'max_ms_level': 3,
'abs_peak_cutoff': 1000,
'precursor_mz_precision': 0.005,
'mz_precision': 5.0,
'mz_precision_abs': 0.001,
'ionisation_mode': 1,
}
query = self.jobquery.add_ms_data(params)
script = "{magma} read_ms_data --ms_data_format 'mzxml'"
script += " -i '1' -m '3' -a '1000.0'"
script += " -p '5.0' -q '0.001' --precursor_mz_precision '0.005'"
script += " --call_back_url '/' ms_data.dat {db}\n"
expected_query = JobQuery(directory=self.jobdir,
prestaged=['ms_data.dat'],
script=script,
status_callback_url='/',
)
self.assertEqual(query, expected_query)
self.assertMultiLineEqual('foo', self.fetch_file('ms_data.dat'))
def test_with_annotate(self):
params = MultiDict(molid=123,
scenario=[{'type': 'phase1', 'steps': '2'},
{'type': 'phase2', 'steps': '1'}],
ms_intensity_cutoff=200000,
msms_intensity_cutoff=10,
max_broken_bonds=4,
max_water_losses=1,
)
query = self.jobquery.metabolize_one(params, True)
script = "echo '123' | {magma} metabolize -g -j - --scenario scenario.csv"
script += " --call_back_url '/' {db} |"
script += "{magma} annotate"
script += " -c '200000.0' -d '10.0'"
script += " -b '4'"
script += " --max_water_losses '1' --ncpus '1' --call_back_url '/' -j -"
script += " --fast {db}\n"
expected_query = JobQuery(directory=self.jobdir,
prestaged=['scenario.csv'],
script=script,
status_callback_url='/',
)
self.assertEqual(query, expected_query)
self.assertMultiLineEqual(
'phase1,2\nphase2,1\n', self.fetch_file('scenario.csv'))
def test_with_metabolize_and_annotate(self):
params = MultiDict(structure_format='smiles',
structures='CCO Ethanol',
metabolize='on',
scenario=[{'type': 'phase1', 'steps': '2'},
{'type': 'phase2', 'steps': '1'}],
ms_intensity_cutoff=200000,
msms_intensity_cutoff=10,
max_broken_bonds=4,
max_water_losses=1,
)
query = self.jobquery.add_structures(params, True)
sf = 'structures.dat'
script = "{magma} add_structures -g -t 'smiles'"
script += " structures.dat {db} |"
script += "{magma} metabolize -g --scenario scenario.csv"
script += " --call_back_url '/' -j - {db} |"
script += "{magma} annotate -c '200000.0'"
script += " -d '10.0' -b '4'"
script += " --max_water_losses '1' --ncpus '1' --call_back_url '/'"
script += " -j - --fast {db}\n"
expected_query = JobQuery(directory=self.jobdir,
prestaged=[sf, 'scenario.csv'],
script=script,
status_callback_url='/',
)
self.assertEqual(query, expected_query)
self.assertMultiLineEqual(
'CCO Ethanol', self.fetch_file('structures.dat'))
self.assertMultiLineEqual(
'phase1,2\nphase2,1\n', self.fetch_file('scenario.csv'))
def test_invalid(self):
from colander import Invalid, SchemaNode
n = SchemaNode(JobQuery.File(), name='filefield')
with self.assertRaises(Invalid) as e:
n.deserialize(12345)
self.assertDictEqual(e.exception.asdict(),
{'filefield': '12345 is not a cgi.FieldStorage'})
def test_without_metabolize(self):
self.maxDiff = 100000
import tempfile
from cgi import FieldStorage
ms_data_file = tempfile.NamedTemporaryFile()
ms_data_file.write('foo')
ms_data_file.flush()
msfield = FieldStorage()
msfield.file = ms_data_file
params = MultiDict(ionisation_mode=1,
ms_intensity_cutoff=200000,
msms_intensity_cutoff=10,
abs_peak_cutoff=1000,
precursor_mz_precision=0.005,
max_broken_bonds=4,
max_water_losses=1,
mz_precision=5.0,
mz_precision_abs=0.001,
scenario=[{'type': 'phase1', 'steps': '2'},
{'type': 'phase2', 'steps': '1'}],
max_ms_level=3,
structures='C1CCCC1 comp1',
ms_data_file=msfield,
structure_format='smiles',
ms_data_format='mzxml',
structure_database='',
min_refscore=1,
max_mz=9999,
)
query = self.jobquery.allinone(params)
expected_script = "{magma} read_ms_data --ms_data_format 'mzxml'"
expected_script += " -i '1' -m '3' -a '1000.0'"
expected_script += " -p '5.0' -q '0.001'"
expected_script += " --precursor_mz_precision '0.005'"
expected_script += " --call_back_url '/'"
expected_script += " ms_data.dat {db}\n"
expected_script += "{magma} add_structures -g -t 'smiles'"
expected_script += " structures.dat {db}\n"
expected_script += "{magma} annotate -c '200000.0'"
expected_script += " -d '10.0'"
expected_script += " -b '4'"
expected_script += " --max_water_losses '1' --ncpus '1' --call_back_url '/'"
expected_script += " --fast {db}\n"
expected_query = JobQuery(directory=self.jobdir,
prestaged=['ms_data.dat', 'structures.dat'],
script=expected_script,
status_callback_url='/',
)
self.assertEqual(query, expected_query)
self.assertMultiLineEqual(params['structures'],
self.fetch_file('structures.dat'))
self.assertMultiLineEqual('foo', self.fetch_file('ms_data.dat'))
def test_jobquery_restricted(self):
status_cb_url = 'http://example/status/{}.json'.format(self.job.id)
jobquery = self.job.jobquery(status_cb_url, True, 1)
ejobquery = JobQuery(self.jobdir,
status_callback_url=status_cb_url,
restricted=True,
)
self.assertEqual(jobquery, ejobquery)
def test_structures_as_string(self):
params = {'structure_format': 'smiles', 'structures': 'CCO Ethanol'}
query = self.jobquery.add_structures(params)
sf = 'structures.dat'
script = "{magma} add_structures -g -t 'smiles' structures.dat {db}\n"
expected_query = JobQuery(directory=self.jobdir,
prestaged=[sf],
script=script,
status_callback_url='/',
)
self.assertEqual(query, expected_query)
self.assertMultiLineEqual(params['structures'], self.fetch_file(sf))
def test_submitQuery(self):
self.factory.init_script = "# make magma available"
job = self.factory.fromScratch(u'bob')
self.factory.script_fn = 'script.sh'
cmd = "magma add_structures -t smiles structures.dat results.db\n"
jobquery = JobQuery(job.dir, cmd, ['structures.dat'])
status_cb_url = 'http://example.com/status/{}.json'.format(job.id)
jobquery.status_callback_url = status_cb_url
launcher_url = 'http://localhost:9998/job/'
launcher_url += '70a00fe2-f698-41ed-b28c-b37c22f10440'
self.factory.submitJob2Launcher = Mock(return_value=launcher_url)
self.factory.submitQuery(jobquery, job)
job_script_fn = os.path.join(jobquery.dir, self.factory.script_fn)
job_script = open(job_script_fn).read()
exp_script = "# make magma available\n"
exp_script += "magma add_structures -t smiles "
exp_script += "structures.dat results.db\n"
self.assertMultiLineEqual(job_script, exp_script)
jobmanager_query = {'jobdir': jobquery.dir + '/',
'executable': "/bin/sh",
'prestaged': [self.factory.script_fn,
'results.db',
'structures.dat'
],
"poststaged": ['results.db'],
"stderr": "stderr.txt",
"stdout": "stdout.txt",
'arguments': [self.factory.script_fn],
'status_callback_url': status_cb_url
}
self.factory.submitJob2Launcher.assert_called_with(jobmanager_query)
self.assertEqual(job.state, u'PENDING')
self.assertEqual(job.launcher_url, launcher_url)
def test_missing_params(self):
params = {'max_broken_bonds': 4,
'max_water_losses': 1,
}
query = self.jobquery.annotate(params)
script = "{magma} annotate -c '0.0' -d '0.0'"
script += " -b '4'"
script += " --max_water_losses '1' --ncpus '1' --call_back_url '/' --fast {db}\n"
expected_query = JobQuery(directory=self.jobdir,
prestaged=[],
script=script,
status_callback_url='/',
)
self.assertEqual(query, expected_query)
def test_with_jsonified_scenario(self):
scenario = '[{"steps": "2", "type": "phase1"}, '
scenario += '{"steps": "1", "type": "phase2"}]'
params = dict(scenario=scenario)
query = self.jobquery.metabolize(params)
script = "{magma} metabolize -g --scenario scenario.csv"
script += " --call_back_url '/' {db}\n"
expected_query = JobQuery(directory=self.jobdir,
prestaged=['scenario.csv'],
script=script,
status_callback_url='/',
)
self.assertEqual(query, expected_query)
self.assertMultiLineEqual(
'phase1,2\nphase2,1\n', self.fetch_file('scenario.csv'))
def test_without_molecule_and_with_structure_database(self):
params = dict(ionisation_mode=1,
ms_intensity_cutoff=200000,
msms_intensity_cutoff=10,
abs_peak_cutoff=1000,
precursor_mz_precision=0.005,
max_broken_bonds=4,
max_water_losses=1,
mz_precision=5.0,
mz_precision_abs=0.001,
scenario=[{'type': 'phase1', 'steps': '2'},
{'type': 'phase2', 'steps': '1'}],
max_ms_level=3,
structures='',
ms_data='bla',
structure_format='smiles',
ms_data_format='mzxml',
structure_database='pubchem',
min_refscore=1,
max_mz=1200,
)
query = self.jobquery.allinone(params)
expected_script = "{magma} read_ms_data --ms_data_format 'mzxml'"
expected_script += " -i '1' -m '3' -a '1000.0'"
expected_script += " -p '5.0' -q '0.001'"
expected_script += " --precursor_mz_precision '0.005'"
expected_script += " --call_back_url '/'"
expected_script += " ms_data.dat {db}\n"
expected_script += "{magma} annotate -c '200000.0'"
expected_script += " -d '10.0'"
expected_script += " -b '4'"
expected_script += " --max_water_losses '1' --ncpus '1' --call_back_url '/'"
expected_script += " --structure_database 'pubchem'"
expected_script += " --db_options ',1200,False,True,1'"
expected_script += " --fast {db}\n"
expected_query = JobQuery(directory=self.jobdir,
prestaged=['ms_data.dat'],
script=expected_script,
status_callback_url='/',
)
self.assertEqual(query, expected_query)
def test_it(self):
params = MultiDict(scenario=[{'type': 'phase1', 'steps': '2'},
{'type': 'phase2', 'steps': '1'}]
)
query = self.jobquery.metabolize(params)
script = "{magma} metabolize -g --scenario scenario.csv"
script += " --call_back_url '/' {db}\n"
expected_query = JobQuery(directory=self.jobdir,
prestaged=['scenario.csv'],
script=script,
status_callback_url='/',
)
self.assertEqual(query, expected_query)
self.assertMultiLineEqual(
'phase1,2\nphase2,1\n', self.fetch_file('scenario.csv'))
def test_defaults(self):
expected = dict(scenario=[
{'type': 'phase1', 'steps': '2'},
{'type': 'phase2', 'steps': '1'}
],
ionisation_mode=1,
ms_data_format='mzxml',
ms_data_area='',
ms_intensity_cutoff=0.0,
msms_intensity_cutoff=5,
mz_precision=5.0,
mz_precision_abs=0.001,
abs_peak_cutoff=5000,
max_ms_level=10,
precursor_mz_precision=0.005,
max_broken_bonds=3,
max_water_losses=1,
)
self.assertDictEqual(expected, JobQuery.defaults())
def test_structures_as_file(self):
import tempfile
from cgi import FieldStorage
sfile = tempfile.TemporaryFile()
sfile.write('foo')
sfile.flush()
sfield = FieldStorage()
sfield.file = sfile
params = {'structure_format': 'smiles', 'structures_file': sfield}
query = self.jobquery.add_structures(params)
script = "{magma} add_structures -g -t 'smiles' structures.dat {db}\n"
expected_query = JobQuery(directory=self.jobdir,
prestaged=['structures.dat'],
script=script,
status_callback_url='/',
)
self.assertEqual(query, expected_query)
self.assertMultiLineEqual('foo', self.fetch_file('structures.dat'))
def test_defaults_example(self):
example_tree = [
'353.087494: 69989984 (',
' 191.055756: 54674544 (',
' 85.029587: 2596121,',
' 93.034615: 1720164,',
' 109.029442: 917026,',
' 111.045067: 1104891 (',
' 81.034691: 28070,',
' 83.014069: 7618,',
' 83.050339: 25471,',
' 93.034599: 36300,',
' 96.021790: 8453',
' ),',
' 127.039917: 2890439 (',
' 57.034718: 16911,',
' 81.034706: 41459,',
' 83.050301: 35131,',
' 85.029533: 236887,',
' 99.045074: 73742,',
' 109.029404: 78094',
' ),',
' 171.029587: 905226,',
' 173.045212: 2285841 (',
' 71.013992: 27805,',
' 93.034569: 393710,',
' 111.008629: 26219,',
' 111.045029: 339595,',
' 137.024292: 27668,',
' 155.034653: 145773',
' ),',
' 191.055725: 17000514',
' ),',
' 353.087097: 4146696',
' )'
]
expected = dict(ms_data="\n".join(example_tree),
ms_data_format='mass_tree',
ionisation_mode=-1,
)
self.assertDictEqual(expected, JobQuery.defaults('example'))
def test_defaults_example2(self):
example_tree = [
'C16H17O9: 69989984 (',
' C7H11O6: 54674544 (',
' C4H5O2: 2596121,',
' C6H5O: 1720164,',
' C6H5O2: 917026,',
' C6H7O2: 1104891 (',
' C5H5O: 28070,',
' C4H3O2: 7618,',
' C5H7O: 25471,',
' C6H5O: 36300,',
' C5H4O2: 8453',
' ),',
' C6H7O3: 2890439 (',
' C3H5O: 16911,',
' C5H5O: 41459,',
' C5H7O: 35131,',
' C4H5O2: 236887,',
' C5H7O2: 73742,',
' C6H5O2: 78094',
' ),',
' C7H7O5: 905226,',
' C7H9O5: 2285841 (',
' C3H3O2: 27805,',
' C6H5O: 393710,',
' C5H3O3: 26219,',
' C6H7O2: 339595,',
' C7H5O3: 27668,',
' C7H7O4: 145773',
' ),',
' C7H11O6: 17000514',
' ),',
' C16H17O9: 4146696',
' )',
]
expected = dict(ms_data="\n".join(example_tree),
ms_data_format='form_tree',
ionisation_mode=-1,
)
self.assertDictEqual(expected, JobQuery.defaults('example2'))
def test_with_form_tree_neg_format(self):
params = {'ms_data_format': 'form_tree',
'ionisation_mode': "-1",
'ms_data': 'foo',
'precursor_mz_precision': 0.005,
'mz_precision': 5.0,
'mz_precision_abs': 0.001,
}
query = self.jobquery.add_ms_data(params)
script = "{magma} read_ms_data --ms_data_format 'form_tree_neg'"
script += " -i '-1' -m '0' -a '0.0'"
script += " -p '5.0' -q '0.001' --precursor_mz_precision '0.005'"
script += " --call_back_url '/' ms_data.dat {db}\n"
expected_query = JobQuery(directory=self.jobdir,
prestaged=['ms_data.dat'],
script=script,
status_callback_url='/',
)
self.assertEqual(query, expected_query)
def test_with_annotate(self):
params = {'structure_format': 'smiles',
'structures': 'CCO Ethanol',
'ms_intensity_cutoff': 200000,
'msms_intensity_cutoff': 10,
'max_broken_bonds': 4,
'max_water_losses': 1,
}
query = self.jobquery.add_structures(params, True)
sf = 'structures.dat'
script = "{magma} add_structures -g -t 'smiles' structures.dat {db} |"
script += "{magma} annotate -c '200000.0'"
script += " -d '10.0' -b '4'"
script += " --max_water_losses '1' --ncpus '1' --call_back_url '/'"
script += " -j - --fast {db}\n"
expected_query = JobQuery(directory=self.jobdir,
prestaged=[sf],
script=script,
status_callback_url='/',
)
self.assertEqual(query, expected_query)
self.assertMultiLineEqual(
'CCO Ethanol', self.fetch_file('structures.dat'))
def test_eq_status_callback_url(self):
job1 = JobQuery(self.jobdir,
status_callback_url='/somewhere',
)
self.assertNotEqual(job1, self.jobquery)
def test_eq_script(self):
job1 = JobQuery(self.jobdir,
script='b',
)
self.assertNotEqual(job1, self.jobquery)
def test_emptystring(self):
from colander import null
self.assertEquals(JobQuery.File().deserialize(None, ''), null)
def test_escape_single_quote(self):
jq = JobQuery('/y')
self.assertEquals(jq.escape("'"), ''')
def test_eq_prestaged(self):
job1 = JobQuery(self.jobdir,
prestaged=[1],
)
self.assertNotEqual(job1, self.jobquery)
def setUp(self):
self.jobdir = tempfile.mkdtemp()
self.jobquery = JobQuery(directory=self.jobdir,
script='',
status_callback_url='/',
)
def test_eq_dir(self):
job1 = JobQuery('/otherdir')
self.assertNotEqual(job1, self.jobquery)
def test_eq_restricted(self):
job1 = JobQuery(self.jobdir,
restricted=True,
)
self.assertNotEqual(job1, self.jobquery)
def defaults(self):
"""Returns defaults settings to run a job"""
selection = self.request.params.get('selection')
return {'success': True,
'data': JobQuery.defaults(selection)}
def test_serialize(self):
self.assertEquals(JobQuery.File().serialize(None, 12345), 12345)
