def test_validate_modeling_project(self):
with biosimulators_utils.__main__.App(argv=[
'validate-project',
os.path.join(os.path.dirname(__file__), 'fixtures',
'mock-file'),
]) as app:
archive = CombineArchive(contents=[])
with mock.patch(
'biosimulators_utils.combine.io.CombineArchiveReader.run',
return_value=archive):
with mock.patch(
'biosimulators_utils.combine.validation.validate',
return_value=([], [])):
app.run()
with biosimulators_utils.__main__.App(argv=[
'validate-project',
os.path.join(
os.path.dirname(__file__), 'fixtures',
'Ciliberto-J-Cell-Biol-2003-morphogenesis-checkpoint.omex'
),
]) as app:
with capturer.CaptureOutput(merged=False, relay=False) as captured:
app.run()
stdout = captured.stdout.get_text()
self.assertRegex(stdout,
'Archive contains 1 SED-ML documents with 1 models')
# warnings
with biosimulators_utils.__main__.App(argv=[
'validate-project',
os.path.join(os.path.dirname(__file__), 'fixtures',
'mock-file'),
]) as app:
archive = CombineArchive(
contents=[CombineArchiveContent(),
CombineArchiveContent()])
with mock.patch(
'biosimulators_utils.combine.io.CombineArchiveReader.run',
return_value=archive):
with mock.patch(
'biosimulators_utils.combine.validation.validate',
return_value=([['Bigger error']], [['Big warning']])):
with self.assertWarnsRegex(BioSimulatorsWarning,
'- Big warning'):
with self.assertRaisesRegex(SystemExit,
'- Bigger error'):
app.run()
with biosimulators_utils.__main__.App(argv=[
'validate-project',
os.path.join(os.path.dirname(__file__), 'fixtures',
'mock-file'),
]) as app:
archive = CombineArchive(contents=[])
with mock.patch(
'biosimulators_utils.combine.io.CombineArchiveReader.run',
return_value=archive):
with self.assertRaisesRegex(
SystemExit, 'must have at least one content element'):
with self.assertWarnsRegex(
BioSimulatorsWarning,
'does not contain any SED-ML files'):
app.run()
# error
with biosimulators_utils.__main__.App(argv=[
'validate-project',
os.path.join(os.path.dirname(__file__), 'fixtures',
'not-a-file'),
]) as app:
with self.assertRaisesRegex(SystemExit, 'is not a file'):
app.run()
with biosimulators_utils.__main__.App(argv=[
'validate-project',
os.path.join(os.path.dirname(__file__), 'fixtures',
'mock-file'),
]) as app:
archive = CombineArchive(
contents=[CombineArchiveContent(),
CombineArchiveContent()])
with mock.patch(
'biosimulators_utils.combine.io.CombineArchiveReader.run',
return_value=archive):
with self.assertRaisesRegex(SystemExit,
'- Content element must'):
app.run()
def test_read_omex_meta_files_for_archive(self):
shutil.copyfile(os.path.join(self.FIXTURE_DIR, 'biosimulations.rdf'),
os.path.join(self.dir_name, 'biosimulations.rdf'))
shutil.copyfile(os.path.join(self.FIXTURE_DIR, 'biosimulations-with-file-annotations.rdf'),
os.path.join(self.dir_name, 'biosimulations-with-file-annotations.rdf'))
archive = CombineArchive()
archive.contents = [
CombineArchiveContent(
location='biosimulations.rdf',
format=CombineArchiveContentFormat.OMEX_METADATA,
),
CombineArchiveContent(
location='biosimulations-with-file-annotations.rdf',
format=CombineArchiveContentFormat.OMEX_METADATA,
),
]
config = Config(
OMEX_METADATA_SCHEMA=data_model.OmexMetadataSchema.biosimulations,
)
md, errors, warnings = io.read_omex_meta_files_for_archive(
archive, self.dir_name, config=config)
self.assertIn('The COMBINE archive should only contain one instance of predicate', flatten_nested_list_of_strings(errors))
self.assertEqual(warnings, [])
self.assertEqual(len(md), 2)
self.assertEqual(sorted(m['uri'] for m in md), sorted(['.', './sim.sedml/figure1']))
md, errors, warnings = io.read_omex_meta_files_for_archive(archive, self.dir_name)
self.assertIn('The COMBINE archive should only contain one instance of predicate', flatten_nested_list_of_strings(errors))
shutil.copyfile(os.path.join(self.FIXTURE_DIR, 'biosimulations-abbrev-third-a.rdf'),
os.path.join(self.dir_name, 'biosimulations-abbrev-third-a.rdf'))
shutil.copyfile(os.path.join(self.FIXTURE_DIR, 'biosimulations-abbrev-third-b.rdf'),
os.path.join(self.dir_name, 'biosimulations-abbrev-third-b.rdf'))
shutil.copyfile(os.path.join(self.FIXTURE_DIR, 'biosimulations-abbrev-third-c.rdf'),
os.path.join(self.dir_name, 'biosimulations-abbrev-third-c.rdf'))
archive = CombineArchive()
archive.contents = [
CombineArchiveContent(
location='biosimulations-abbrev-third-a.rdf',
format=CombineArchiveContentFormat.OMEX_METADATA,
),
CombineArchiveContent(
location='biosimulations-abbrev-third-b.rdf',
format=CombineArchiveContentFormat.OMEX_METADATA,
),
CombineArchiveContent(
location='biosimulations-abbrev-third-c.rdf',
format=CombineArchiveContentFormat.OMEX_METADATA,
),
]
config = Config(
OMEX_METADATA_SCHEMA=data_model.OmexMetadataSchema.biosimulations,
)
md, errors, warnings = io.read_omex_meta_files_for_archive(
archive, self.dir_name, config=config)
self.assertEqual(errors, [])
self.assertEqual(warnings, [])
self.assertEqual(len(md), 2)
self.assertEqual(sorted(m['uri'] for m in md), sorted(['.', './sim.sedml/figure1']))
def test_1(self):
archive = CombineArchive(contents=[
CombineArchiveContent(
location='sim.sedml',
format='http://identifiers.org/combine.specifications/sed-ml',
),
CombineArchiveContent(
location='model.xml',
format='http://identifiers.org/combine.specifications/sbml',
),
], )
in_dir = os.path.join(self.tmp_dir, 'archive')
archive_filename = os.path.join(self.tmp_dir, 'archive.omex')
CombineArchiveWriter().run(archive, in_dir, archive_filename)
def sed_task_executer(task, variables):
pass
out_dir = os.path.join(self.tmp_dir, 'outputs')
def exec_sed_doc(task_executer,
filename,
working_dir,
base_out_dir,
rel_path,
apply_xml_model_changes=False,
indent=0,
log=None,
log_level=None,
config=None):
out_dir = os.path.join(base_out_dir, rel_path)
if not os.path.isdir(out_dir):
os.makedirs(out_dir)
with open(os.path.join(out_dir, 'report1.csv'), 'w') as file:
file.write('ABC')
with open(os.path.join(out_dir, 'report2.csv'), 'w') as file:
file.write('DEF')
with open(os.path.join(base_out_dir, 'reports.h5'), 'w') as file:
file.write('DEF')
return ReportResults({
'report1': 'ABC',
'report2': 'DEF',
}), None
with mock.patch('biosimulators_utils.sedml.exec.exec_sed_doc',
side_effect=exec_sed_doc):
sed_doc = SedDocument(
tasks=[Task(id='task_1')],
outputs=[Report(id='output_1')],
)
with mock.patch.object(SedmlSimulationReader,
'run',
return_value=sed_doc):
sed_doc_executer = functools.partial(exec_sed_doc,
sed_task_executer)
config = get_config()
config.REPORT_FORMATS = [ReportFormat.h5, ReportFormat.csv]
config.VIZ_FORMATS = []
config.BUNDLE_OUTPUTS = True
config.KEEP_INDIVIDUAL_OUTPUTS = True
config.COLLECT_COMBINE_ARCHIVE_RESULTS = False
results, _ = exec.exec_sedml_docs_in_archive(sed_doc_executer,
archive_filename,
out_dir,
config=config)
self.assertEqual(results, None)
config.COLLECT_COMBINE_ARCHIVE_RESULTS = True
results, _ = exec.exec_sedml_docs_in_archive(sed_doc_executer,
archive_filename,
out_dir,
config=config)
self.assertEqual(
results,
SedDocumentResults({
'sim.sedml':
ReportResults({
'report1': 'ABC',
'report2': 'DEF',
})
}))
self.assertEqual(
sorted(os.listdir(out_dir)),
sorted(['reports.h5', 'reports.zip', 'sim.sedml', 'log.yml']))
self.assertEqual(
sorted(os.listdir(os.path.join(out_dir, 'sim.sedml'))),
sorted(['report1.csv', 'report2.csv']))
archive.contents[0].format = CombineArchiveContentFormat.TEXT
CombineArchiveWriter().run(archive, in_dir, archive_filename)
with self.assertRaisesRegex(
NoSedmlError, 'does not contain any executing SED-ML files'):
with mock.patch('biosimulators_utils.sedml.exec.exec_sed_doc',
side_effect=exec_sed_doc):
sed_doc_executer = functools.partial(exec_sed_doc,
sed_task_executer)
config = get_config()
config.REPORT_FORMATS = [ReportFormat.h5, ReportFormat.csv]
config.VIZ_FORMATS = []
config.BUNDLE_OUTPUTS = True
config.KEEP_INDIVIDUAL_OUTPUTS = True
config.DEBUG = True
exec.exec_sedml_docs_in_archive(sed_doc_executer,
archive_filename,
out_dir,
config=config)
archive.contents[
0].format = 'http://identifiers.org/combine.specifications/sed-ml'
CombineArchiveWriter().run(archive, in_dir, archive_filename)
out_dir = os.path.join(self.tmp_dir, 'outputs-with-error')
def exec_sed_doc(task_executer,
filename,
working_dir,
base_out_dir,
rel_path,
apply_xml_model_changes=False,
indent=0,
log=None,
log_level=None,
config=None):
out_dir = os.path.join(base_out_dir, rel_path)
if not os.path.isdir(out_dir):
os.makedirs(out_dir)
with open(os.path.join(out_dir, 'report1.csv'), 'w') as file:
file.write('ABC')
with open(os.path.join(out_dir, 'report2.csv'), 'w') as file:
file.write('DEF')
with open(os.path.join(base_out_dir, 'reports.h5'), 'w') as file:
file.write('DEF')
raise ValueError('An error')
sed_doc = SedDocument(
tasks=[Task(id='task_1')],
outputs=[Report(id='output_1')],
)
with mock.patch.object(SedmlSimulationReader,
'run',
return_value=sed_doc):
sed_doc_executer = functools.partial(exec_sed_doc,
sed_task_executer)
with self.assertRaisesRegex(CombineArchiveExecutionError,
'An error'):
config = get_config()
config.REPORT_FORMATS = [ReportFormat.h5, ReportFormat.csv]
config.VIZ_FORMATS = []
config.BUNDLE_OUTPUTS = True
config.KEEP_INDIVIDUAL_OUTPUTS = True
_, log = exec.exec_sedml_docs_in_archive(sed_doc_executer,
archive_filename,
out_dir,
config=config)
if log.exception:
raise log.exception
self.assertEqual(
sorted(os.listdir(out_dir)),
sorted(['reports.h5', 'reports.zip', 'sim.sedml', 'log.yml']))
self.assertEqual(
sorted(os.listdir(os.path.join(out_dir, 'sim.sedml'))),
sorted(['report1.csv', 'report2.csv']))
def test_exec_sedml_docs_in_archive_without_log(self):
archive = CombineArchive(contents=[
CombineArchiveContent(
location='sim.sedml',
format='http://identifiers.org/combine.specifications/sed-ml',
),
CombineArchiveContent(
location='model.xml',
format='http://identifiers.org/combine.specifications/sbml',
),
], )
sed_doc = SedDocument()
model = Model(id='model_1',
source='model.xml',
language=ModelLanguage.SBML.value)
sed_doc.models.append(model)
sim = UniformTimeCourseSimulation(
id='sim_1',
initial_time=0.,
output_start_time=0.,
output_end_time=10.,
number_of_points=10,
algorithm=Algorithm(kisao_id='KISAO_0000019'))
sed_doc.simulations.append(sim)
task = Task(id='task_1', model=model, simulation=sim)
sed_doc.tasks.append(task)
sed_doc.data_generators.append(
DataGenerator(
id='data_gen_1',
variables=[
Variable(
id='var_1',
target=
"/sbml:sbml/sbml:model/sbml:listOfSpecies/sbml:species[@id='Trim']",
target_namespaces={
'sbml': 'http://www.sbml.org/sbml/level2/version4'
},
task=task)
],
math='var_1',
))
sed_doc.data_generators.append(
DataGenerator(
id='data_gen_2',
variables=[
Variable(
id='var_2',
target=
"/sbml:sbml/sbml:model/sbml:listOfSpecies/sbml:species[@id='Clb']",
target_namespaces={
'sbml': 'http://www.sbml.org/sbml/level2/version4'
},
task=task)
],
math='var_2',
))
report = Report(id='output_1')
sed_doc.outputs.append(report)
report.data_sets.append(
DataSet(id='data_set_1',
label='data_set_1',
data_generator=sed_doc.data_generators[0]))
report.data_sets.append(
DataSet(id='data_set_2',
label='data_set_2',
data_generator=sed_doc.data_generators[1]))
archive_dirname = os.path.join(self.tmp_dir, 'archive')
os.makedirs(archive_dirname)
shutil.copyfile(
os.path.join(os.path.dirname(__file__), '..', 'fixtures',
'BIOMD0000000297.xml'),
os.path.join(archive_dirname, 'model.xml'))
SedmlSimulationWriter().run(sed_doc,
os.path.join(archive_dirname, 'sim.sedml'))
archive_filename = os.path.join(self.tmp_dir, 'archive.omex')
CombineArchiveWriter().run(archive, archive_dirname, archive_filename)
def sed_task_executer(task, variables, log=None, config=None):
if log:
log.algorithm = task.simulation.algorithm.kisao_id
log.simulator_details = {
'attrib': 'value',
}
return VariableResults({
'var_1':
numpy.linspace(0., 10., task.simulation.number_of_points + 1),
'var_2':
numpy.linspace(10., 20., task.simulation.number_of_points + 1),
}), log
def sed_task_executer_error(task, variables, log=None, config=None):
raise ValueError('Big error')
out_dir = os.path.join(self.tmp_dir, 'outputs')
config = get_config()
config.REPORT_FORMATS = []
config.VIZ_FORMATS = []
config.COLLECT_COMBINE_ARCHIVE_RESULTS = True
config.LOG = True
# with log
sed_doc_executer = functools.partial(sedml_exec.exec_sed_doc,
sed_task_executer)
results, log = exec.exec_sedml_docs_in_archive(
sed_doc_executer,
archive_filename,
out_dir,
apply_xml_model_changes=False,
config=config)
self.assertEqual(set(results.keys()), set(['sim.sedml']))
self.assertEqual(set(results['sim.sedml'].keys()), set(['output_1']))
self.assertEqual(set(results['sim.sedml']['output_1'].keys()),
set(['data_set_1', 'data_set_2']))
numpy.testing.assert_allclose(
results['sim.sedml']['output_1']['data_set_1'],
numpy.linspace(0., 10., 11))
numpy.testing.assert_allclose(
results['sim.sedml']['output_1']['data_set_2'],
numpy.linspace(10., 20., 11))
self.assertEqual(log.exception, None)
self.assertEqual(
log.sed_documents['sim.sedml'].tasks['task_1'].algorithm,
task.simulation.algorithm.kisao_id)
self.assertEqual(
log.sed_documents['sim.sedml'].tasks['task_1'].simulator_details,
{'attrib': 'value'})
sed_doc_executer = functools.partial(sedml_exec.exec_sed_doc,
sed_task_executer_error)
results, log = exec.exec_sedml_docs_in_archive(
sed_doc_executer,
archive_filename,
out_dir,
apply_xml_model_changes=False,
config=config)
self.assertIsInstance(log.exception, CombineArchiveExecutionError)
config.DEBUG = True
sed_doc_executer = functools.partial(sedml_exec.exec_sed_doc,
sed_task_executer_error)
with self.assertRaisesRegex(ValueError, 'Big error'):
exec.exec_sedml_docs_in_archive(sed_doc_executer,
archive_filename,
out_dir,
apply_xml_model_changes=False,
config=config)
# without log
config.COLLECT_COMBINE_ARCHIVE_RESULTS = False
config.LOG = False
config.DEBUG = False
sed_doc_executer = functools.partial(sedml_exec.exec_sed_doc,
sed_task_executer)
results, log = exec.exec_sedml_docs_in_archive(
sed_doc_executer,
archive_filename,
out_dir,
apply_xml_model_changes=False,
config=config)
self.assertEqual(results, None)
self.assertEqual(log, None)
sed_doc_executer = functools.partial(sedml_exec.exec_sed_doc,
sed_task_executer_error)
with self.assertRaisesRegex(CombineArchiveExecutionError, 'Big error'):
exec.exec_sedml_docs_in_archive(sed_doc_executer,
archive_filename,
out_dir,
apply_xml_model_changes=False,
config=config)
config.DEBUG = True
sed_doc_executer = functools.partial(sedml_exec.exec_sed_doc,
sed_task_executer_error)
with self.assertRaisesRegex(ValueError, 'Big error'):
exec.exec_sedml_docs_in_archive(sed_doc_executer,
archive_filename,
out_dir,
apply_xml_model_changes=False,
config=config)
def test_2(self):
archive = CombineArchive(contents=[
CombineArchiveContent(
location='dir1/dir2/sim.sedml',
format='http://identifiers.org/combine.specifications/sed-ml',
),
CombineArchiveContent(
location='model.xml',
format='http://identifiers.org/combine.specifications/sbml',
),
], )
in_dir = os.path.join(self.tmp_dir, 'archive')
archive_filename = os.path.join(self.tmp_dir, 'archive.omex')
CombineArchiveWriter().run(archive, in_dir, archive_filename)
def sed_task_executer(task, variables):
pass
out_dir = os.path.join(self.tmp_dir, 'outputs')
config = get_config()
config.REPORT_FORMATS = [ReportFormat.csv]
config.VIZ_FORMATS = [VizFormat.pdf]
def exec_sed_doc(task_executer,
filename,
working_dir,
base_out_dir,
rel_path='.',
apply_xml_model_changes=False,
indent=0,
log=None,
log_level=None,
config=config):
out_dir = os.path.join(base_out_dir, rel_path)
if not os.path.isdir(out_dir):
os.makedirs(out_dir)
with open(os.path.join(out_dir, 'report1.csv'), 'w') as file:
file.write('ABC')
with open(os.path.join(out_dir, 'report2.csv'), 'w') as file:
file.write('DEF')
with open(os.path.join(out_dir, 'plot1.pdf'), 'w') as file:
file.write('GHI')
with open(os.path.join(out_dir, 'plot2.pdf'), 'w') as file:
file.write('JKL')
return None, None
with mock.patch('biosimulators_utils.sedml.exec.exec_sed_doc',
side_effect=exec_sed_doc):
with mock.patch.object(SedmlSimulationReader,
'run',
return_value=SedDocument()):
sed_doc_executer = functools.partial(exec_sed_doc,
sed_task_executer)
config = get_config()
config.REPORT_FORMATS = [ReportFormat.h5, ReportFormat.csv]
config.BUNDLE_OUTPUTS = True
config.KEEP_INDIVIDUAL_OUTPUTS = True
exec.exec_sedml_docs_in_archive(sed_doc_executer,
archive_filename,
out_dir,
config=config)
self.assertEqual(
sorted(os.listdir(out_dir)),
sorted(['reports.zip', 'plots.zip', 'dir1', 'log.yml']))
self.assertEqual(os.listdir(os.path.join(out_dir, 'dir1')), ['dir2'])
self.assertEqual(os.listdir(os.path.join(out_dir, 'dir1', 'dir2')),
['sim.sedml'])
self.assertEqual(
sorted(
os.listdir(os.path.join(out_dir, 'dir1', 'dir2',
'sim.sedml'))),
sorted(['report1.csv', 'report2.csv', 'plot1.pdf', 'plot2.pdf']))
archive_dir = os.path.join(self.tmp_dir, 'archive')
archive = ArchiveReader().run(os.path.join(out_dir, 'reports.zip'),
archive_dir)
self.assertEqual(
sorted(file.archive_path for file in archive.files),
sorted([
'dir1/dir2/sim.sedml/report1.csv',
'dir1/dir2/sim.sedml/report2.csv',
]))
with open(
os.path.join(archive_dir, 'dir1', 'dir2', 'sim.sedml',
'report1.csv'), 'r') as file:
self.assertEqual(file.read(), 'ABC')
with open(
os.path.join(archive_dir, 'dir1', 'dir2', 'sim.sedml',
'report2.csv'), 'r') as file:
self.assertEqual(file.read(), 'DEF')
archive = ArchiveReader().run(os.path.join(out_dir, 'plots.zip'),
archive_dir)
self.assertEqual(
sorted(file.archive_path for file in archive.files),
sorted([
'dir1/dir2/sim.sedml/plot1.pdf',
'dir1/dir2/sim.sedml/plot2.pdf',
]))
with open(
os.path.join(archive_dir, 'dir1', 'dir2', 'sim.sedml',
'plot1.pdf'), 'r') as file:
self.assertEqual(file.read(), 'GHI')
with open(
os.path.join(archive_dir, 'dir1', 'dir2', 'sim.sedml',
'plot2.pdf'), 'r') as file:
self.assertEqual(file.read(), 'JKL')
# don't bundle outputs, don't keep individual outputs
out_dir = os.path.join(self.tmp_dir, 'outputs-2')
os.makedirs(out_dir)
os.makedirs(os.path.join(out_dir, 'dir1'))
with open(os.path.join(out_dir, 'extra-file'), 'w'):
pass
with open(os.path.join(out_dir, 'dir1', 'extra-file'), 'w'):
pass
with mock.patch('biosimulators_utils.sedml.exec.exec_sed_doc',
side_effect=exec_sed_doc):
with mock.patch.object(SedmlSimulationReader,
'run',
return_value=SedDocument()):
sed_doc_executer = functools.partial(exec_sed_doc,
sed_task_executer)
config = get_config()
config.REPORT_FORMATS = [ReportFormat.h5, ReportFormat.csv]
config.BUNDLE_OUTPUTS = False
config.KEEP_INDIVIDUAL_OUTPUTS = False
exec.exec_sedml_docs_in_archive(sed_doc_executer,
archive_filename,
out_dir,
config=config)
self.assertEqual(sorted(os.listdir(out_dir)),
sorted(['log.yml', 'extra-file', 'dir1']))
self.assertEqual(sorted(os.listdir(os.path.join(out_dir, 'dir1'))),
sorted(['extra-file']))
out_dir = os.path.join(self.tmp_dir, 'outputs-3')
with mock.patch('biosimulators_utils.sedml.exec.exec_sed_doc',
side_effect=exec_sed_doc):
with mock.patch.object(SedmlSimulationReader,
'run',
return_value=SedDocument()):
sed_doc_executer = functools.partial(exec_sed_doc,
sed_task_executer)
exec.exec_sedml_docs_in_archive(sed_doc_executer,
archive_filename, out_dir)
self.assertIn('log.yml', os.listdir(out_dir))
def test_capturer_not_available(self):
archive = CombineArchive(contents=[
CombineArchiveContent(
location='dir1/dir2/sim.sedml',
format='http://identifiers.org/combine.specifications/sed-ml',
),
], )
in_dir = os.path.join(self.tmp_dir, 'archive')
archive_filename = os.path.join(self.tmp_dir, 'archive.omex')
CombineArchiveWriter().run(archive, in_dir, archive_filename)
def sed_task_executer(task, variables):
pass
out_dir = os.path.join(self.tmp_dir, 'outputs')
config = get_config()
config.REPORT_FORMATS = [ReportFormat.csv]
config.VIZ_FORMATS = []
config.BUNDLE_OUTPUTS = True
config.KEEP_INDIVIDUAL_OUTPUTS = True
def exec_sed_doc(task_executer,
filename,
working_dir,
base_out_dir,
rel_path='.',
apply_xml_model_changes=False,
indent=0,
log=None,
log_level=None,
config=config):
out_dir = os.path.join(base_out_dir, rel_path)
if not os.path.isdir(out_dir):
os.makedirs(out_dir)
with open(os.path.join(out_dir, 'report1.csv'), 'w') as file:
file.write('ABC')
return None, None
builtin_import = builtins.__import__
def import_mock(name, *args):
if name == 'capturer':
raise ModuleNotFoundError
return builtin_import(name, *args)
with mock.patch('builtins.__import__', side_effect=import_mock):
importlib.reload(log_utils)
with mock.patch('biosimulators_utils.sedml.exec.exec_sed_doc',
side_effect=exec_sed_doc):
with mock.patch.object(SedmlSimulationReader,
'run',
return_value=SedDocument()):
sed_doc_executer = functools.partial(
exec_sed_doc, sed_task_executer)
config = get_config()
config.BUNDLE_OUTPUTS = True
config.KEEP_INDIVIDUAL_OUTPUTS = True
_, log = exec.exec_sedml_docs_in_archive(sed_doc_executer,
archive_filename,
out_dir,
config=config)
self.assertNotEqual(log.output, None)
for doc_log in log.sed_documents.values():
self.assertNotEqual(doc_log.output, None)
importlib.reload(log_utils)
def test_validate_biosimulations_metadata_for_uri(self):
config = Config(
OMEX_METADATA_SCHEMA=data_model.OmexMetadataSchema.biosimulations)
md, _, _ = read_omex_meta_file(self.FIXTURE, config=config)
md = md[0]
errors, warnings = validate_biosimulations_metadata_for_uri(md)
self.assertEqual(errors, [])
self.assertIn('thumbnails could not be validated',
flatten_nested_list_of_strings(warnings))
errors, warnings = validate_biosimulations_metadata_for_uri(
md, working_dir=self.dir_name)
self.assertIn('is not a file', flatten_nested_list_of_strings(errors))
self.assertEqual(warnings, [])
shutil.copyfile(
os.path.join(self.FIXTURES_DIR, 'images',
'PNG_transparency_demonstration_1.png'),
os.path.join(self.dir_name, 'thumbnail.png'))
errors, warnings = validate_biosimulations_metadata_for_uri(
md, working_dir=self.dir_name)
self.assertEqual(errors, [])
self.assertEqual(warnings, [])
shutil.copyfile(
os.path.join(self.FIXTURES_DIR, 'images',
'PNG_transparency_demonstration_1.png'),
os.path.join(self.dir_name, 'thumbnail.png'))
archive = CombineArchive(contents=[
CombineArchiveContent(
location='thumbnail.png',
format=CombineArchiveContentFormat.PNG.value,
),
])
errors, warnings = validate_biosimulations_metadata_for_uri(
md,
validate_minimal_metadata=True,
archive=archive,
working_dir=self.dir_name)
self.assertEqual(errors, [])
self.assertEqual(warnings, [])
archive.contents[0].format = CombineArchiveContentFormat.PDF.value
errors, warnings = validate_biosimulations_metadata_for_uri(
md,
validate_minimal_metadata=True,
archive=archive,
working_dir=self.dir_name)
self.assertNotEqual(errors, [])
self.assertEqual(warnings, [])
md2 = copy.deepcopy(md)
md2['title'] = None
errors, warnings = validate_biosimulations_metadata_for_uri(
md2, validate_minimal_metadata=True, working_dir=self.dir_name)
self.assertIn('is required', flatten_nested_list_of_strings(errors))
self.assertEqual(warnings, [])
md2 = copy.deepcopy(md)
md2['creators'][0]['uri'] = 'xyz'
errors, warnings = validate_biosimulations_metadata_for_uri(
md2, validate_minimal_metadata=True, working_dir=self.dir_name)
self.assertIn('is not a valid URI',
flatten_nested_list_of_strings(errors))
self.assertEqual(warnings, [])
md2 = copy.deepcopy(md)
md2['created'] = 'xyz'
errors, warnings = validate_biosimulations_metadata_for_uri(
md2, validate_minimal_metadata=True, working_dir=self.dir_name)
self.assertIn('is not a valid date',
flatten_nested_list_of_strings(errors))
self.assertEqual(warnings, [])
md2 = copy.deepcopy(md)
md2['modified'].append('xyz')
errors, warnings = validate_biosimulations_metadata_for_uri(
md2, validate_minimal_metadata=True, working_dir=self.dir_name)
self.assertIn('is not a valid date',
flatten_nested_list_of_strings(errors))
self.assertEqual(warnings, [])
md2 = copy.deepcopy(md)
md2['creators'][0]['uri'] = 'http://identifiers.org/pubmed:1234'
errors, warnings = validate_biosimulations_metadata_for_uri(
md2, validate_minimal_metadata=True, working_dir=self.dir_name)
self.assertEqual(errors, [])
self.assertEqual(warnings, [])
md2 = copy.deepcopy(md)
md2['creators'][0]['uri'] = 'http://identifiers.org/pubmed/1234'
errors, warnings = validate_biosimulations_metadata_for_uri(
md2, validate_minimal_metadata=True, working_dir=self.dir_name)
self.assertEqual(errors, [])
self.assertEqual(warnings, [])
md2 = copy.deepcopy(md)
md2['creators'][0]['uri'] = 'http://identifiers.org/PubMed:1234'
errors, warnings = validate_biosimulations_metadata_for_uri(
md2, validate_minimal_metadata=True, working_dir=self.dir_name)
self.assertEqual(errors, [])
self.assertEqual(warnings, [])
md2 = copy.deepcopy(md)
md2['creators'][0]['uri'] = 'http://identifiers.org/ncbi/pubmed:1234'
errors, warnings = validate_biosimulations_metadata_for_uri(
md2, validate_minimal_metadata=True, working_dir=self.dir_name)
self.assertEqual(errors, [])
self.assertEqual(warnings, [])
md2 = copy.deepcopy(md)
md2['creators'][0]['uri'] = 'http://identifiers.org/NCBI/pubmed:1234'
errors, warnings = validate_biosimulations_metadata_for_uri(
md2, validate_minimal_metadata=True, working_dir=self.dir_name)
self.assertEqual(errors, [])
self.assertEqual(warnings, [])
md2 = copy.deepcopy(md)
md2['creators'][0][
'uri'] = 'http://identifiers.org/not-a-namespace:invalid'
errors, warnings = validate_biosimulations_metadata_for_uri(
md2, validate_minimal_metadata=True, working_dir=self.dir_name)
self.assertIn('is not a valid prefix',
flatten_nested_list_of_strings(errors))
self.assertEqual(warnings, [])
md2 = copy.deepcopy(md)
md2['creators'][0]['uri'] = 'http://identifiers.org/pubmed:invalid'
errors, warnings = validate_biosimulations_metadata_for_uri(
md2, validate_minimal_metadata=True, working_dir=self.dir_name)
self.assertIn('is not valid for',
flatten_nested_list_of_strings(errors))
self.assertIn('not a valid Identifiers.org identifier',
flatten_nested_list_of_strings(errors))
self.assertEqual(warnings, [])
md2 = copy.deepcopy(md)
md2['creators'][0]['uri'] = 'http://identifiers.org/ncbi:pubmed:1234'
errors, warnings = validate_biosimulations_metadata_for_uri(
md2, validate_minimal_metadata=True, working_dir=self.dir_name)
self.assertIn('is not a valid prefix',
flatten_nested_list_of_strings(errors))
self.assertEqual(warnings, [])
md2 = copy.deepcopy(md)
md2['thumbnails'][0] = 'x'
errors, warnings = validate_biosimulations_metadata_for_uri(
md2, validate_minimal_metadata=True, working_dir=self.dir_name)
self.assertIn('is not a file', flatten_nested_list_of_strings(errors))
self.assertEqual(warnings, [])
md2 = copy.deepcopy(md)
errors, warnings = validate_biosimulations_metadata_for_uri(
md2,
validate_minimal_metadata=True,
working_dir=self.dir_name,
archive=CombineArchive(contents=[
CombineArchiveContent(
location=os.path.relpath(md2['thumbnails'][0], '.'),
format=CombineArchiveContentFormat.TEXT.value,
)
]))
self.assertIn('must be one of the following',
flatten_nested_list_of_strings(errors))
self.assertEqual(warnings, [])
md2 = copy.deepcopy(md)
md2['created'] = 'x'
errors, warnings = validate_biosimulations_metadata_for_uri(
md2, validate_minimal_metadata=True, working_dir=self.dir_name)
self.assertIn('not a valid date',
flatten_nested_list_of_strings(errors))
self.assertEqual(warnings, [])
md2 = copy.deepcopy(md)
md2['modified'][0] = 'x'
errors, warnings = validate_biosimulations_metadata_for_uri(
md2, validate_minimal_metadata=True, working_dir=self.dir_name)
self.assertIn('not a valid date',
flatten_nested_list_of_strings(errors))
self.assertEqual(warnings, [])
def handler(body, files=None):
''' Create a COMBINE/OMEX archive.
Args:
body (:obj:`dict`): dictionary with schema ``CreateCombineArchiveSpecsAndFiles`` with the
specifications of the COMBINE/OMEX archive to create
files (:obj:`list` of :obj:`werkzeug.datastructures.FileStorage`, optional): files (e.g., SBML
file)
Returns:
:obj:`werkzeug.wrappers.response.Response` or :obj:`str`: response with COMBINE/OMEX
archive or a URL to a COMBINE/OMEX archive
'''
download = body.get('download', False)
archive_specs = body['specs']
files = connexion.request.files.getlist('files')
# create temporary working directory
temp_dirname = get_temp_dir()
# create temporary files for archive
archive_dirname = os.path.join(temp_dirname, 'archive')
archive_filename = os.path.join(temp_dirname, 'archive.omex')
# initialize archive
archive = CombineArchive()
# build map from model filenames to file objects
filename_map = {file.filename: file for file in files}
# add files to archive
for content in archive_specs['contents']:
content_type = content['location']['value']['_type']
if content_type == 'SedDocument':
sed_doc = export_sed_doc(content['location']['value'])
# save SED-ML document to file
try:
SedmlSimulationWriter().run(
sed_doc,
os.path.join(archive_dirname, content['location']['path']),
validate_models_with_languages=False)
except ValueError as exception:
raise BadRequestException(
title=
'`{}` does not contain a configuration for a valid SED-ML document.'
.format(content['location']['value']),
instance=exception,
)
elif content_type == 'CombineArchiveContentFile':
file = filename_map.get(content['location']['value']['filename'],
None)
if not file:
raise BadRequestException(
title='File with name `{}` was not uploaded'.format(
content['location']['value']['filename']),
instance=ValueError(),
)
filename = os.path.join(archive_dirname,
content['location']['path'])
if not os.path.isdir(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
file.save(filename)
elif content_type == 'CombineArchiveContentUrl':
filename = os.path.join(archive_dirname,
content['location']['path'])
if not os.path.isdir(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
content_url = content['location']['value']['url']
try:
response = requests.get(content_url)
response.raise_for_status()
except requests.exceptions.RequestException as exception:
title = 'COMBINE/OMEX archive content could not be loaded from `{}`'.format(
content_url)
raise BadRequestException(
title=title,
instance=exception,
)
with open(filename, 'wb') as file:
file.write(response.content)
else:
raise BadRequestException(
title='Content of type `{}` is not supported'.format(
content_type),
instance=NotImplementedError('Invalid content')
) # pragma: no cover: unreachable due to schema validation
content = CombineArchiveContent(
location=content['location']['path'],
format=content['format'],
master=content['master'],
)
archive.contents.append(content)
# package COMBINE/OMEX archive
CombineArchiveWriter().run(archive, archive_dirname, archive_filename)
if download:
return flask.send_file(archive_filename,
mimetype='application/zip',
as_attachment=True,
attachment_filename='archive.omex')
else:
# save COMBINE/OMEX archive to S3 bucket
archive_url = src.s3.save_temporary_combine_archive_to_s3_bucket(
archive_filename, public=True)
# return URL for archive in S3 bucket
return archive_url
def _build_combine_archive(self):
task = Task(
id='task',
model=Model(
id='model',
source='bounce1.txt',
language=ModelLanguage.Smoldyn.value,
),
simulation=UniformTimeCourseSimulation(
id='sim',
initial_time=0.,
output_start_time=0.1,
output_end_time=0.2,
number_of_points=10,
algorithm=Algorithm(kisao_id='KISAO_0000057',
changes=[
AlgorithmParameterChange(
kisao_id='KISAO_0000488',
new_value='10'),
])),
)
variables = [
Variable(id='time', symbol=Symbol.time.value, task=task),
Variable(id='red', target='molcount red', task=task),
Variable(id='green', target='molcount green', task=task),
]
doc = SedDocument(
models=[task.model],
simulations=[task.simulation],
tasks=[task],
data_generators=[
DataGenerator(
id='data_gen_time',
variables=[
Variable(id='var_time',
symbol=Symbol.time.value,
task=task)
],
math='var_time',
),
DataGenerator(
id='data_gen_red',
variables=[
Variable(id='var_red',
target='molcount red',
task=task)
],
math='var_red',
),
DataGenerator(
id='data_gen_green',
variables=[
Variable(id='var_green',
target='molcount green',
task=task)
],
math='var_green',
),
],
)
doc.outputs.append(
Report(id='report',
data_sets=[
DataSet(id='data_set_time',
label='time',
data_generator=doc.data_generators[0]),
DataSet(id='data_set_red',
label='red',
data_generator=doc.data_generators[1]),
DataSet(id='data_set_green',
label='green',
data_generator=doc.data_generators[2]),
]))
archive_dirname = os.path.join(self.dirname, 'archive')
os.makedirs(archive_dirname)
shutil.copyfile(
os.path.join(self.EXAMPLES_DIRNAME, 'S1_intro', 'bounce1.txt'),
os.path.join(archive_dirname, 'bounce1.txt'))
sim_filename = os.path.join(archive_dirname, 'sim_1.sedml')
SedmlSimulationWriter().run(doc, sim_filename)
archive = CombineArchive(contents=[
CombineArchiveContent('bounce1.txt',
CombineArchiveContentFormat.Smoldyn.value),
CombineArchiveContent('sim_1.sedml',
CombineArchiveContentFormat.SED_ML.value),
], )
archive_filename = os.path.join(self.dirname, 'archive.omex')
CombineArchiveWriter().run(archive, archive_dirname, archive_filename)
return doc, archive_filename
def test_no_validation(self):
archive_dirname = os.path.join(self.tmp_dir, 'archive')
os.mkdir(archive_dirname)
# OMEX manifests
archive = CombineArchive()
errors, warnings = validate(archive, archive_dirname)
self.assertIn('must have at least one content',
flatten_nested_list_of_strings(errors))
self.assertNotEqual(warnings, [])
with mock.patch.dict('os.environ', {'VALIDATE_OMEX_MANIFESTS': '0'}):
errors, warnings = validate(archive, archive_dirname)
self.assertEqual(errors, [])
self.assertEqual(warnings, [])
# SED-ML
archive = CombineArchive()
archive.contents.append(
CombineArchiveContent(
location='simulation.sedml',
format=CombineArchiveContentFormat.SED_ML.value,
))
sedml_filename = os.path.join(archive_dirname, 'simulation.sedml')
with open(sedml_filename, 'w') as file:
file.write('invalid')
errors, warnings = validate(archive, archive_dirname)
self.assertIn('Missing XML declaration',
flatten_nested_list_of_strings(errors))
self.assertEqual(warnings, [])
with mock.patch.dict('os.environ', {
'VALIDATE_OMEX_MANIFESTS': '0',
'VALIDATE_SEDML': '0',
}):
errors, warnings = validate(archive, archive_dirname)
self.assertEqual(errors, [])
self.assertEqual(warnings, [])
os.remove(sedml_filename)
# models
archive = CombineArchive()
archive.contents.append(
CombineArchiveContent(
location='simulation.sedml',
format=CombineArchiveContentFormat.SED_ML.value,
))
model_filename = os.path.join(archive_dirname, 'model.xml')
shutil.copyfile(
os.path.join(os.path.dirname(__file__), '..', 'fixtures',
'BIOMD0000000297.xml'), model_filename)
sed_doc = SedDocument()
sed_doc.models.append(
Model(id='model',
source=model_filename,
language=ModelLanguage.SBML.value))
sedml_filename = os.path.join(archive_dirname, 'simulation.sedml')
SedmlSimulationWriter().run(sed_doc, sedml_filename)
with open(model_filename, 'w') as file:
file.write('invalid')
errors, warnings = validate(archive, archive_dirname)
self.assertIn('Missing XML declaration',
flatten_nested_list_of_strings(errors))
self.assertEqual(warnings, [])
with mock.patch.dict('os.environ', {
'VALIDATE_OMEX_MANIFESTS': '0',
'VALIDATE_SEDML_MODELS': '0',
}):
errors, warnings = validate(archive, archive_dirname)
self.assertEqual(errors, [])
self.assertEqual(warnings, [])
os.remove(sedml_filename)
os.remove(model_filename)
# images
archive = CombineArchive()
archive.contents.append(
CombineArchiveContent(
location='image.png',
format=CombineArchiveContentFormat.PNG.value,
))
errors, warnings = validate(
archive,
archive_dirname,
formats_to_validate=[CombineArchiveContentFormat.PNG])
self.assertIn('The PNG file at location `image.png` is invalid.',
flatten_nested_list_of_strings(errors))
self.assertNotEqual(warnings, [])
with mock.patch.dict('os.environ', {
'VALIDATE_OMEX_MANIFESTS': '0',
'VALIDATE_IMAGES': '0',
}):
errors, warnings = validate(
archive,
archive_dirname,
formats_to_validate=[CombineArchiveContentFormat.PNG])
self.assertEqual(errors, [])
self.assertEqual(warnings, [])
# OMEX metadata
archive = CombineArchive()
archive.contents.append(
CombineArchiveContent(
location='metadata.rdf',
format=CombineArchiveContentFormat.OMEX_METADATA.value,
))
metadata_file = os.path.join(archive_dirname, 'metadata.rdf')
shutil.copyfile(
os.path.join(os.path.dirname(__file__), '..', 'fixtures',
'omex-metadata', 'invalid.rdf'), metadata_file)
errors, warnings = validate(
archive,
archive_dirname,
formats_to_validate=[CombineArchiveContentFormat.OMEX_METADATA])
self.assertIn(
'The OMEX Metadata file at location `metadata.rdf` is invalid.',
flatten_nested_list_of_strings(errors))
self.assertNotEqual(warnings, [])
with mock.patch.dict('os.environ', {
'VALIDATE_OMEX_MANIFESTS': '0',
'VALIDATE_OMEX_METADATA': '0',
}):
errors, warnings = validate(
archive,
archive_dirname,
formats_to_validate=[
CombineArchiveContentFormat.OMEX_METADATA
])
self.assertEqual(errors, [])
self.assertEqual(warnings, [])
os.remove(metadata_file)
def test_error_handling(self):
os.remove(os.path.join(self.tmp_dir, 'thumbnail.png'))
archive = CombineArchive()
errors, warnings = validate(archive, self.tmp_dir)
self.assertEqual(len(errors), 1)
self.assertEqual(len(errors[0]), 1)
self.assertIn('must have at least one content element', errors[0][0])
self.assertEqual(len(warnings), 1)
self.assertEqual(len(warnings[0]), 1)
self.assertIn('does not contain any SED-ML files', warnings[0][0])
archive = CombineArchive(contents=[
None,
])
errors, warnings = validate(archive, self.tmp_dir)
self.assertEqual(len(errors), 1)
self.assertEqual(len(warnings), 1)
self.assertIn('must be an instance of',
flatten_nested_list_of_strings(errors))
self.assertIn('does not contain any SED-ML files',
flatten_nested_list_of_strings(warnings))
archive = CombineArchive(contents=[
CombineArchiveContent(),
])
errors, warnings = validate(archive, self.tmp_dir)
self.assertEqual(len(errors), 1)
self.assertEqual(len(warnings), 1)
self.assertIn('must have a location',
flatten_nested_list_of_strings(errors))
self.assertIn('must have a format',
flatten_nested_list_of_strings(errors))
self.assertIn('does not contain any SED-ML files',
flatten_nested_list_of_strings(warnings))
archive = CombineArchive(contents=[
CombineArchiveContent(
location='plain.txt',
format='plain/text',
),
])
errors, warnings = validate(archive, self.tmp_dir)
self.assertEqual(len(errors), 1)
self.assertEqual(len(warnings), 1)
self.assertIn('is not a file', flatten_nested_list_of_strings(errors))
self.assertIn('does not contain any SED-ML files',
flatten_nested_list_of_strings(warnings))
with open(os.path.join(self.tmp_dir, 'sim.sedml'), 'w') as file:
pass
archive = CombineArchive(contents=[
CombineArchiveContent(
location='sim.sedml',
format=CombineArchiveContentFormat.SED_ML,
),
])
errors, warnings = validate(archive, self.tmp_dir)
self.assertEqual(len(errors), 1)
self.assertEqual(warnings, [])
self.assertIn('is invalid', flatten_nested_list_of_strings(errors))
archive = CombineArchive(contents=[
CombineArchiveContent(
location='sim.sedml',
format=CombineArchiveContentFormat.SED_ML,
),
])
with mock.patch.object(SedmlSimulationReader,
'run',
side_effect=ValueError('other error')):
with self.assertRaisesRegex(ValueError, 'other error'):
validate(archive, self.tmp_dir)
def side_effect(self,
filename,
validate_models_with_languages=False,
config=None):
self.warnings = [['my warning']]
with mock.patch.object(SedmlSimulationReader, 'run', side_effect):
errors, warnings = validate(archive, self.tmp_dir)
self.assertEqual(errors, [])
self.assertEqual(len(warnings), 1)
self.assertIn('may be invalid',
flatten_nested_list_of_strings(warnings))
self.assertIn('my warning', flatten_nested_list_of_strings(warnings))