def test_get_config(self):
with mock.patch.dict(os.environ, {
'REPORT_FORMATS': '',
'VIZ_FORMATS': ''
}):
config = get_config()
self.assertEqual(config.REPORT_FORMATS, [])
self.assertEqual(config.VIZ_FORMATS, [])
with mock.patch.dict(os.environ, {
'REPORT_FORMATS': 'h5',
'VIZ_FORMATS': 'pdf'
}):
config = get_config()
self.assertEqual(config.REPORT_FORMATS, [ReportFormat.h5])
self.assertEqual(config.VIZ_FORMATS, [VizFormat.pdf])
def test_exec_sedml_docs_in_combine_archive(self):
doc, archive_filename = self._build_combine_archive()
out_dir = os.path.join(self.dirname, 'out')
config = get_config()
config.REPORT_FORMATS = [ReportFormat.h5]
_, log = smoldyn.biosimulators.combine.exec_sedml_docs_in_combine_archive(
archive_filename, out_dir, config=config)
if log.exception:
raise log.exception
self._assert_combine_archive_outputs(doc, out_dir)
def test_exec_sedml_docs_in_combine_archive(self):
doc, archive_filename = self._build_combine_archive()
out_dir = os.path.join(self.dirname, 'out')
config = get_config()
config.REPORT_FORMATS = [report_data_model.ReportFormat.h5, report_data_model.ReportFormat.csv]
config.BUNDLE_OUTPUTS = True
config.KEEP_INDIVIDUAL_OUTPUTS = True
_, log = core.exec_sedml_docs_in_combine_archive(archive_filename, out_dir, config=config)
if log.exception:
raise log.exception
self._assert_combine_archive_outputs(doc, out_dir)
def test_more_complex_archive(self):
archive_filename = os.path.join(os.path.dirname(__file__), 'fixtures', 'BIOMD0000000297.edited.omex')
config = get_config()
config.REPORT_FORMATS = [report_data_model.ReportFormat.h5, report_data_model.ReportFormat.csv]
config.VIZ_FORMATS = []
config.BUNDLE_OUTPUTS = True
config.KEEP_INDIVIDUAL_OUTPUTS = True
_, log = core.exec_sedml_docs_in_combine_archive(archive_filename, self.dirname, config=config)
if log.exception:
raise log.exception
self.assertEqual(set(['reports.zip', 'reports.h5', 'ex1', 'ex2']).difference(set(os.listdir(self.dirname))), set())
self.assertEqual(set(os.listdir(os.path.join(self.dirname, 'ex1'))), set(['BIOMD0000000297.sedml']))
self.assertEqual(set(os.listdir(os.path.join(self.dirname, 'ex2'))), set(['BIOMD0000000297.sedml']))
self.assertEqual(set(os.listdir(os.path.join(self.dirname, 'ex1', 'BIOMD0000000297.sedml'))),
set(['two_species.csv', 'three_species.csv']))
self.assertEqual(set(os.listdir(os.path.join(self.dirname, 'ex2', 'BIOMD0000000297.sedml'))),
set(['one_species.csv', 'four_species.csv']))
archive = ArchiveReader().run(os.path.join(self.dirname, 'reports.zip'))
self.assertEqual(
sorted(file.archive_path for file in archive.files),
sorted([
'ex1/BIOMD0000000297.sedml/two_species.csv',
'ex1/BIOMD0000000297.sedml/three_species.csv',
'ex2/BIOMD0000000297.sedml/one_species.csv',
'ex2/BIOMD0000000297.sedml/four_species.csv',
]),
)
report = sedml_data_model.Report(
data_sets=[
sedml_data_model.DataSet(id='data_set_time_two_species', label='time'),
sedml_data_model.DataSet(id='data_set_Cln4', label='Cln4'),
sedml_data_model.DataSet(id='data_set_Swe13', label='Swe13'),
]
)
report_results = ReportReader().run(report, self.dirname,
'ex1/BIOMD0000000297.sedml/two_species',
format=report_data_model.ReportFormat.h5)
self.assertEqual(sorted(report_results.keys()), sorted(['data_set_time_two_species', 'data_set_Cln4', 'data_set_Swe13']))
numpy.testing.assert_allclose(report_results['data_set_time_two_species'], numpy.linspace(0., 1., 10 + 1))
def test_exec_sedml_docs_in_combine_archive_with_all_algorithms(self):
for alg in gen_algorithms_from_specs(
os.path.join(os.path.dirname(__file__), '..',
'biosimulators.json')).values():
doc, archive_filename = self._build_combine_archive(algorithm=alg)
out_dir = os.path.join(self.dirname, alg.kisao_id)
config = get_config()
config.REPORT_FORMATS = [
report_data_model.ReportFormat.h5,
report_data_model.ReportFormat.csv
]
config.BUNDLE_OUTPUTS = True
config.KEEP_INDIVIDUAL_OUTPUTS = True
_, log = exec_sedml_docs_in_combine_archive(archive_filename,
out_dir,
config=config)
if log.exception:
raise log.exception
self._assert_combine_archive_outputs(doc, out_dir)
def handler(body, archiveFile=None):
""" Execute the SED-ML files in a COMBINE/OMEX archive.
Args:
body (:obj:`dict`): dictionary with schema ``SimulationRun`` with the
specifications of the COMBINE/OMEX archive to execute and the simulator to execute it with
archiveFile (:obj:`werkzeug.datastructures.FileStorage`, optional): COMBINE/OMEX file
Returns:
:obj:`werkzeug.wrappers.response.Response`: response with the results and log of the run in the
``SimulationRunResults`` schema
"""
archive_file = archiveFile
archive_url = body.get('archiveUrl', None)
simulator_id = body['simulator']
env_vars = body.get('environment', {}).get('variables', [])
# set up environment (i.e. options)
env = {}
for env_var in env_vars:
key = env_var['key']
if key not in IGNORED_ENV_VARS:
env[key] = env_var['value']
if 'REPORT_FORMATS' not in env:
env['REPORT_FORMATS'] = 'h5'
with mock.patch.dict('os.environ', env):
config = get_config()
# process requested return type
accept = connexion.request.headers.get('Accept', 'application/json')
if accept in ['application/json']:
config.COLLECT_COMBINE_ARCHIVE_RESULTS = True
config.COLLECT_SED_DOCUMENT_RESULTS = True
config.REPORT_FORMATS = []
config.VIZ_FORMATS = []
config.BUNDLE_OUTPUTS = False
config.KEEP_INDIVIDUAL_OUTPUTS = True
config.LOG_PATH = ''
return_type = 'json'
elif accept in ['application/x-hdf', 'application/x-hdf5']:
config.COLLECT_COMBINE_ARCHIVE_RESULTS = False
config.COLLECT_SED_DOCUMENT_RESULTS = False
config.REPORT_FORMATS = [
ReportFormat[format.strip().lower()]
for format in env.get('REPORT_FORMATS', 'h5').split(',')
]
config.VIZ_FORMATS = []
config.BUNDLE_OUTPUTS = False
config.KEEP_INDIVIDUAL_OUTPUTS = True
config.LOG_PATH = ''
return_type = 'h5'
elif accept in ['application/zip']:
config.COLLECT_COMBINE_ARCHIVE_RESULTS = False
config.COLLECT_SED_DOCUMENT_RESULTS = False
config.REPORT_FORMATS = [
ReportFormat[format.strip().lower()]
for format in env.get('REPORT_FORMATS', 'h5').split(',')
]
config.VIZ_FORMATS = [
VizFormat[format.strip().lower()]
for format in env.get('VIZ_FORMATS', 'pdf').split(',')
]
config.BUNDLE_OUTPUTS = False
config.KEEP_INDIVIDUAL_OUTPUTS = True
return_type = 'zip'
else:
raise BadRequestException(
title=
'`Accept` header must be one of `application/hdf5`, `application/json`, or `application/zip`.',
instance=NotImplementedError(),
)
# get the COMBINE/OMEX archive
if archive_file and archive_url:
raise BadRequestException(
title=
'Only one of `archiveFile` or `archiveUrl` can be used at a time.',
instance=ValueError(),
)
# get COMBINE/OMEX archive
archive_filename = get_temp_file(suffix='.omex')
if archive_file:
archive_file.save(archive_filename)
else:
try:
response = requests.get(archive_url)
response.raise_for_status()
except requests.exceptions.RequestException as exception:
title = 'COMBINE/OMEX archive could not be loaded from `{}`'.format(
archive_url)
raise BadRequestException(
title=title,
instance=exception,
)
# save archive to local temporary file
with open(archive_filename, 'wb') as file:
file.write(response.content)
# get the simulator
simulator = next(
(simulator
for simulator in get_simulators() if simulator['id'] == simulator_id),
None)
if simulator is None:
raise BadRequestException(
title=
'`{}` is not a BioSimulators id of a simulation tool that is available for execution.'
.format(simulator_id),
instance=ValueError(),
)
# execute the simulation
out_dir = get_temp_dir()
with mock.patch.dict('os.environ', env):
results, log = exec_in_subprocess(
use_simulator_api_to_exec_sedml_docs_in_combine_archive,
simulator['api']['module'],
archive_filename,
out_dir,
timeout=TIMEOUT,
config=config)
# transform the results
if return_type == 'json':
archive_dirname = get_temp_dir()
with zipfile.ZipFile(archive_filename, 'r') as zip_file:
zip_file.extractall(archive_dirname)
outputs = []
for sed_doc_location, sed_doc_outputs_results in (results
or {}).items():
sed_doc = SedmlSimulationReader().run(
os.path.join(archive_dirname, sed_doc_location))
for output in sed_doc.outputs:
if output.id not in sed_doc_outputs_results:
continue
output_results = sed_doc_outputs_results.get(output.id, None)
if isinstance(output, Report):
type = 'SedReport'
report = output
elif isinstance(output, Plot2D):
type = 'SedPlot2D'
report = get_report_for_plot2d(output)
elif isinstance(output, Plot3D):
type = 'SedPlot3D'
report = get_report_for_plot3d(output)
else: # pragma: no cover #
raise NotImplementedError(
'Outputs of type `{}` are not supported.'.format(
output.__class__.__name__))
data = []
for data_set in report.data_sets:
if data_set.id not in output_results:
continue
data_set_results = output_results[data_set.id]
data.append({
'_type':
'SimulationRunOutputDatum',
'id':
data_set.id,
'label':
data_set.label,
'name':
data_set.name,
'shape':
'' if data_set_results is None else ','.join(
str(dim_len)
for dim_len in data_set_results.shape),
'type':
'__None__' if data_set_results is None else
data_set_results.dtype.name,
'values':
None if data_set_results is None else
data_set_results.tolist(),
})
outputs.append({
'_type': 'SimulationRunOutput',
'outputId': sed_doc_location + '/' + output.id,
'name': output.name,
'type': type,
'data': data,
})
# return
return {
'_type': 'SimulationRunResults',
'outputs': outputs,
'log': log,
}
elif return_type == 'h5':
h5_filename = os.path.join(out_dir, get_config().H5_REPORTS_PATH)
return flask.send_file(h5_filename,
mimetype=accept,
as_attachment=True,
attachment_filename='outputs.h5')
else:
zip_filename = get_temp_file()
with zipfile.ZipFile(zip_filename, 'w') as zip_file:
for root, dirs, files in os.walk(out_dir):
for file in files:
zip_file.write(
os.path.join(root, file),
os.path.relpath(os.path.join(root, file), out_dir))
return flask.send_file(zip_filename,
mimetype=accept,
as_attachment=True,
attachment_filename='outputs.zip')
def exec_sed_task(task,
variables,
preprocessed_task=None,
log=None,
config=None):
''' Execute a task and save its results
Args:
task (:obj:`Task`): task
variables (:obj:`list` of :obj:`Variable`): variables that should be recorded
preprocessed_task (:obj:`dict`, optional): preprocessed information about the task, including possible
model changes and variables. This can be used to avoid repeatedly executing the same initialization
for repeated calls to this method.
log (:obj:`TaskLog`, optional): log for the task
config (:obj:`Config`, optional): BioSimulators common configuration
Returns:
:obj:`tuple`:
:obj:`VariableResults`: results of variables
:obj:`TaskLog`: log
'''
if not config:
config = get_config()
if config.LOG and not log:
log = TaskLog()
sed_model_changes = task.model.changes
sed_simulation = task.simulation
if preprocessed_task is None:
preprocessed_task = preprocess_sed_task(task, variables, config=config)
sed_model_changes = list(
filter(
lambda change: change.target in preprocessed_task[
'sed_smoldyn_simulation_change_map'], sed_model_changes))
# read Smoldyn configuration
smoldyn_simulation = preprocessed_task['simulation']
# apply model changes to the Smoldyn configuration
sed_smoldyn_simulation_change_map = preprocessed_task[
'sed_smoldyn_simulation_change_map']
for change in sed_model_changes:
smoldyn_change = sed_smoldyn_simulation_change_map.get(
change.target, None)
if smoldyn_change is None or smoldyn_change.execution != SimulationChangeExecution.simulation:
raise NotImplementedError(
'Target `{}` can only be changed during simulation preprocessing.'
.format(change.target))
apply_change_to_smoldyn_simulation(smoldyn_simulation, change,
smoldyn_change)
# get the Smoldyn representation of the SED uniform time course simulation
smoldyn_simulation_run_timecourse_args = get_smoldyn_run_timecourse_args(
sed_simulation)
# execute the simulation
smoldyn_run_args = dict(
**smoldyn_simulation_run_timecourse_args,
**preprocessed_task['simulation_run_alg_param_args'],
)
smoldyn_simulation.run(**smoldyn_run_args,
overwrite=True,
display=False,
quit_at_end=False)
# get the result of each SED variable
variable_output_cmd_map = preprocessed_task['variable_output_cmd_map']
smoldyn_output_files = preprocessed_task['output_files']
variable_results = get_variable_results(sed_simulation.number_of_steps,
variables, variable_output_cmd_map,
smoldyn_output_files)
# cleanup output files
for smoldyn_output_file in smoldyn_output_files.values():
os.remove(smoldyn_output_file.filename)
# log simulation
if config.LOG:
log.algorithm = sed_simulation.algorithm.kisao_id
log.simulator_details = {
'class': 'smoldyn.Simulation',
'instanceAttributes': preprocessed_task['simulation_attrs'],
'method': 'run',
'methodArguments': smoldyn_run_args,
}
# return the values of the variables and log
return variable_results, log
def exec_sed_task(task,
variables,
preprocessed_task=None,
log=None,
config=None):
''' Execute a task and save its results
Args:
task (:obj:`Task`): task
variables (:obj:`list` of :obj:`Variable`): variables that should be recorded
preprocessed_task (:obj:`dict`, optional): preprocessed information about the task, including possible
model changes and variables. This can be used to avoid repeatedly executing the same initialization
for repeated calls to this method.
log (:obj:`TaskLog`, optional): log for the task
config (:obj:`Config`, optional): BioSimulators common configuration
Returns:
:obj:`tuple`:
:obj:`VariableResults`: results of variables
:obj:`TaskLog`: log
Raises:
:obj:`ValueError`: if the task or an aspect of the task is not valid, or the requested output variables
could not be recorded
:obj:`NotImplementedError`: if the task is not of a supported type or involves an unsuported feature
'''
config = config or get_config()
if config.LOG and not log:
log = TaskLog()
if preprocessed_task is None:
preprocessed_task = preprocess_sed_task(task, variables, config=config)
sim = task.simulation
# get model
model = preprocessed_task['model']['model']
# modify model
raise_errors_warnings(
validation.validate_model_change_types(task.model.changes,
(ModelAttributeChange, )),
error_summary='Changes for model `{}` are not supported.'.format(
task.model.id))
change_target_model_obj_map = preprocessed_task['model'][
'change_target_model_obj_map']
for change in task.model.changes:
model_obj = change_target_model_obj_map[change.target]
new_value = float(change.new_value)
if isinstance(model_obj, gillespy2.core.parameter.Parameter):
model_obj.value = new_value
else:
model_obj.initial_value = new_value
# Validate that start time is 0 because this is the only option that GillesPy2 supports
if sim.initial_time < 0:
raise NotImplementedError(
'Negative initial simulation time {} is not supported. Initial time must be >= 0.'
.format(sim.initial_time))
# set the simulation time span
number_of_points = (sim.output_end_time - sim.initial_time) / \
(sim.output_end_time - sim.output_start_time) * sim.number_of_points
if number_of_points != math.floor(number_of_points):
raise NotImplementedError(
'Time course must specify an integer number of time points')
number_of_points = int(number_of_points)
timespan = numpy.linspace(sim.initial_time, sim.output_end_time,
number_of_points + 1)
model.timespan(timespan)
# Simulate the model from ``sim.start_time`` to ``sim.output_end_time``
# and record ``sim.number_of_points`` + 1 time points
solver = preprocessed_task['simulation']['solver']
solver_args = preprocessed_task['simulation']['solver_args']
results_dict = model.run(solver, **solver_args)[0]
# transform the results to an instance of :obj:`VariableResults`
variable_target_sbml_id_map = preprocessed_task['model'][
'variable_target_sbml_id_map']
variable_results = VariableResults()
parameters = model.get_all_parameters()
for variable in variables:
if variable.symbol:
variable_results[variable.id] = results_dict['time'][-(
sim.number_of_points + 1):]
elif variable.target:
sbml_id = variable_target_sbml_id_map[variable.target]
dynamics = results_dict.get(sbml_id, None)
if dynamics is None:
variable_results[variable.id] = numpy.full(
(sim.number_of_points + 1, ), parameters[sbml_id].value)
else:
variable_results[
variable.id] = dynamics[-(sim.number_of_points + 1):]
# log action
if config.LOG:
log.algorithm = preprocessed_task['simulation']['algorithm_kisao_id']
log.simulator_details = {
'method': solver.__module__ + '.' + solver.__name__,
'arguments': solver_args,
}
# return results and log
return variable_results, log
def preprocess_sed_task(task, variables, config=None):
""" Preprocess a SED task, including its possible model changes and variables. This is useful for avoiding
repeatedly initializing tasks on repeated calls of :obj:`exec_sed_task`.
Args:
task (:obj:`Task`): task
variables (:obj:`list` of :obj:`Variable`): variables that should be recorded
config (:obj:`Config`, optional): BioSimulators common configuration
Returns:
:obj:`dict`: preprocessed information about the task
"""
config = config or get_config()
sim = task.simulation
if config.VALIDATE_SEDML:
raise_errors_warnings(validation.validate_task(task),
error_summary='Task `{}` is invalid.'.format(
task.id))
raise_errors_warnings(
validation.validate_model_language(task.model.language,
ModelLanguage.SBML),
error_summary='Language for model `{}` is not supported.'.format(
task.model.id))
raise_errors_warnings(
validation.validate_model_change_types(task.model.changes,
(ModelAttributeChange, )),
error_summary='Changes for model `{}` are not supported.'.format(
task.model.id))
raise_errors_warnings(
*validation.validate_model_changes(task.model),
error_summary='Changes for model `{}` are invalid.'.format(
task.model.id))
raise_errors_warnings(validation.validate_simulation_type(
sim, (UniformTimeCourseSimulation, )),
error_summary='{} `{}` is not supported.'.format(
sim.__class__.__name__, sim.id))
raise_errors_warnings(
*validation.validate_simulation(sim),
error_summary='Simulation `{}` is invalid.'.format(sim.id))
raise_errors_warnings(
*validation.validate_data_generator_variables(variables),
error_summary='Data generator variables for task `{}` are invalid.'
.format(task.id))
model_etree = lxml.etree.parse(task.model.source)
change_target_sbml_id_map = validation.validate_target_xpaths(
task.model.changes, model_etree, attr='id')
variable_target_sbml_id_map = validation.validate_target_xpaths(
variables, model_etree, attr='id')
# Read the SBML-encoded model located at `task.model.source`
model, errors = gillespy2.import_SBML(task.model.source)
if model is None or errors:
raise ValueError('Model at {} could not be imported:\n - {}'.format(
task.model.source,
'\n - '.join(message for message, code in errors)))
# preprocess model changes
parameters = model.get_all_parameters()
species = model.get_all_species()
change_target_model_obj_map = {}
invalid_changes = []
for change in task.model.changes:
sbml_id = change_target_sbml_id_map[change.target]
model_obj = parameters.get(sbml_id, species.get(sbml_id, None))
if model_obj is None:
invalid_changes.append(change.target)
else:
change_target_model_obj_map[change.target] = model_obj
if invalid_changes:
raise ValueError(''.join([
'The following model targets cannot be changed:\n - {}\n\n'.
format('\n - '.join(sorted(invalid_changes)), ),
'Model change targets must have one of the following SBML ids:\n - {}'
.format(
'\n - '.join(
sorted(list(parameters.keys()) + list(species.keys()))), ),
]))
# Load the algorithm specified by `sim.algorithm`
algorithm_substitution_policy = get_algorithm_substitution_policy(
config=config)
exec_kisao_id = get_preferred_substitute_algorithm_by_ids(
sim.algorithm.kisao_id,
KISAO_ALGORITHM_MAP.keys(),
substitution_policy=algorithm_substitution_policy)
algorithm = KISAO_ALGORITHM_MAP[exec_kisao_id]
solver = algorithm.solver
if solver == gillespy2.SSACSolver and (model.get_all_events() or
model.get_all_assignment_rules()):
solver = gillespy2.NumPySSASolver
# Apply the algorithm parameter changes specified by `sim.algorithm.parameter_changes`
algorithm_params = {}
if exec_kisao_id == sim.algorithm.kisao_id:
for change in sim.algorithm.changes:
parameter = algorithm.parameters.get(change.kisao_id, None)
if parameter:
try:
parameter.set_value(algorithm_params, change.new_value)
except (NotImplementedError, ValueError) as exception:
if (ALGORITHM_SUBSTITUTION_POLICY_LEVELS[
algorithm_substitution_policy] <=
ALGORITHM_SUBSTITUTION_POLICY_LEVELS[
AlgorithmSubstitutionPolicy.NONE]):
raise
else:
warn(
'Unsuported value `{}` for algorithm parameter `{}` was ignored:\n {}'
.format(change.new_value, change.kisao_id,
str(exception).replace('\n', '\n ')),
BioSimulatorsWarning)
else:
if (ALGORITHM_SUBSTITUTION_POLICY_LEVELS[
algorithm_substitution_policy] <=
ALGORITHM_SUBSTITUTION_POLICY_LEVELS[
AlgorithmSubstitutionPolicy.NONE]):
msg = "".join([
"Algorithm parameter with KiSAO id '{}' is not supported. "
.format(change.kisao_id),
"Parameter must have one of the following KiSAO ids:\n - {}"
.format('\n - '.join(
'{}: {}'.format(kisao_id, parameter.name)
for kisao_id, parameter in
algorithm.parameters.items())),
])
raise NotImplementedError(msg)
else:
msg = "".join([
"Algorithm parameter with KiSAO id '{}' was ignored because it is not supported. "
.format(change.kisao_id),
"Parameter must have one of the following KiSAO ids:\n - {}"
.format('\n - '.join(
'{}: {}'.format(kisao_id, parameter.name)
for kisao_id, parameter in
algorithm.parameters.items())),
])
warn(msg, BioSimulatorsWarning)
# determine allowed variable targets
predicted_ids = list(species.keys()) + list(parameters.keys())
unpredicted_symbols = set()
unpredicted_targets = set()
for variable in variables:
if variable.symbol:
if variable.symbol != Symbol.time:
unpredicted_symbols.add(variable.symbol)
else:
if variable_target_sbml_id_map[
variable.target] not in predicted_ids:
unpredicted_targets.add(variable.target)
if unpredicted_symbols:
raise NotImplementedError("".join([
"The following variable symbols are not supported:\n - {}\n\n".
format('\n - '.join(sorted(unpredicted_symbols)), ),
"Symbols must be one of the following:\n - {}".format(
Symbol.time),
]))
if unpredicted_targets:
raise ValueError(''.join([
'The following variable targets could not be recorded:\n - {}\n\n'
.format('\n - '.join(sorted(unpredicted_targets)), ),
'Targets must have one of the following SBML ids:\n - {}'.format(
'\n - '.join(sorted(predicted_ids)), ),
]))
# return preprocessed information about the task
return {
'model': {
'model': model,
'change_target_model_obj_map': change_target_model_obj_map,
'variable_target_sbml_id_map': variable_target_sbml_id_map,
},
'simulation': {
'algorithm_kisao_id': exec_kisao_id,
'solver': solver,
'solver_args': dict(**algorithm.solver_args, **algorithm_params),
}
}
def preprocess_sed_task(task, variables, config=None):
""" Preprocess a SED task, including its possible model changes and variables. This is useful for avoiding
repeatedly initializing tasks on repeated calls of :obj:`exec_sed_task`.
Args:
task (:obj:`Task`): task
variables (:obj:`list` of :obj:`Variable`): variables that should be recorded
config (:obj:`Config`, optional): BioSimulators common configuration
Returns:
:obj:`dict`: preprocessed information about the task
"""
config = config or get_config()
if config.VALIDATE_SEDML:
raise_errors_warnings(
validation.validate_task(task),
error_summary='Task `{}` is invalid.'.format(task.id))
raise_errors_warnings(
validation.validate_model_language(task.model.language, ModelLanguage.BNGL),
error_summary='Language for model `{}` is not supported.'.format(task.model.id))
raise_errors_warnings(
validation.validate_model_change_types(task.model.changes, (ModelAttributeChange, )),
error_summary='Changes for model `{}` are not supported.'.format(task.model.id))
raise_errors_warnings(
*validation.validate_model_changes(task.model),
error_summary='Changes for model `{}` are invalid.'.format(task.model.id))
raise_errors_warnings(
validation.validate_simulation_type(task.simulation, (UniformTimeCourseSimulation, )),
error_summary='{} `{}` is not supported.'.format(
task.simulation.__class__.__name__,
task.simulation.id))
raise_errors_warnings(
*validation.validate_simulation(task.simulation),
error_summary='Simulation `{}` is invalid.'.format(task.simulation.id))
raise_errors_warnings(
*validation.validate_data_generator_variables(variables),
error_summary='Data generator variables for task `{}` are invalid.'.format(task.id))
# read the model from the BNGL file
bionetgen_task = read_task(task.model.source)
if bionetgen_task.actions:
warnings.warn('Actions in the BNGL file were ignored.', IgnoredBnglFileContentWarning)
bionetgen_task.actions = []
# validate and apply the model attribute changes to the BioNetGen task
model_changes = {}
for change in task.model.changes:
model_changes[change.target] = preprocess_model_attribute_change(bionetgen_task, change)
# add observables for the variables to the BioNetGen model
add_variables_to_model(bionetgen_task.model, variables)
# apply the SED algorithm and its parameters to the BioNetGen task
simulation_actions, alg_kisao_id = create_actions_for_simulation(task.simulation)
# return the values of the variables and log
return {
'bionetgen_task': bionetgen_task,
'model_changes': model_changes,
'simulation_actions': simulation_actions,
'algorithm_kisao_id': alg_kisao_id,
}
def exec_sed_task(task, variables, preprocessed_task=None, log=None, config=None):
""" Execute a task and save its results
Args:
task (:obj:`Task`): SED task
variables (:obj:`list` of :obj:`Variable`): variables that should be recorded
preprocessed_task (:obj:`dict`, optional): preprocessed information about the task, including possible
model changes and variables. This can be used to avoid repeatedly executing the same initialization
for repeated calls to this method.
log (:obj:`TaskLog`, optional): log for the task
config (:obj:`Config`, optional): BioSimulators common configuration
Returns:
:obj:`tuple`:
:obj:`VariableResults`: results of variables
:obj:`TaskLog`: log
"""
""" Validate task
* Model is encoded in BNGL
* Model changes are instances of :obj:`ModelAttributeChange`
* Simulation is an instance of :obj:`UniformTimeCourseSimulation`
* Time course is valid
* initial time <= output start time <= output end time
* Number of points is an non-negative integer
Note:
* Model attribute changes are validated by
* :obj:`add_model_attribute_change_to_task`
* BioNetGen
* Data generator variables are validated by
* :obj:`add_variables_to_task`
* BioNetGen
* :obj:`get_variables_results_from_observable_results`
"""
config = config or get_config()
if config.LOG and not log:
log = TaskLog()
if preprocessed_task is None:
preprocessed_task = preprocess_sed_task(task, variables, config=config)
# read the model from the BNGL file
bionetgen_task = preprocessed_task['bionetgen_task']
preprocessed_actions = bionetgen_task.actions
bionetgen_task.actions = copy.deepcopy(preprocessed_actions)
# validate and apply the model attribute changes to the BioNetGen task
for change in task.model.changes:
add_model_attribute_change_to_task(bionetgen_task, change, preprocessed_task['model_changes'][change.target])
# apply the SED algorithm and its parameters to the BioNetGen task
alg_kisao_id = preprocessed_task['algorithm_kisao_id']
# execute the task
bionetgen_task.actions.extend(preprocessed_task['simulation_actions'])
observable_results = exec_bionetgen_task(bionetgen_task, verbose=config.VERBOSE)
# get predicted values of the variables
variable_results = get_variables_results_from_observable_results(observable_results, variables)
for key in variable_results.keys():
variable_results[key] = variable_results[key][-(task.simulation.number_of_points + 1):]
# log action
if config.LOG:
log.algorithm = alg_kisao_id
log.simulator_details = {
'actions': bionetgen_task.actions,
}
# clean up
bionetgen_task.actions = preprocessed_actions
# return the values of the variables and log
return variable_results, log
def test_exec_sedml_docs_in_combine_archive_with_all_algorithms(self):
algorithms = gen_algorithms_from_specs(os.path.join(os.path.dirname(__file__), '..', 'biosimulators.json')).values()
for i_alg, alg in enumerate(algorithms):
alg_props = KISAO_ALGORITHM_MAP[alg.kisao_id]
print('Testing algorithm {} of {}: {} ({})'.format(i_alg + 1, len(algorithms), alg_props.name, alg.kisao_id))
alg.changes = []
for param_kisao_id, param_props in alg_props.parameters.items():
if param_kisao_id == 'KISAO_0000488':
new_value = '1'
else:
new_value = param_props.default
if isinstance(new_value, enum.Enum):
new_value = new_value.value
if new_value is None:
new_value = ''
else:
new_value = str(new_value)
alg.changes.append(sedml_data_model.AlgorithmParameterChange(
kisao_id=param_kisao_id,
new_value=new_value,
))
doc, archive_filename = self._build_combine_archive(algorithm=alg)
variables = []
for data_gen in doc.data_generators:
for var in data_gen.variables:
variables.append(var)
doc.tasks[0].model.source = os.path.join(os.path.dirname(__file__),
'fixtures', 'BIOMD0000000297.edited', 'ex1', 'BIOMD0000000297.xml')
results, _ = core.exec_sed_task(doc.tasks[0], variables, log=TaskLog())
self.assertEqual(set(results.keys()), set(var.id for var in variables))
alg.changes = []
for param_kisao_id, param_props in alg_props.parameters.items():
new_value = param_props.default
if isinstance(new_value, enum.Enum):
new_value = new_value.value
if new_value is not None:
new_value = str(new_value)
alg.changes.append(sedml_data_model.AlgorithmParameterChange(
kisao_id=param_kisao_id,
new_value=new_value,
))
doc, archive_filename = self._build_combine_archive(algorithm=alg)
out_dir = os.path.join(self.dirname, alg.kisao_id)
config = get_config()
config.REPORT_FORMATS = [report_data_model.ReportFormat.h5, report_data_model.ReportFormat.csv]
config.BUNDLE_OUTPUTS = True
config.KEEP_INDIVIDUAL_OUTPUTS = True
_, log = core.exec_sedml_docs_in_combine_archive(archive_filename, out_dir, config=config)
if log.exception:
raise log.exception
self._assert_combine_archive_outputs(doc, out_dir)
def test_2(self):
archive = CombineArchive(contents=[
CombineArchiveContent(
location='dir1/dir2/sim.sedml',
format='http://identifiers.org/combine.specifications/sed-ml',
),
CombineArchiveContent(
location='model.xml',
format='http://identifiers.org/combine.specifications/sbml',
),
], )
in_dir = os.path.join(self.tmp_dir, 'archive')
archive_filename = os.path.join(self.tmp_dir, 'archive.omex')
CombineArchiveWriter().run(archive, in_dir, archive_filename)
def sed_task_executer(task, variables):
pass
out_dir = os.path.join(self.tmp_dir, 'outputs')
config = get_config()
config.REPORT_FORMATS = [ReportFormat.csv]
config.VIZ_FORMATS = [VizFormat.pdf]
def exec_sed_doc(task_executer,
filename,
working_dir,
base_out_dir,
rel_path='.',
apply_xml_model_changes=False,
indent=0,
log=None,
log_level=None,
config=config):
out_dir = os.path.join(base_out_dir, rel_path)
if not os.path.isdir(out_dir):
os.makedirs(out_dir)
with open(os.path.join(out_dir, 'report1.csv'), 'w') as file:
file.write('ABC')
with open(os.path.join(out_dir, 'report2.csv'), 'w') as file:
file.write('DEF')
with open(os.path.join(out_dir, 'plot1.pdf'), 'w') as file:
file.write('GHI')
with open(os.path.join(out_dir, 'plot2.pdf'), 'w') as file:
file.write('JKL')
return None, None
with mock.patch('biosimulators_utils.sedml.exec.exec_sed_doc',
side_effect=exec_sed_doc):
with mock.patch.object(SedmlSimulationReader,
'run',
return_value=SedDocument()):
sed_doc_executer = functools.partial(exec_sed_doc,
sed_task_executer)
config = get_config()
config.REPORT_FORMATS = [ReportFormat.h5, ReportFormat.csv]
config.BUNDLE_OUTPUTS = True
config.KEEP_INDIVIDUAL_OUTPUTS = True
exec.exec_sedml_docs_in_archive(sed_doc_executer,
archive_filename,
out_dir,
config=config)
self.assertEqual(
sorted(os.listdir(out_dir)),
sorted(['reports.zip', 'plots.zip', 'dir1', 'log.yml']))
self.assertEqual(os.listdir(os.path.join(out_dir, 'dir1')), ['dir2'])
self.assertEqual(os.listdir(os.path.join(out_dir, 'dir1', 'dir2')),
['sim.sedml'])
self.assertEqual(
sorted(
os.listdir(os.path.join(out_dir, 'dir1', 'dir2',
'sim.sedml'))),
sorted(['report1.csv', 'report2.csv', 'plot1.pdf', 'plot2.pdf']))
archive_dir = os.path.join(self.tmp_dir, 'archive')
archive = ArchiveReader().run(os.path.join(out_dir, 'reports.zip'),
archive_dir)
self.assertEqual(
sorted(file.archive_path for file in archive.files),
sorted([
'dir1/dir2/sim.sedml/report1.csv',
'dir1/dir2/sim.sedml/report2.csv',
]))
with open(
os.path.join(archive_dir, 'dir1', 'dir2', 'sim.sedml',
'report1.csv'), 'r') as file:
self.assertEqual(file.read(), 'ABC')
with open(
os.path.join(archive_dir, 'dir1', 'dir2', 'sim.sedml',
'report2.csv'), 'r') as file:
self.assertEqual(file.read(), 'DEF')
archive = ArchiveReader().run(os.path.join(out_dir, 'plots.zip'),
archive_dir)
self.assertEqual(
sorted(file.archive_path for file in archive.files),
sorted([
'dir1/dir2/sim.sedml/plot1.pdf',
'dir1/dir2/sim.sedml/plot2.pdf',
]))
with open(
os.path.join(archive_dir, 'dir1', 'dir2', 'sim.sedml',
'plot1.pdf'), 'r') as file:
self.assertEqual(file.read(), 'GHI')
with open(
os.path.join(archive_dir, 'dir1', 'dir2', 'sim.sedml',
'plot2.pdf'), 'r') as file:
self.assertEqual(file.read(), 'JKL')
# don't bundle outputs, don't keep individual outputs
out_dir = os.path.join(self.tmp_dir, 'outputs-2')
os.makedirs(out_dir)
os.makedirs(os.path.join(out_dir, 'dir1'))
with open(os.path.join(out_dir, 'extra-file'), 'w'):
pass
with open(os.path.join(out_dir, 'dir1', 'extra-file'), 'w'):
pass
with mock.patch('biosimulators_utils.sedml.exec.exec_sed_doc',
side_effect=exec_sed_doc):
with mock.patch.object(SedmlSimulationReader,
'run',
return_value=SedDocument()):
sed_doc_executer = functools.partial(exec_sed_doc,
sed_task_executer)
config = get_config()
config.REPORT_FORMATS = [ReportFormat.h5, ReportFormat.csv]
config.BUNDLE_OUTPUTS = False
config.KEEP_INDIVIDUAL_OUTPUTS = False
exec.exec_sedml_docs_in_archive(sed_doc_executer,
archive_filename,
out_dir,
config=config)
self.assertEqual(sorted(os.listdir(out_dir)),
sorted(['log.yml', 'extra-file', 'dir1']))
self.assertEqual(sorted(os.listdir(os.path.join(out_dir, 'dir1'))),
sorted(['extra-file']))
out_dir = os.path.join(self.tmp_dir, 'outputs-3')
with mock.patch('biosimulators_utils.sedml.exec.exec_sed_doc',
side_effect=exec_sed_doc):
with mock.patch.object(SedmlSimulationReader,
'run',
return_value=SedDocument()):
sed_doc_executer = functools.partial(exec_sed_doc,
sed_task_executer)
exec.exec_sedml_docs_in_archive(sed_doc_executer,
archive_filename, out_dir)
self.assertIn('log.yml', os.listdir(out_dir))
def preprocess_sed_task(task, variables, config=None):
""" Preprocess a SED task, including its possible model changes and variables. This is useful for avoiding
repeatedly initializing tasks on repeated calls of :obj:`exec_sed_task`.
Args:
task (:obj:`Task`): task
variables (:obj:`list` of :obj:`Variable`): variables that should be recorded
config (:obj:`Config`, optional): BioSimulators common configuration
Returns:
:obj:`dict`: preprocessed information about the task
"""
config = config or get_config()
sed_model = task.model
sed_simulation = task.simulation
if config.VALIDATE_SEDML:
raise_errors_warnings(validation.validate_task(task),
error_summary='Task `{}` is invalid.'.format(
task.id))
raise_errors_warnings(
validation.validate_model_language(sed_model.language,
ModelLanguage.Smoldyn),
error_summary='Language for model `{}` is not supported.'.format(
sed_model.id))
raise_errors_warnings(
validation.validate_model_change_types(sed_model.changes,
(ModelAttributeChange, )),
error_summary='Changes for model `{}` are not supported.'.format(
sed_model.id))
raise_errors_warnings(
*validation.validate_model_changes(sed_model),
error_summary='Changes for model `{}` are invalid.'.format(
sed_model.id))
raise_errors_warnings(validation.validate_simulation_type(
sed_simulation, (UniformTimeCourseSimulation, )),
error_summary='{} `{}` is not supported.'.format(
sed_simulation.__class__.__name__,
sed_simulation.id))
raise_errors_warnings(
*validation.validate_simulation(sed_simulation),
error_summary='Simulation `{}` is invalid.'.format(
sed_simulation.id))
raise_errors_warnings(
*validation.validate_data_generator_variables(variables),
error_summary='Data generator variables for task `{}` are invalid.'
.format(task.id))
if sed_simulation.algorithm.kisao_id not in KISAO_ALGORITHMS_MAP:
msg = 'Algorithm `{}` is not supported. The following algorithms are supported:{}'.format(
sed_simulation.algorithm.kisao_id,
''.join('\n {}: {}'.format(kisao_id, alg_props['name'])
for kisao_id, alg_props in KISAO_ALGORITHMS_MAP.items()))
raise NotImplementedError(msg)
# read Smoldyn configuration
simulation_configuration = read_smoldyn_simulation_configuration(
sed_model.source)
normalize_smoldyn_simulation_configuration(simulation_configuration)
# turn off Smoldyn's graphics
disable_smoldyn_graphics_in_simulation_configuration(
simulation_configuration)
# preprocess model changes
sed_smoldyn_preprocessing_change_map = {}
sed_smoldyn_simulation_change_map = {}
for change in sed_model.changes:
smoldyn_change = validate_model_change(change)
if smoldyn_change.execution == SimulationChangeExecution.preprocessing:
sed_smoldyn_preprocessing_change_map[change] = smoldyn_change
else:
sed_smoldyn_simulation_change_map[change.target] = smoldyn_change
# apply preprocessing-time changes
for change, smoldyn_change in sed_smoldyn_preprocessing_change_map.items():
apply_change_to_smoldyn_simulation_configuration(
simulation_configuration, change, smoldyn_change)
# write the modified Smoldyn configuration to a temporary file
fid, smoldyn_configuration_filename = tempfile.mkstemp(suffix='.txt')
os.close(fid)
write_smoldyn_simulation_configuration(simulation_configuration,
smoldyn_configuration_filename)
# initialize a simulation from the Smoldyn file
smoldyn_simulation = init_smoldyn_simulation_from_configuration_file(
smoldyn_configuration_filename)
# clean up temporary file
os.remove(smoldyn_configuration_filename)
# apply the SED algorithm parameters to the Smoldyn simulation and to the arguments to its ``run`` method
smoldyn_simulation_attrs = {}
smoldyn_simulation_run_alg_param_args = {}
for sed_algorithm_parameter_change in sed_simulation.algorithm.changes:
val = get_smoldyn_instance_attr_or_run_algorithm_parameter_arg(
sed_algorithm_parameter_change)
if val['type'] == AlgorithmParameterType.run_argument:
smoldyn_simulation_run_alg_param_args[val['name']] = val['value']
else:
smoldyn_simulation_attrs[val['name']] = val['value']
# apply the SED algorithm parameters to the Smoldyn simulation and to the arguments to its ``run`` method
for attr_name, value in smoldyn_simulation_attrs.items():
setter = getattr(smoldyn_simulation, attr_name)
setter(value)
# validate SED variables
variable_output_cmd_map = validate_variables(variables)
# Setup Smoldyn output files for the SED variables
smoldyn_configuration_dirname = os.path.dirname(
smoldyn_configuration_filename)
smoldyn_output_files = add_smoldyn_output_files_for_sed_variables(
smoldyn_configuration_dirname, variables, variable_output_cmd_map,
smoldyn_simulation)
# return preprocessed information
return {
'simulation': smoldyn_simulation,
'simulation_attrs': smoldyn_simulation_attrs,
'simulation_run_alg_param_args': smoldyn_simulation_run_alg_param_args,
'sed_smoldyn_simulation_change_map': sed_smoldyn_simulation_change_map,
'variable_output_cmd_map': variable_output_cmd_map,
'output_files': smoldyn_output_files,
}
def test_capturer_not_available(self):
archive = CombineArchive(contents=[
CombineArchiveContent(
location='dir1/dir2/sim.sedml',
format='http://identifiers.org/combine.specifications/sed-ml',
),
], )
in_dir = os.path.join(self.tmp_dir, 'archive')
archive_filename = os.path.join(self.tmp_dir, 'archive.omex')
CombineArchiveWriter().run(archive, in_dir, archive_filename)
def sed_task_executer(task, variables):
pass
out_dir = os.path.join(self.tmp_dir, 'outputs')
config = get_config()
config.REPORT_FORMATS = [ReportFormat.csv]
config.VIZ_FORMATS = []
config.BUNDLE_OUTPUTS = True
config.KEEP_INDIVIDUAL_OUTPUTS = True
def exec_sed_doc(task_executer,
filename,
working_dir,
base_out_dir,
rel_path='.',
apply_xml_model_changes=False,
indent=0,
log=None,
log_level=None,
config=config):
out_dir = os.path.join(base_out_dir, rel_path)
if not os.path.isdir(out_dir):
os.makedirs(out_dir)
with open(os.path.join(out_dir, 'report1.csv'), 'w') as file:
file.write('ABC')
return None, None
builtin_import = builtins.__import__
def import_mock(name, *args):
if name == 'capturer':
raise ModuleNotFoundError
return builtin_import(name, *args)
with mock.patch('builtins.__import__', side_effect=import_mock):
importlib.reload(log_utils)
with mock.patch('biosimulators_utils.sedml.exec.exec_sed_doc',
side_effect=exec_sed_doc):
with mock.patch.object(SedmlSimulationReader,
'run',
return_value=SedDocument()):
sed_doc_executer = functools.partial(
exec_sed_doc, sed_task_executer)
config = get_config()
config.BUNDLE_OUTPUTS = True
config.KEEP_INDIVIDUAL_OUTPUTS = True
_, log = exec.exec_sedml_docs_in_archive(sed_doc_executer,
archive_filename,
out_dir,
config=config)
self.assertNotEqual(log.output, None)
for doc_log in log.sed_documents.values():
self.assertNotEqual(doc_log.output, None)
importlib.reload(log_utils)
def test(self):
task_log = data_model.TaskLog(
id='task_1',
status=data_model.Status.FAILED,
exception=ValueError('Big error'),
skip_reason=NotImplementedError('Skip rationale'),
output='Stdout/err',
duration=10.5,
algorithm='KISAO_0000019',
)
self.assertEqual(
task_log.to_json(), {
'id': 'task_1',
'status': 'FAILED',
'exception': {
'type': 'ValueError',
'message': 'Big error',
},
'skipReason': {
'type': 'NotImplementedError',
'message': 'Skip rationale',
},
'output': 'Stdout/err',
'duration': 10.5,
'algorithm': 'KISAO_0000019',
'simulatorDetails': None,
})
task_log = data_model.TaskLog(id='task_1',
status=data_model.Status.RUNNING)
self.assertEqual(
task_log.to_json(), {
'id': 'task_1',
'status': 'RUNNING',
'exception': None,
'skipReason': None,
'output': None,
'duration': None,
'algorithm': None,
'simulatorDetails': None,
})
report_log = data_model.ReportLog(id='report_1',
status=data_model.Status.RUNNING,
data_sets={
'data_set_1':
data_model.Status.QUEUED,
'data_set_2':
data_model.Status.SUCCEEDED,
})
self.assertEqual(
report_log.to_json(), {
'id':
'report_1',
'status':
'RUNNING',
'exception':
None,
'skipReason':
None,
'output':
None,
'duration':
None,
'dataSets': [
{
'id': 'data_set_1',
'status': 'QUEUED'
},
{
'id': 'data_set_2',
'status': 'SUCCEEDED'
},
]
})
plot2d_log = data_model.Plot2DLog(id='plot_1',
status=data_model.Status.RUNNING,
curves={
'curve_1':
data_model.Status.QUEUED,
'curve_2':
data_model.Status.SUCCEEDED,
})
self.assertEqual(
plot2d_log.to_json(), {
'id':
'plot_1',
'status':
'RUNNING',
'exception':
None,
'skipReason':
None,
'output':
None,
'duration':
None,
'curves': [
{
'id': 'curve_1',
'status': 'QUEUED'
},
{
'id': 'curve_2',
'status': 'SUCCEEDED'
},
]
})
plot3d_log = data_model.Plot3DLog(id='plot_2',
status=data_model.Status.RUNNING,
surfaces={
'surface_1':
data_model.Status.QUEUED,
'surface_2':
data_model.Status.SUCCEEDED,
})
self.assertEqual(
plot3d_log.to_json(), {
'id':
'plot_2',
'status':
'RUNNING',
'exception':
None,
'skipReason':
None,
'output':
None,
'duration':
None,
'surfaces': [
{
'id': 'surface_1',
'status': 'QUEUED'
},
{
'id': 'surface_2',
'status': 'SUCCEEDED'
},
]
})
doc_log = data_model.SedDocumentLog(
location='doc_1',
status=data_model.Status.RUNNING,
tasks={
'task_1': task_log,
},
outputs={
'report_1': report_log,
'plot_1': plot2d_log,
'plot_2': plot3d_log,
},
)
self.assertEqual(
doc_log.to_json(), {
'location':
'doc_1',
'status':
'RUNNING',
'exception':
None,
'skipReason':
None,
'output':
None,
'duration':
None,
'tasks': [
task_log.to_json(),
],
'outputs': [
report_log.to_json(),
plot2d_log.to_json(),
plot3d_log.to_json(),
],
})
archive_log = data_model.CombineArchiveLog(
status=data_model.Status.RUNNING,
sed_documents={
'doc_1': doc_log,
},
)
self.assertEqual(
archive_log.to_json(), {
'status': 'RUNNING',
'exception': None,
'skipReason': None,
'output': None,
'duration': None,
'sedDocuments': [
doc_log.to_json(),
],
})
doc_log.parent = archive_log
task_log.parent = doc_log
report_log.parent = doc_log
plot2d_log.parent = doc_log
plot3d_log.parent = doc_log
archive_log.out_dir = os.path.join(self.dirname, 'log')
archive_log.export()
doc_log.export()
task_log.export()
report_log.export()
plot2d_log.export()
plot3d_log.export()
with open(os.path.join(archive_log.out_dir,
get_config().LOG_PATH), 'r') as file:
self.assertEqual(yaml.load(file, Loader=yaml.FullLoader),
archive_log.to_json())
def _test(simulator_module, example_combine_archive, tmp_dirname):
api = get_simulator_api(simulator_module, False)
# __version__
if not hasattr(api, '__version__'):
raise NotImplementedError(
'API must have a `__version__` attribute whose value is a non-empty string (e.g., 1.0.1)'
)
if not isinstance(api.__version__, str):
raise ValueError(
'API must have a `__version__` attribute whose value is a non-empty string (e.g., 1.0.1), not `{}`'
.format(api.__version__.__class__.__name__))
if api.__version__ == '':
raise ValueError(
'API must have a `__version__` attribute whose value is a non-empty string (e.g., 1.0.1), not `{}`'
.format(api.__version__))
# get_simulator_version
if not hasattr(api, 'get_simulator_version'):
raise NotImplementedError(
'API must have a `get_simulator_version` callable that returns a non-empty string (e.g., 1.0.1)'
)
if not callable(api.get_simulator_version):
raise ValueError(
'`get_simulator_version` must be a callable that returns a non-empty string (e.g., 1.0.1), not `{}`'
.format(api.get_simulator_version.__class__.__name__))
simulator_version = api.get_simulator_version()
if not isinstance(simulator_version, str):
raise ValueError(
'`get_simulator_version` must return a non-empty string (e.g., 1.0.1), not `{}`'
.format(simulator_version.__class__.__name__))
if simulator_version == '':
raise ValueError(
'`get_simulator_version` must return a non-empty string (e.g., 1.0.1), not `{}`'
.format(simulator_version))
# exec_sedml_docs_in_combine_archive
if not hasattr(api, 'exec_sedml_docs_in_combine_archive'):
raise NotImplementedError(
'API must have a `exec_sedml_docs_in_combine_archive` callable'
)
if not callable(api.exec_sedml_docs_in_combine_archive):
raise ValueError(
'`exec_sedml_docs_in_combine_archive` must be a callable, not `{}`'
.format(
api.exec_sedml_docs_in_combine_archive.__class__.__name__))
response = requests.get(EXAMPLES_BASE_URL + '/' +
example_combine_archive)
response.raise_for_status()
archive_filename = os.path.join(tmp_dirname, 'archive.omex')
with open(archive_filename, 'wb') as file:
file.write(response.content)
out_dir = os.path.join(tmp_dirname, 'out')
config = get_config()
config.COLLECT_COMBINE_ARCHIVE_RESULTS = True
config.COLLECT_SED_DOCUMENT_RESULTS = True
config.DEBUG = True
results, log = api.exec_sedml_docs_in_combine_archive(archive_filename,
out_dir,
config=config)
# exec_sed_doc
if not hasattr(api, 'exec_sed_doc'):
raise NotImplementedError(
'API must have a `exec_sed_doc` callable')
if not callable(api.exec_sed_doc):
raise ValueError(
'`exec_sed_doc` must be a callable, not `{}`'.format(
api.exec_sed_doc.__class__.__name__))
# exec_sed_task
if not hasattr(api, 'exec_sed_task'):
raise NotImplementedError(
'API must have a `exec_sed_task` callable')
if not callable(api.exec_sed_task):
raise ValueError(
'`exec_sed_task` must be a callable, not `{}`'.format(
api.exec_sed_task.__class__.__name__))
# preprocess_sed_task
if not hasattr(api, 'preprocess_sed_task'):
raise NotImplementedError(
'API must have a `preprocess_sed_task` callable')
if not callable(api.preprocess_sed_task):
raise ValueError(
'`preprocess_sed_task` must be a callable, not `{}`'.format(
api.preprocess_sed_task.__class__.__name__))
def test_exec_sedml_docs_in_archive_error_handling(self):
def exec_sed_doc(task_executer,
filename,
working_dir,
base_out_dir,
rel_path,
apply_xml_model_changes=False,
indent=0,
log=None,
log_level=None,
config=None):
return None, None
def sed_task_executer(task, variables):
pass
sed_doc_executer = functools.partial(exec_sed_doc, sed_task_executer)
config = get_config()
config.DEBUG = True
# valid archive
archive_filename = os.path.join(
os.path.dirname(__file__), '..', 'fixtures',
'Ciliberto-J-Cell-Biol-2003-morphogenesis-checkpoint.omex')
exec.exec_sedml_docs_in_archive(sed_doc_executer,
archive_filename,
self.tmp_dir,
config=config)
# invalid archive
archive_filename = os.path.join(
os.path.dirname(__file__), '..', 'fixtures', 'sedml-validation',
'invalid-omex-manifest-missing-attribute.omex')
with self.assertRaisesRegex(
ValueError,
re.compile('is not a valid COMBINE/OMEX archive.\n - ',
re.MULTILINE)):
exec.exec_sedml_docs_in_archive(sed_doc_executer,
archive_filename,
self.tmp_dir,
config=config)
with self.assertRaisesRegex(ValueError,
'must have the required attributes'):
exec.exec_sedml_docs_in_archive(sed_doc_executer,
archive_filename,
self.tmp_dir,
config=config)
# invalid SED-ML file in archive
archive_filename = os.path.join(
os.path.dirname(__file__), '..', 'fixtures', 'sedml-validation',
'invalid-sedml-missing-attribute.omex')
with self.assertRaisesRegex(
ValueError,
re.compile('is not a valid COMBINE/OMEX archive.\n - ',
re.MULTILINE)):
exec.exec_sedml_docs_in_archive(sed_doc_executer,
archive_filename,
self.tmp_dir,
config=config)
with self.assertRaisesRegex(ValueError,
'must have the required attributes'):
exec.exec_sedml_docs_in_archive(sed_doc_executer,
archive_filename,
self.tmp_dir,
config=config)
def test_exec_sedml_docs_in_archive_without_log(self):
archive = CombineArchive(contents=[
CombineArchiveContent(
location='sim.sedml',
format='http://identifiers.org/combine.specifications/sed-ml',
),
CombineArchiveContent(
location='model.xml',
format='http://identifiers.org/combine.specifications/sbml',
),
], )
sed_doc = SedDocument()
model = Model(id='model_1',
source='model.xml',
language=ModelLanguage.SBML.value)
sed_doc.models.append(model)
sim = UniformTimeCourseSimulation(
id='sim_1',
initial_time=0.,
output_start_time=0.,
output_end_time=10.,
number_of_points=10,
algorithm=Algorithm(kisao_id='KISAO_0000019'))
sed_doc.simulations.append(sim)
task = Task(id='task_1', model=model, simulation=sim)
sed_doc.tasks.append(task)
sed_doc.data_generators.append(
DataGenerator(
id='data_gen_1',
variables=[
Variable(
id='var_1',
target=
"/sbml:sbml/sbml:model/sbml:listOfSpecies/sbml:species[@id='Trim']",
target_namespaces={
'sbml': 'http://www.sbml.org/sbml/level2/version4'
},
task=task)
],
math='var_1',
))
sed_doc.data_generators.append(
DataGenerator(
id='data_gen_2',
variables=[
Variable(
id='var_2',
target=
"/sbml:sbml/sbml:model/sbml:listOfSpecies/sbml:species[@id='Clb']",
target_namespaces={
'sbml': 'http://www.sbml.org/sbml/level2/version4'
},
task=task)
],
math='var_2',
))
report = Report(id='output_1')
sed_doc.outputs.append(report)
report.data_sets.append(
DataSet(id='data_set_1',
label='data_set_1',
data_generator=sed_doc.data_generators[0]))
report.data_sets.append(
DataSet(id='data_set_2',
label='data_set_2',
data_generator=sed_doc.data_generators[1]))
archive_dirname = os.path.join(self.tmp_dir, 'archive')
os.makedirs(archive_dirname)
shutil.copyfile(
os.path.join(os.path.dirname(__file__), '..', 'fixtures',
'BIOMD0000000297.xml'),
os.path.join(archive_dirname, 'model.xml'))
SedmlSimulationWriter().run(sed_doc,
os.path.join(archive_dirname, 'sim.sedml'))
archive_filename = os.path.join(self.tmp_dir, 'archive.omex')
CombineArchiveWriter().run(archive, archive_dirname, archive_filename)
def sed_task_executer(task, variables, log=None, config=None):
if log:
log.algorithm = task.simulation.algorithm.kisao_id
log.simulator_details = {
'attrib': 'value',
}
return VariableResults({
'var_1':
numpy.linspace(0., 10., task.simulation.number_of_points + 1),
'var_2':
numpy.linspace(10., 20., task.simulation.number_of_points + 1),
}), log
def sed_task_executer_error(task, variables, log=None, config=None):
raise ValueError('Big error')
out_dir = os.path.join(self.tmp_dir, 'outputs')
config = get_config()
config.REPORT_FORMATS = []
config.VIZ_FORMATS = []
config.COLLECT_COMBINE_ARCHIVE_RESULTS = True
config.LOG = True
# with log
sed_doc_executer = functools.partial(sedml_exec.exec_sed_doc,
sed_task_executer)
results, log = exec.exec_sedml_docs_in_archive(
sed_doc_executer,
archive_filename,
out_dir,
apply_xml_model_changes=False,
config=config)
self.assertEqual(set(results.keys()), set(['sim.sedml']))
self.assertEqual(set(results['sim.sedml'].keys()), set(['output_1']))
self.assertEqual(set(results['sim.sedml']['output_1'].keys()),
set(['data_set_1', 'data_set_2']))
numpy.testing.assert_allclose(
results['sim.sedml']['output_1']['data_set_1'],
numpy.linspace(0., 10., 11))
numpy.testing.assert_allclose(
results['sim.sedml']['output_1']['data_set_2'],
numpy.linspace(10., 20., 11))
self.assertEqual(log.exception, None)
self.assertEqual(
log.sed_documents['sim.sedml'].tasks['task_1'].algorithm,
task.simulation.algorithm.kisao_id)
self.assertEqual(
log.sed_documents['sim.sedml'].tasks['task_1'].simulator_details,
{'attrib': 'value'})
sed_doc_executer = functools.partial(sedml_exec.exec_sed_doc,
sed_task_executer_error)
results, log = exec.exec_sedml_docs_in_archive(
sed_doc_executer,
archive_filename,
out_dir,
apply_xml_model_changes=False,
config=config)
self.assertIsInstance(log.exception, CombineArchiveExecutionError)
config.DEBUG = True
sed_doc_executer = functools.partial(sedml_exec.exec_sed_doc,
sed_task_executer_error)
with self.assertRaisesRegex(ValueError, 'Big error'):
exec.exec_sedml_docs_in_archive(sed_doc_executer,
archive_filename,
out_dir,
apply_xml_model_changes=False,
config=config)
# without log
config.COLLECT_COMBINE_ARCHIVE_RESULTS = False
config.LOG = False
config.DEBUG = False
sed_doc_executer = functools.partial(sedml_exec.exec_sed_doc,
sed_task_executer)
results, log = exec.exec_sedml_docs_in_archive(
sed_doc_executer,
archive_filename,
out_dir,
apply_xml_model_changes=False,
config=config)
self.assertEqual(results, None)
self.assertEqual(log, None)
sed_doc_executer = functools.partial(sedml_exec.exec_sed_doc,
sed_task_executer_error)
with self.assertRaisesRegex(CombineArchiveExecutionError, 'Big error'):
exec.exec_sedml_docs_in_archive(sed_doc_executer,
archive_filename,
out_dir,
apply_xml_model_changes=False,
config=config)
config.DEBUG = True
sed_doc_executer = functools.partial(sedml_exec.exec_sed_doc,
sed_task_executer_error)
with self.assertRaisesRegex(ValueError, 'Big error'):
exec.exec_sedml_docs_in_archive(sed_doc_executer,
archive_filename,
out_dir,
apply_xml_model_changes=False,
config=config)
def test_1(self):
archive = CombineArchive(contents=[
CombineArchiveContent(
location='sim.sedml',
format='http://identifiers.org/combine.specifications/sed-ml',
),
CombineArchiveContent(
location='model.xml',
format='http://identifiers.org/combine.specifications/sbml',
),
], )
in_dir = os.path.join(self.tmp_dir, 'archive')
archive_filename = os.path.join(self.tmp_dir, 'archive.omex')
CombineArchiveWriter().run(archive, in_dir, archive_filename)
def sed_task_executer(task, variables):
pass
out_dir = os.path.join(self.tmp_dir, 'outputs')
def exec_sed_doc(task_executer,
filename,
working_dir,
base_out_dir,
rel_path,
apply_xml_model_changes=False,
indent=0,
log=None,
log_level=None,
config=None):
out_dir = os.path.join(base_out_dir, rel_path)
if not os.path.isdir(out_dir):
os.makedirs(out_dir)
with open(os.path.join(out_dir, 'report1.csv'), 'w') as file:
file.write('ABC')
with open(os.path.join(out_dir, 'report2.csv'), 'w') as file:
file.write('DEF')
with open(os.path.join(base_out_dir, 'reports.h5'), 'w') as file:
file.write('DEF')
return ReportResults({
'report1': 'ABC',
'report2': 'DEF',
}), None
with mock.patch('biosimulators_utils.sedml.exec.exec_sed_doc',
side_effect=exec_sed_doc):
sed_doc = SedDocument(
tasks=[Task(id='task_1')],
outputs=[Report(id='output_1')],
)
with mock.patch.object(SedmlSimulationReader,
'run',
return_value=sed_doc):
sed_doc_executer = functools.partial(exec_sed_doc,
sed_task_executer)
config = get_config()
config.REPORT_FORMATS = [ReportFormat.h5, ReportFormat.csv]
config.VIZ_FORMATS = []
config.BUNDLE_OUTPUTS = True
config.KEEP_INDIVIDUAL_OUTPUTS = True
config.COLLECT_COMBINE_ARCHIVE_RESULTS = False
results, _ = exec.exec_sedml_docs_in_archive(sed_doc_executer,
archive_filename,
out_dir,
config=config)
self.assertEqual(results, None)
config.COLLECT_COMBINE_ARCHIVE_RESULTS = True
results, _ = exec.exec_sedml_docs_in_archive(sed_doc_executer,
archive_filename,
out_dir,
config=config)
self.assertEqual(
results,
SedDocumentResults({
'sim.sedml':
ReportResults({
'report1': 'ABC',
'report2': 'DEF',
})
}))
self.assertEqual(
sorted(os.listdir(out_dir)),
sorted(['reports.h5', 'reports.zip', 'sim.sedml', 'log.yml']))
self.assertEqual(
sorted(os.listdir(os.path.join(out_dir, 'sim.sedml'))),
sorted(['report1.csv', 'report2.csv']))
archive.contents[0].format = CombineArchiveContentFormat.TEXT
CombineArchiveWriter().run(archive, in_dir, archive_filename)
with self.assertRaisesRegex(
NoSedmlError, 'does not contain any executing SED-ML files'):
with mock.patch('biosimulators_utils.sedml.exec.exec_sed_doc',
side_effect=exec_sed_doc):
sed_doc_executer = functools.partial(exec_sed_doc,
sed_task_executer)
config = get_config()
config.REPORT_FORMATS = [ReportFormat.h5, ReportFormat.csv]
config.VIZ_FORMATS = []
config.BUNDLE_OUTPUTS = True
config.KEEP_INDIVIDUAL_OUTPUTS = True
config.DEBUG = True
exec.exec_sedml_docs_in_archive(sed_doc_executer,
archive_filename,
out_dir,
config=config)
archive.contents[
0].format = 'http://identifiers.org/combine.specifications/sed-ml'
CombineArchiveWriter().run(archive, in_dir, archive_filename)
out_dir = os.path.join(self.tmp_dir, 'outputs-with-error')
def exec_sed_doc(task_executer,
filename,
working_dir,
base_out_dir,
rel_path,
apply_xml_model_changes=False,
indent=0,
log=None,
log_level=None,
config=None):
out_dir = os.path.join(base_out_dir, rel_path)
if not os.path.isdir(out_dir):
os.makedirs(out_dir)
with open(os.path.join(out_dir, 'report1.csv'), 'w') as file:
file.write('ABC')
with open(os.path.join(out_dir, 'report2.csv'), 'w') as file:
file.write('DEF')
with open(os.path.join(base_out_dir, 'reports.h5'), 'w') as file:
file.write('DEF')
raise ValueError('An error')
sed_doc = SedDocument(
tasks=[Task(id='task_1')],
outputs=[Report(id='output_1')],
)
with mock.patch.object(SedmlSimulationReader,
'run',
return_value=sed_doc):
sed_doc_executer = functools.partial(exec_sed_doc,
sed_task_executer)
with self.assertRaisesRegex(CombineArchiveExecutionError,
'An error'):
config = get_config()
config.REPORT_FORMATS = [ReportFormat.h5, ReportFormat.csv]
config.VIZ_FORMATS = []
config.BUNDLE_OUTPUTS = True
config.KEEP_INDIVIDUAL_OUTPUTS = True
_, log = exec.exec_sedml_docs_in_archive(sed_doc_executer,
archive_filename,
out_dir,
config=config)
if log.exception:
raise log.exception
self.assertEqual(
sorted(os.listdir(out_dir)),
sorted(['reports.h5', 'reports.zip', 'sim.sedml', 'log.yml']))
self.assertEqual(
sorted(os.listdir(os.path.join(out_dir, 'sim.sedml'))),
sorted(['report1.csv', 'report2.csv']))
