def test_get_config(self): with mock.patch.dict(os.environ, { 'REPORT_FORMATS': '', 'VIZ_FORMATS': '' }): config = get_config() self.assertEqual(config.REPORT_FORMATS, []) self.assertEqual(config.VIZ_FORMATS, []) with mock.patch.dict(os.environ, { 'REPORT_FORMATS': 'h5', 'VIZ_FORMATS': 'pdf' }): config = get_config() self.assertEqual(config.REPORT_FORMATS, [ReportFormat.h5]) self.assertEqual(config.VIZ_FORMATS, [VizFormat.pdf])
def test_exec_sedml_docs_in_combine_archive(self): doc, archive_filename = self._build_combine_archive() out_dir = os.path.join(self.dirname, 'out') config = get_config() config.REPORT_FORMATS = [ReportFormat.h5] _, log = smoldyn.biosimulators.combine.exec_sedml_docs_in_combine_archive( archive_filename, out_dir, config=config) if log.exception: raise log.exception self._assert_combine_archive_outputs(doc, out_dir)
def test_exec_sedml_docs_in_combine_archive(self): doc, archive_filename = self._build_combine_archive() out_dir = os.path.join(self.dirname, 'out') config = get_config() config.REPORT_FORMATS = [report_data_model.ReportFormat.h5, report_data_model.ReportFormat.csv] config.BUNDLE_OUTPUTS = True config.KEEP_INDIVIDUAL_OUTPUTS = True _, log = core.exec_sedml_docs_in_combine_archive(archive_filename, out_dir, config=config) if log.exception: raise log.exception self._assert_combine_archive_outputs(doc, out_dir)
def test_more_complex_archive(self): archive_filename = os.path.join(os.path.dirname(__file__), 'fixtures', 'BIOMD0000000297.edited.omex') config = get_config() config.REPORT_FORMATS = [report_data_model.ReportFormat.h5, report_data_model.ReportFormat.csv] config.VIZ_FORMATS = [] config.BUNDLE_OUTPUTS = True config.KEEP_INDIVIDUAL_OUTPUTS = True _, log = core.exec_sedml_docs_in_combine_archive(archive_filename, self.dirname, config=config) if log.exception: raise log.exception self.assertEqual(set(['reports.zip', 'reports.h5', 'ex1', 'ex2']).difference(set(os.listdir(self.dirname))), set()) self.assertEqual(set(os.listdir(os.path.join(self.dirname, 'ex1'))), set(['BIOMD0000000297.sedml'])) self.assertEqual(set(os.listdir(os.path.join(self.dirname, 'ex2'))), set(['BIOMD0000000297.sedml'])) self.assertEqual(set(os.listdir(os.path.join(self.dirname, 'ex1', 'BIOMD0000000297.sedml'))), set(['two_species.csv', 'three_species.csv'])) self.assertEqual(set(os.listdir(os.path.join(self.dirname, 'ex2', 'BIOMD0000000297.sedml'))), set(['one_species.csv', 'four_species.csv'])) archive = ArchiveReader().run(os.path.join(self.dirname, 'reports.zip')) self.assertEqual( sorted(file.archive_path for file in archive.files), sorted([ 'ex1/BIOMD0000000297.sedml/two_species.csv', 'ex1/BIOMD0000000297.sedml/three_species.csv', 'ex2/BIOMD0000000297.sedml/one_species.csv', 'ex2/BIOMD0000000297.sedml/four_species.csv', ]), ) report = sedml_data_model.Report( data_sets=[ sedml_data_model.DataSet(id='data_set_time_two_species', label='time'), sedml_data_model.DataSet(id='data_set_Cln4', label='Cln4'), sedml_data_model.DataSet(id='data_set_Swe13', label='Swe13'), ] ) report_results = ReportReader().run(report, self.dirname, 'ex1/BIOMD0000000297.sedml/two_species', format=report_data_model.ReportFormat.h5) self.assertEqual(sorted(report_results.keys()), sorted(['data_set_time_two_species', 'data_set_Cln4', 'data_set_Swe13'])) numpy.testing.assert_allclose(report_results['data_set_time_two_species'], numpy.linspace(0., 1., 10 + 1))
def test_exec_sedml_docs_in_combine_archive_with_all_algorithms(self): for alg in gen_algorithms_from_specs( os.path.join(os.path.dirname(__file__), '..', 'biosimulators.json')).values(): doc, archive_filename = self._build_combine_archive(algorithm=alg) out_dir = os.path.join(self.dirname, alg.kisao_id) config = get_config() config.REPORT_FORMATS = [ report_data_model.ReportFormat.h5, report_data_model.ReportFormat.csv ] config.BUNDLE_OUTPUTS = True config.KEEP_INDIVIDUAL_OUTPUTS = True _, log = exec_sedml_docs_in_combine_archive(archive_filename, out_dir, config=config) if log.exception: raise log.exception self._assert_combine_archive_outputs(doc, out_dir)
def handler(body, archiveFile=None): """ Execute the SED-ML files in a COMBINE/OMEX archive. Args: body (:obj:`dict`): dictionary with schema ``SimulationRun`` with the specifications of the COMBINE/OMEX archive to execute and the simulator to execute it with archiveFile (:obj:`werkzeug.datastructures.FileStorage`, optional): COMBINE/OMEX file Returns: :obj:`werkzeug.wrappers.response.Response`: response with the results and log of the run in the ``SimulationRunResults`` schema """ archive_file = archiveFile archive_url = body.get('archiveUrl', None) simulator_id = body['simulator'] env_vars = body.get('environment', {}).get('variables', []) # set up environment (i.e. options) env = {} for env_var in env_vars: key = env_var['key'] if key not in IGNORED_ENV_VARS: env[key] = env_var['value'] if 'REPORT_FORMATS' not in env: env['REPORT_FORMATS'] = 'h5' with mock.patch.dict('os.environ', env): config = get_config() # process requested return type accept = connexion.request.headers.get('Accept', 'application/json') if accept in ['application/json']: config.COLLECT_COMBINE_ARCHIVE_RESULTS = True config.COLLECT_SED_DOCUMENT_RESULTS = True config.REPORT_FORMATS = [] config.VIZ_FORMATS = [] config.BUNDLE_OUTPUTS = False config.KEEP_INDIVIDUAL_OUTPUTS = True config.LOG_PATH = '' return_type = 'json' elif accept in ['application/x-hdf', 'application/x-hdf5']: config.COLLECT_COMBINE_ARCHIVE_RESULTS = False config.COLLECT_SED_DOCUMENT_RESULTS = False config.REPORT_FORMATS = [ ReportFormat[format.strip().lower()] for format in env.get('REPORT_FORMATS', 'h5').split(',') ] config.VIZ_FORMATS = [] config.BUNDLE_OUTPUTS = False config.KEEP_INDIVIDUAL_OUTPUTS = True config.LOG_PATH = '' return_type = 'h5' elif accept in ['application/zip']: config.COLLECT_COMBINE_ARCHIVE_RESULTS = False config.COLLECT_SED_DOCUMENT_RESULTS = False config.REPORT_FORMATS = [ ReportFormat[format.strip().lower()] for format in env.get('REPORT_FORMATS', 'h5').split(',') ] config.VIZ_FORMATS = [ VizFormat[format.strip().lower()] for format in env.get('VIZ_FORMATS', 'pdf').split(',') ] config.BUNDLE_OUTPUTS = False config.KEEP_INDIVIDUAL_OUTPUTS = True return_type = 'zip' else: raise BadRequestException( title= '`Accept` header must be one of `application/hdf5`, `application/json`, or `application/zip`.', instance=NotImplementedError(), ) # get the COMBINE/OMEX archive if archive_file and archive_url: raise BadRequestException( title= 'Only one of `archiveFile` or `archiveUrl` can be used at a time.', instance=ValueError(), ) # get COMBINE/OMEX archive archive_filename = get_temp_file(suffix='.omex') if archive_file: archive_file.save(archive_filename) else: try: response = requests.get(archive_url) response.raise_for_status() except requests.exceptions.RequestException as exception: title = 'COMBINE/OMEX archive could not be loaded from `{}`'.format( archive_url) raise BadRequestException( title=title, instance=exception, ) # save archive to local temporary file with open(archive_filename, 'wb') as file: file.write(response.content) # get the simulator simulator = next( (simulator for simulator in get_simulators() if simulator['id'] == simulator_id), None) if simulator is None: raise BadRequestException( title= '`{}` is not a BioSimulators id of a simulation tool that is available for execution.' .format(simulator_id), instance=ValueError(), ) # execute the simulation out_dir = get_temp_dir() with mock.patch.dict('os.environ', env): results, log = exec_in_subprocess( use_simulator_api_to_exec_sedml_docs_in_combine_archive, simulator['api']['module'], archive_filename, out_dir, timeout=TIMEOUT, config=config) # transform the results if return_type == 'json': archive_dirname = get_temp_dir() with zipfile.ZipFile(archive_filename, 'r') as zip_file: zip_file.extractall(archive_dirname) outputs = [] for sed_doc_location, sed_doc_outputs_results in (results or {}).items(): sed_doc = SedmlSimulationReader().run( os.path.join(archive_dirname, sed_doc_location)) for output in sed_doc.outputs: if output.id not in sed_doc_outputs_results: continue output_results = sed_doc_outputs_results.get(output.id, None) if isinstance(output, Report): type = 'SedReport' report = output elif isinstance(output, Plot2D): type = 'SedPlot2D' report = get_report_for_plot2d(output) elif isinstance(output, Plot3D): type = 'SedPlot3D' report = get_report_for_plot3d(output) else: # pragma: no cover # raise NotImplementedError( 'Outputs of type `{}` are not supported.'.format( output.__class__.__name__)) data = [] for data_set in report.data_sets: if data_set.id not in output_results: continue data_set_results = output_results[data_set.id] data.append({ '_type': 'SimulationRunOutputDatum', 'id': data_set.id, 'label': data_set.label, 'name': data_set.name, 'shape': '' if data_set_results is None else ','.join( str(dim_len) for dim_len in data_set_results.shape), 'type': '__None__' if data_set_results is None else data_set_results.dtype.name, 'values': None if data_set_results is None else data_set_results.tolist(), }) outputs.append({ '_type': 'SimulationRunOutput', 'outputId': sed_doc_location + '/' + output.id, 'name': output.name, 'type': type, 'data': data, }) # return return { '_type': 'SimulationRunResults', 'outputs': outputs, 'log': log, } elif return_type == 'h5': h5_filename = os.path.join(out_dir, get_config().H5_REPORTS_PATH) return flask.send_file(h5_filename, mimetype=accept, as_attachment=True, attachment_filename='outputs.h5') else: zip_filename = get_temp_file() with zipfile.ZipFile(zip_filename, 'w') as zip_file: for root, dirs, files in os.walk(out_dir): for file in files: zip_file.write( os.path.join(root, file), os.path.relpath(os.path.join(root, file), out_dir)) return flask.send_file(zip_filename, mimetype=accept, as_attachment=True, attachment_filename='outputs.zip')
def exec_sed_task(task, variables, preprocessed_task=None, log=None, config=None): ''' Execute a task and save its results Args: task (:obj:`Task`): task variables (:obj:`list` of :obj:`Variable`): variables that should be recorded preprocessed_task (:obj:`dict`, optional): preprocessed information about the task, including possible model changes and variables. This can be used to avoid repeatedly executing the same initialization for repeated calls to this method. log (:obj:`TaskLog`, optional): log for the task config (:obj:`Config`, optional): BioSimulators common configuration Returns: :obj:`tuple`: :obj:`VariableResults`: results of variables :obj:`TaskLog`: log ''' if not config: config = get_config() if config.LOG and not log: log = TaskLog() sed_model_changes = task.model.changes sed_simulation = task.simulation if preprocessed_task is None: preprocessed_task = preprocess_sed_task(task, variables, config=config) sed_model_changes = list( filter( lambda change: change.target in preprocessed_task[ 'sed_smoldyn_simulation_change_map'], sed_model_changes)) # read Smoldyn configuration smoldyn_simulation = preprocessed_task['simulation'] # apply model changes to the Smoldyn configuration sed_smoldyn_simulation_change_map = preprocessed_task[ 'sed_smoldyn_simulation_change_map'] for change in sed_model_changes: smoldyn_change = sed_smoldyn_simulation_change_map.get( change.target, None) if smoldyn_change is None or smoldyn_change.execution != SimulationChangeExecution.simulation: raise NotImplementedError( 'Target `{}` can only be changed during simulation preprocessing.' .format(change.target)) apply_change_to_smoldyn_simulation(smoldyn_simulation, change, smoldyn_change) # get the Smoldyn representation of the SED uniform time course simulation smoldyn_simulation_run_timecourse_args = get_smoldyn_run_timecourse_args( sed_simulation) # execute the simulation smoldyn_run_args = dict( **smoldyn_simulation_run_timecourse_args, **preprocessed_task['simulation_run_alg_param_args'], ) smoldyn_simulation.run(**smoldyn_run_args, overwrite=True, display=False, quit_at_end=False) # get the result of each SED variable variable_output_cmd_map = preprocessed_task['variable_output_cmd_map'] smoldyn_output_files = preprocessed_task['output_files'] variable_results = get_variable_results(sed_simulation.number_of_steps, variables, variable_output_cmd_map, smoldyn_output_files) # cleanup output files for smoldyn_output_file in smoldyn_output_files.values(): os.remove(smoldyn_output_file.filename) # log simulation if config.LOG: log.algorithm = sed_simulation.algorithm.kisao_id log.simulator_details = { 'class': 'smoldyn.Simulation', 'instanceAttributes': preprocessed_task['simulation_attrs'], 'method': 'run', 'methodArguments': smoldyn_run_args, } # return the values of the variables and log return variable_results, log
def exec_sed_task(task, variables, preprocessed_task=None, log=None, config=None): ''' Execute a task and save its results Args: task (:obj:`Task`): task variables (:obj:`list` of :obj:`Variable`): variables that should be recorded preprocessed_task (:obj:`dict`, optional): preprocessed information about the task, including possible model changes and variables. This can be used to avoid repeatedly executing the same initialization for repeated calls to this method. log (:obj:`TaskLog`, optional): log for the task config (:obj:`Config`, optional): BioSimulators common configuration Returns: :obj:`tuple`: :obj:`VariableResults`: results of variables :obj:`TaskLog`: log Raises: :obj:`ValueError`: if the task or an aspect of the task is not valid, or the requested output variables could not be recorded :obj:`NotImplementedError`: if the task is not of a supported type or involves an unsuported feature ''' config = config or get_config() if config.LOG and not log: log = TaskLog() if preprocessed_task is None: preprocessed_task = preprocess_sed_task(task, variables, config=config) sim = task.simulation # get model model = preprocessed_task['model']['model'] # modify model raise_errors_warnings( validation.validate_model_change_types(task.model.changes, (ModelAttributeChange, )), error_summary='Changes for model `{}` are not supported.'.format( task.model.id)) change_target_model_obj_map = preprocessed_task['model'][ 'change_target_model_obj_map'] for change in task.model.changes: model_obj = change_target_model_obj_map[change.target] new_value = float(change.new_value) if isinstance(model_obj, gillespy2.core.parameter.Parameter): model_obj.value = new_value else: model_obj.initial_value = new_value # Validate that start time is 0 because this is the only option that GillesPy2 supports if sim.initial_time < 0: raise NotImplementedError( 'Negative initial simulation time {} is not supported. Initial time must be >= 0.' .format(sim.initial_time)) # set the simulation time span number_of_points = (sim.output_end_time - sim.initial_time) / \ (sim.output_end_time - sim.output_start_time) * sim.number_of_points if number_of_points != math.floor(number_of_points): raise NotImplementedError( 'Time course must specify an integer number of time points') number_of_points = int(number_of_points) timespan = numpy.linspace(sim.initial_time, sim.output_end_time, number_of_points + 1) model.timespan(timespan) # Simulate the model from ``sim.start_time`` to ``sim.output_end_time`` # and record ``sim.number_of_points`` + 1 time points solver = preprocessed_task['simulation']['solver'] solver_args = preprocessed_task['simulation']['solver_args'] results_dict = model.run(solver, **solver_args)[0] # transform the results to an instance of :obj:`VariableResults` variable_target_sbml_id_map = preprocessed_task['model'][ 'variable_target_sbml_id_map'] variable_results = VariableResults() parameters = model.get_all_parameters() for variable in variables: if variable.symbol: variable_results[variable.id] = results_dict['time'][-( sim.number_of_points + 1):] elif variable.target: sbml_id = variable_target_sbml_id_map[variable.target] dynamics = results_dict.get(sbml_id, None) if dynamics is None: variable_results[variable.id] = numpy.full( (sim.number_of_points + 1, ), parameters[sbml_id].value) else: variable_results[ variable.id] = dynamics[-(sim.number_of_points + 1):] # log action if config.LOG: log.algorithm = preprocessed_task['simulation']['algorithm_kisao_id'] log.simulator_details = { 'method': solver.__module__ + '.' + solver.__name__, 'arguments': solver_args, } # return results and log return variable_results, log
def preprocess_sed_task(task, variables, config=None): """ Preprocess a SED task, including its possible model changes and variables. This is useful for avoiding repeatedly initializing tasks on repeated calls of :obj:`exec_sed_task`. Args: task (:obj:`Task`): task variables (:obj:`list` of :obj:`Variable`): variables that should be recorded config (:obj:`Config`, optional): BioSimulators common configuration Returns: :obj:`dict`: preprocessed information about the task """ config = config or get_config() sim = task.simulation if config.VALIDATE_SEDML: raise_errors_warnings(validation.validate_task(task), error_summary='Task `{}` is invalid.'.format( task.id)) raise_errors_warnings( validation.validate_model_language(task.model.language, ModelLanguage.SBML), error_summary='Language for model `{}` is not supported.'.format( task.model.id)) raise_errors_warnings( validation.validate_model_change_types(task.model.changes, (ModelAttributeChange, )), error_summary='Changes for model `{}` are not supported.'.format( task.model.id)) raise_errors_warnings( *validation.validate_model_changes(task.model), error_summary='Changes for model `{}` are invalid.'.format( task.model.id)) raise_errors_warnings(validation.validate_simulation_type( sim, (UniformTimeCourseSimulation, )), error_summary='{} `{}` is not supported.'.format( sim.__class__.__name__, sim.id)) raise_errors_warnings( *validation.validate_simulation(sim), error_summary='Simulation `{}` is invalid.'.format(sim.id)) raise_errors_warnings( *validation.validate_data_generator_variables(variables), error_summary='Data generator variables for task `{}` are invalid.' .format(task.id)) model_etree = lxml.etree.parse(task.model.source) change_target_sbml_id_map = validation.validate_target_xpaths( task.model.changes, model_etree, attr='id') variable_target_sbml_id_map = validation.validate_target_xpaths( variables, model_etree, attr='id') # Read the SBML-encoded model located at `task.model.source` model, errors = gillespy2.import_SBML(task.model.source) if model is None or errors: raise ValueError('Model at {} could not be imported:\n - {}'.format( task.model.source, '\n - '.join(message for message, code in errors))) # preprocess model changes parameters = model.get_all_parameters() species = model.get_all_species() change_target_model_obj_map = {} invalid_changes = [] for change in task.model.changes: sbml_id = change_target_sbml_id_map[change.target] model_obj = parameters.get(sbml_id, species.get(sbml_id, None)) if model_obj is None: invalid_changes.append(change.target) else: change_target_model_obj_map[change.target] = model_obj if invalid_changes: raise ValueError(''.join([ 'The following model targets cannot be changed:\n - {}\n\n'. format('\n - '.join(sorted(invalid_changes)), ), 'Model change targets must have one of the following SBML ids:\n - {}' .format( '\n - '.join( sorted(list(parameters.keys()) + list(species.keys()))), ), ])) # Load the algorithm specified by `sim.algorithm` algorithm_substitution_policy = get_algorithm_substitution_policy( config=config) exec_kisao_id = get_preferred_substitute_algorithm_by_ids( sim.algorithm.kisao_id, KISAO_ALGORITHM_MAP.keys(), substitution_policy=algorithm_substitution_policy) algorithm = KISAO_ALGORITHM_MAP[exec_kisao_id] solver = algorithm.solver if solver == gillespy2.SSACSolver and (model.get_all_events() or model.get_all_assignment_rules()): solver = gillespy2.NumPySSASolver # Apply the algorithm parameter changes specified by `sim.algorithm.parameter_changes` algorithm_params = {} if exec_kisao_id == sim.algorithm.kisao_id: for change in sim.algorithm.changes: parameter = algorithm.parameters.get(change.kisao_id, None) if parameter: try: parameter.set_value(algorithm_params, change.new_value) except (NotImplementedError, ValueError) as exception: if (ALGORITHM_SUBSTITUTION_POLICY_LEVELS[ algorithm_substitution_policy] <= ALGORITHM_SUBSTITUTION_POLICY_LEVELS[ AlgorithmSubstitutionPolicy.NONE]): raise else: warn( 'Unsuported value `{}` for algorithm parameter `{}` was ignored:\n {}' .format(change.new_value, change.kisao_id, str(exception).replace('\n', '\n ')), BioSimulatorsWarning) else: if (ALGORITHM_SUBSTITUTION_POLICY_LEVELS[ algorithm_substitution_policy] <= ALGORITHM_SUBSTITUTION_POLICY_LEVELS[ AlgorithmSubstitutionPolicy.NONE]): msg = "".join([ "Algorithm parameter with KiSAO id '{}' is not supported. " .format(change.kisao_id), "Parameter must have one of the following KiSAO ids:\n - {}" .format('\n - '.join( '{}: {}'.format(kisao_id, parameter.name) for kisao_id, parameter in algorithm.parameters.items())), ]) raise NotImplementedError(msg) else: msg = "".join([ "Algorithm parameter with KiSAO id '{}' was ignored because it is not supported. " .format(change.kisao_id), "Parameter must have one of the following KiSAO ids:\n - {}" .format('\n - '.join( '{}: {}'.format(kisao_id, parameter.name) for kisao_id, parameter in algorithm.parameters.items())), ]) warn(msg, BioSimulatorsWarning) # determine allowed variable targets predicted_ids = list(species.keys()) + list(parameters.keys()) unpredicted_symbols = set() unpredicted_targets = set() for variable in variables: if variable.symbol: if variable.symbol != Symbol.time: unpredicted_symbols.add(variable.symbol) else: if variable_target_sbml_id_map[ variable.target] not in predicted_ids: unpredicted_targets.add(variable.target) if unpredicted_symbols: raise NotImplementedError("".join([ "The following variable symbols are not supported:\n - {}\n\n". format('\n - '.join(sorted(unpredicted_symbols)), ), "Symbols must be one of the following:\n - {}".format( Symbol.time), ])) if unpredicted_targets: raise ValueError(''.join([ 'The following variable targets could not be recorded:\n - {}\n\n' .format('\n - '.join(sorted(unpredicted_targets)), ), 'Targets must have one of the following SBML ids:\n - {}'.format( '\n - '.join(sorted(predicted_ids)), ), ])) # return preprocessed information about the task return { 'model': { 'model': model, 'change_target_model_obj_map': change_target_model_obj_map, 'variable_target_sbml_id_map': variable_target_sbml_id_map, }, 'simulation': { 'algorithm_kisao_id': exec_kisao_id, 'solver': solver, 'solver_args': dict(**algorithm.solver_args, **algorithm_params), } }
def preprocess_sed_task(task, variables, config=None): """ Preprocess a SED task, including its possible model changes and variables. This is useful for avoiding repeatedly initializing tasks on repeated calls of :obj:`exec_sed_task`. Args: task (:obj:`Task`): task variables (:obj:`list` of :obj:`Variable`): variables that should be recorded config (:obj:`Config`, optional): BioSimulators common configuration Returns: :obj:`dict`: preprocessed information about the task """ config = config or get_config() if config.VALIDATE_SEDML: raise_errors_warnings( validation.validate_task(task), error_summary='Task `{}` is invalid.'.format(task.id)) raise_errors_warnings( validation.validate_model_language(task.model.language, ModelLanguage.BNGL), error_summary='Language for model `{}` is not supported.'.format(task.model.id)) raise_errors_warnings( validation.validate_model_change_types(task.model.changes, (ModelAttributeChange, )), error_summary='Changes for model `{}` are not supported.'.format(task.model.id)) raise_errors_warnings( *validation.validate_model_changes(task.model), error_summary='Changes for model `{}` are invalid.'.format(task.model.id)) raise_errors_warnings( validation.validate_simulation_type(task.simulation, (UniformTimeCourseSimulation, )), error_summary='{} `{}` is not supported.'.format( task.simulation.__class__.__name__, task.simulation.id)) raise_errors_warnings( *validation.validate_simulation(task.simulation), error_summary='Simulation `{}` is invalid.'.format(task.simulation.id)) raise_errors_warnings( *validation.validate_data_generator_variables(variables), error_summary='Data generator variables for task `{}` are invalid.'.format(task.id)) # read the model from the BNGL file bionetgen_task = read_task(task.model.source) if bionetgen_task.actions: warnings.warn('Actions in the BNGL file were ignored.', IgnoredBnglFileContentWarning) bionetgen_task.actions = [] # validate and apply the model attribute changes to the BioNetGen task model_changes = {} for change in task.model.changes: model_changes[change.target] = preprocess_model_attribute_change(bionetgen_task, change) # add observables for the variables to the BioNetGen model add_variables_to_model(bionetgen_task.model, variables) # apply the SED algorithm and its parameters to the BioNetGen task simulation_actions, alg_kisao_id = create_actions_for_simulation(task.simulation) # return the values of the variables and log return { 'bionetgen_task': bionetgen_task, 'model_changes': model_changes, 'simulation_actions': simulation_actions, 'algorithm_kisao_id': alg_kisao_id, }
def exec_sed_task(task, variables, preprocessed_task=None, log=None, config=None): """ Execute a task and save its results Args: task (:obj:`Task`): SED task variables (:obj:`list` of :obj:`Variable`): variables that should be recorded preprocessed_task (:obj:`dict`, optional): preprocessed information about the task, including possible model changes and variables. This can be used to avoid repeatedly executing the same initialization for repeated calls to this method. log (:obj:`TaskLog`, optional): log for the task config (:obj:`Config`, optional): BioSimulators common configuration Returns: :obj:`tuple`: :obj:`VariableResults`: results of variables :obj:`TaskLog`: log """ """ Validate task * Model is encoded in BNGL * Model changes are instances of :obj:`ModelAttributeChange` * Simulation is an instance of :obj:`UniformTimeCourseSimulation` * Time course is valid * initial time <= output start time <= output end time * Number of points is an non-negative integer Note: * Model attribute changes are validated by * :obj:`add_model_attribute_change_to_task` * BioNetGen * Data generator variables are validated by * :obj:`add_variables_to_task` * BioNetGen * :obj:`get_variables_results_from_observable_results` """ config = config or get_config() if config.LOG and not log: log = TaskLog() if preprocessed_task is None: preprocessed_task = preprocess_sed_task(task, variables, config=config) # read the model from the BNGL file bionetgen_task = preprocessed_task['bionetgen_task'] preprocessed_actions = bionetgen_task.actions bionetgen_task.actions = copy.deepcopy(preprocessed_actions) # validate and apply the model attribute changes to the BioNetGen task for change in task.model.changes: add_model_attribute_change_to_task(bionetgen_task, change, preprocessed_task['model_changes'][change.target]) # apply the SED algorithm and its parameters to the BioNetGen task alg_kisao_id = preprocessed_task['algorithm_kisao_id'] # execute the task bionetgen_task.actions.extend(preprocessed_task['simulation_actions']) observable_results = exec_bionetgen_task(bionetgen_task, verbose=config.VERBOSE) # get predicted values of the variables variable_results = get_variables_results_from_observable_results(observable_results, variables) for key in variable_results.keys(): variable_results[key] = variable_results[key][-(task.simulation.number_of_points + 1):] # log action if config.LOG: log.algorithm = alg_kisao_id log.simulator_details = { 'actions': bionetgen_task.actions, } # clean up bionetgen_task.actions = preprocessed_actions # return the values of the variables and log return variable_results, log
def test_exec_sedml_docs_in_combine_archive_with_all_algorithms(self): algorithms = gen_algorithms_from_specs(os.path.join(os.path.dirname(__file__), '..', 'biosimulators.json')).values() for i_alg, alg in enumerate(algorithms): alg_props = KISAO_ALGORITHM_MAP[alg.kisao_id] print('Testing algorithm {} of {}: {} ({})'.format(i_alg + 1, len(algorithms), alg_props.name, alg.kisao_id)) alg.changes = [] for param_kisao_id, param_props in alg_props.parameters.items(): if param_kisao_id == 'KISAO_0000488': new_value = '1' else: new_value = param_props.default if isinstance(new_value, enum.Enum): new_value = new_value.value if new_value is None: new_value = '' else: new_value = str(new_value) alg.changes.append(sedml_data_model.AlgorithmParameterChange( kisao_id=param_kisao_id, new_value=new_value, )) doc, archive_filename = self._build_combine_archive(algorithm=alg) variables = [] for data_gen in doc.data_generators: for var in data_gen.variables: variables.append(var) doc.tasks[0].model.source = os.path.join(os.path.dirname(__file__), 'fixtures', 'BIOMD0000000297.edited', 'ex1', 'BIOMD0000000297.xml') results, _ = core.exec_sed_task(doc.tasks[0], variables, log=TaskLog()) self.assertEqual(set(results.keys()), set(var.id for var in variables)) alg.changes = [] for param_kisao_id, param_props in alg_props.parameters.items(): new_value = param_props.default if isinstance(new_value, enum.Enum): new_value = new_value.value if new_value is not None: new_value = str(new_value) alg.changes.append(sedml_data_model.AlgorithmParameterChange( kisao_id=param_kisao_id, new_value=new_value, )) doc, archive_filename = self._build_combine_archive(algorithm=alg) out_dir = os.path.join(self.dirname, alg.kisao_id) config = get_config() config.REPORT_FORMATS = [report_data_model.ReportFormat.h5, report_data_model.ReportFormat.csv] config.BUNDLE_OUTPUTS = True config.KEEP_INDIVIDUAL_OUTPUTS = True _, log = core.exec_sedml_docs_in_combine_archive(archive_filename, out_dir, config=config) if log.exception: raise log.exception self._assert_combine_archive_outputs(doc, out_dir)
def test_2(self): archive = CombineArchive(contents=[ CombineArchiveContent( location='dir1/dir2/sim.sedml', format='http://identifiers.org/combine.specifications/sed-ml', ), CombineArchiveContent( location='model.xml', format='http://identifiers.org/combine.specifications/sbml', ), ], ) in_dir = os.path.join(self.tmp_dir, 'archive') archive_filename = os.path.join(self.tmp_dir, 'archive.omex') CombineArchiveWriter().run(archive, in_dir, archive_filename) def sed_task_executer(task, variables): pass out_dir = os.path.join(self.tmp_dir, 'outputs') config = get_config() config.REPORT_FORMATS = [ReportFormat.csv] config.VIZ_FORMATS = [VizFormat.pdf] def exec_sed_doc(task_executer, filename, working_dir, base_out_dir, rel_path='.', apply_xml_model_changes=False, indent=0, log=None, log_level=None, config=config): out_dir = os.path.join(base_out_dir, rel_path) if not os.path.isdir(out_dir): os.makedirs(out_dir) with open(os.path.join(out_dir, 'report1.csv'), 'w') as file: file.write('ABC') with open(os.path.join(out_dir, 'report2.csv'), 'w') as file: file.write('DEF') with open(os.path.join(out_dir, 'plot1.pdf'), 'w') as file: file.write('GHI') with open(os.path.join(out_dir, 'plot2.pdf'), 'w') as file: file.write('JKL') return None, None with mock.patch('biosimulators_utils.sedml.exec.exec_sed_doc', side_effect=exec_sed_doc): with mock.patch.object(SedmlSimulationReader, 'run', return_value=SedDocument()): sed_doc_executer = functools.partial(exec_sed_doc, sed_task_executer) config = get_config() config.REPORT_FORMATS = [ReportFormat.h5, ReportFormat.csv] config.BUNDLE_OUTPUTS = True config.KEEP_INDIVIDUAL_OUTPUTS = True exec.exec_sedml_docs_in_archive(sed_doc_executer, archive_filename, out_dir, config=config) self.assertEqual( sorted(os.listdir(out_dir)), sorted(['reports.zip', 'plots.zip', 'dir1', 'log.yml'])) self.assertEqual(os.listdir(os.path.join(out_dir, 'dir1')), ['dir2']) self.assertEqual(os.listdir(os.path.join(out_dir, 'dir1', 'dir2')), ['sim.sedml']) self.assertEqual( sorted( os.listdir(os.path.join(out_dir, 'dir1', 'dir2', 'sim.sedml'))), sorted(['report1.csv', 'report2.csv', 'plot1.pdf', 'plot2.pdf'])) archive_dir = os.path.join(self.tmp_dir, 'archive') archive = ArchiveReader().run(os.path.join(out_dir, 'reports.zip'), archive_dir) self.assertEqual( sorted(file.archive_path for file in archive.files), sorted([ 'dir1/dir2/sim.sedml/report1.csv', 'dir1/dir2/sim.sedml/report2.csv', ])) with open( os.path.join(archive_dir, 'dir1', 'dir2', 'sim.sedml', 'report1.csv'), 'r') as file: self.assertEqual(file.read(), 'ABC') with open( os.path.join(archive_dir, 'dir1', 'dir2', 'sim.sedml', 'report2.csv'), 'r') as file: self.assertEqual(file.read(), 'DEF') archive = ArchiveReader().run(os.path.join(out_dir, 'plots.zip'), archive_dir) self.assertEqual( sorted(file.archive_path for file in archive.files), sorted([ 'dir1/dir2/sim.sedml/plot1.pdf', 'dir1/dir2/sim.sedml/plot2.pdf', ])) with open( os.path.join(archive_dir, 'dir1', 'dir2', 'sim.sedml', 'plot1.pdf'), 'r') as file: self.assertEqual(file.read(), 'GHI') with open( os.path.join(archive_dir, 'dir1', 'dir2', 'sim.sedml', 'plot2.pdf'), 'r') as file: self.assertEqual(file.read(), 'JKL') # don't bundle outputs, don't keep individual outputs out_dir = os.path.join(self.tmp_dir, 'outputs-2') os.makedirs(out_dir) os.makedirs(os.path.join(out_dir, 'dir1')) with open(os.path.join(out_dir, 'extra-file'), 'w'): pass with open(os.path.join(out_dir, 'dir1', 'extra-file'), 'w'): pass with mock.patch('biosimulators_utils.sedml.exec.exec_sed_doc', side_effect=exec_sed_doc): with mock.patch.object(SedmlSimulationReader, 'run', return_value=SedDocument()): sed_doc_executer = functools.partial(exec_sed_doc, sed_task_executer) config = get_config() config.REPORT_FORMATS = [ReportFormat.h5, ReportFormat.csv] config.BUNDLE_OUTPUTS = False config.KEEP_INDIVIDUAL_OUTPUTS = False exec.exec_sedml_docs_in_archive(sed_doc_executer, archive_filename, out_dir, config=config) self.assertEqual(sorted(os.listdir(out_dir)), sorted(['log.yml', 'extra-file', 'dir1'])) self.assertEqual(sorted(os.listdir(os.path.join(out_dir, 'dir1'))), sorted(['extra-file'])) out_dir = os.path.join(self.tmp_dir, 'outputs-3') with mock.patch('biosimulators_utils.sedml.exec.exec_sed_doc', side_effect=exec_sed_doc): with mock.patch.object(SedmlSimulationReader, 'run', return_value=SedDocument()): sed_doc_executer = functools.partial(exec_sed_doc, sed_task_executer) exec.exec_sedml_docs_in_archive(sed_doc_executer, archive_filename, out_dir) self.assertIn('log.yml', os.listdir(out_dir))
def preprocess_sed_task(task, variables, config=None): """ Preprocess a SED task, including its possible model changes and variables. This is useful for avoiding repeatedly initializing tasks on repeated calls of :obj:`exec_sed_task`. Args: task (:obj:`Task`): task variables (:obj:`list` of :obj:`Variable`): variables that should be recorded config (:obj:`Config`, optional): BioSimulators common configuration Returns: :obj:`dict`: preprocessed information about the task """ config = config or get_config() sed_model = task.model sed_simulation = task.simulation if config.VALIDATE_SEDML: raise_errors_warnings(validation.validate_task(task), error_summary='Task `{}` is invalid.'.format( task.id)) raise_errors_warnings( validation.validate_model_language(sed_model.language, ModelLanguage.Smoldyn), error_summary='Language for model `{}` is not supported.'.format( sed_model.id)) raise_errors_warnings( validation.validate_model_change_types(sed_model.changes, (ModelAttributeChange, )), error_summary='Changes for model `{}` are not supported.'.format( sed_model.id)) raise_errors_warnings( *validation.validate_model_changes(sed_model), error_summary='Changes for model `{}` are invalid.'.format( sed_model.id)) raise_errors_warnings(validation.validate_simulation_type( sed_simulation, (UniformTimeCourseSimulation, )), error_summary='{} `{}` is not supported.'.format( sed_simulation.__class__.__name__, sed_simulation.id)) raise_errors_warnings( *validation.validate_simulation(sed_simulation), error_summary='Simulation `{}` is invalid.'.format( sed_simulation.id)) raise_errors_warnings( *validation.validate_data_generator_variables(variables), error_summary='Data generator variables for task `{}` are invalid.' .format(task.id)) if sed_simulation.algorithm.kisao_id not in KISAO_ALGORITHMS_MAP: msg = 'Algorithm `{}` is not supported. The following algorithms are supported:{}'.format( sed_simulation.algorithm.kisao_id, ''.join('\n {}: {}'.format(kisao_id, alg_props['name']) for kisao_id, alg_props in KISAO_ALGORITHMS_MAP.items())) raise NotImplementedError(msg) # read Smoldyn configuration simulation_configuration = read_smoldyn_simulation_configuration( sed_model.source) normalize_smoldyn_simulation_configuration(simulation_configuration) # turn off Smoldyn's graphics disable_smoldyn_graphics_in_simulation_configuration( simulation_configuration) # preprocess model changes sed_smoldyn_preprocessing_change_map = {} sed_smoldyn_simulation_change_map = {} for change in sed_model.changes: smoldyn_change = validate_model_change(change) if smoldyn_change.execution == SimulationChangeExecution.preprocessing: sed_smoldyn_preprocessing_change_map[change] = smoldyn_change else: sed_smoldyn_simulation_change_map[change.target] = smoldyn_change # apply preprocessing-time changes for change, smoldyn_change in sed_smoldyn_preprocessing_change_map.items(): apply_change_to_smoldyn_simulation_configuration( simulation_configuration, change, smoldyn_change) # write the modified Smoldyn configuration to a temporary file fid, smoldyn_configuration_filename = tempfile.mkstemp(suffix='.txt') os.close(fid) write_smoldyn_simulation_configuration(simulation_configuration, smoldyn_configuration_filename) # initialize a simulation from the Smoldyn file smoldyn_simulation = init_smoldyn_simulation_from_configuration_file( smoldyn_configuration_filename) # clean up temporary file os.remove(smoldyn_configuration_filename) # apply the SED algorithm parameters to the Smoldyn simulation and to the arguments to its ``run`` method smoldyn_simulation_attrs = {} smoldyn_simulation_run_alg_param_args = {} for sed_algorithm_parameter_change in sed_simulation.algorithm.changes: val = get_smoldyn_instance_attr_or_run_algorithm_parameter_arg( sed_algorithm_parameter_change) if val['type'] == AlgorithmParameterType.run_argument: smoldyn_simulation_run_alg_param_args[val['name']] = val['value'] else: smoldyn_simulation_attrs[val['name']] = val['value'] # apply the SED algorithm parameters to the Smoldyn simulation and to the arguments to its ``run`` method for attr_name, value in smoldyn_simulation_attrs.items(): setter = getattr(smoldyn_simulation, attr_name) setter(value) # validate SED variables variable_output_cmd_map = validate_variables(variables) # Setup Smoldyn output files for the SED variables smoldyn_configuration_dirname = os.path.dirname( smoldyn_configuration_filename) smoldyn_output_files = add_smoldyn_output_files_for_sed_variables( smoldyn_configuration_dirname, variables, variable_output_cmd_map, smoldyn_simulation) # return preprocessed information return { 'simulation': smoldyn_simulation, 'simulation_attrs': smoldyn_simulation_attrs, 'simulation_run_alg_param_args': smoldyn_simulation_run_alg_param_args, 'sed_smoldyn_simulation_change_map': sed_smoldyn_simulation_change_map, 'variable_output_cmd_map': variable_output_cmd_map, 'output_files': smoldyn_output_files, }
def test_capturer_not_available(self): archive = CombineArchive(contents=[ CombineArchiveContent( location='dir1/dir2/sim.sedml', format='http://identifiers.org/combine.specifications/sed-ml', ), ], ) in_dir = os.path.join(self.tmp_dir, 'archive') archive_filename = os.path.join(self.tmp_dir, 'archive.omex') CombineArchiveWriter().run(archive, in_dir, archive_filename) def sed_task_executer(task, variables): pass out_dir = os.path.join(self.tmp_dir, 'outputs') config = get_config() config.REPORT_FORMATS = [ReportFormat.csv] config.VIZ_FORMATS = [] config.BUNDLE_OUTPUTS = True config.KEEP_INDIVIDUAL_OUTPUTS = True def exec_sed_doc(task_executer, filename, working_dir, base_out_dir, rel_path='.', apply_xml_model_changes=False, indent=0, log=None, log_level=None, config=config): out_dir = os.path.join(base_out_dir, rel_path) if not os.path.isdir(out_dir): os.makedirs(out_dir) with open(os.path.join(out_dir, 'report1.csv'), 'w') as file: file.write('ABC') return None, None builtin_import = builtins.__import__ def import_mock(name, *args): if name == 'capturer': raise ModuleNotFoundError return builtin_import(name, *args) with mock.patch('builtins.__import__', side_effect=import_mock): importlib.reload(log_utils) with mock.patch('biosimulators_utils.sedml.exec.exec_sed_doc', side_effect=exec_sed_doc): with mock.patch.object(SedmlSimulationReader, 'run', return_value=SedDocument()): sed_doc_executer = functools.partial( exec_sed_doc, sed_task_executer) config = get_config() config.BUNDLE_OUTPUTS = True config.KEEP_INDIVIDUAL_OUTPUTS = True _, log = exec.exec_sedml_docs_in_archive(sed_doc_executer, archive_filename, out_dir, config=config) self.assertNotEqual(log.output, None) for doc_log in log.sed_documents.values(): self.assertNotEqual(doc_log.output, None) importlib.reload(log_utils)
def test(self): task_log = data_model.TaskLog( id='task_1', status=data_model.Status.FAILED, exception=ValueError('Big error'), skip_reason=NotImplementedError('Skip rationale'), output='Stdout/err', duration=10.5, algorithm='KISAO_0000019', ) self.assertEqual( task_log.to_json(), { 'id': 'task_1', 'status': 'FAILED', 'exception': { 'type': 'ValueError', 'message': 'Big error', }, 'skipReason': { 'type': 'NotImplementedError', 'message': 'Skip rationale', }, 'output': 'Stdout/err', 'duration': 10.5, 'algorithm': 'KISAO_0000019', 'simulatorDetails': None, }) task_log = data_model.TaskLog(id='task_1', status=data_model.Status.RUNNING) self.assertEqual( task_log.to_json(), { 'id': 'task_1', 'status': 'RUNNING', 'exception': None, 'skipReason': None, 'output': None, 'duration': None, 'algorithm': None, 'simulatorDetails': None, }) report_log = data_model.ReportLog(id='report_1', status=data_model.Status.RUNNING, data_sets={ 'data_set_1': data_model.Status.QUEUED, 'data_set_2': data_model.Status.SUCCEEDED, }) self.assertEqual( report_log.to_json(), { 'id': 'report_1', 'status': 'RUNNING', 'exception': None, 'skipReason': None, 'output': None, 'duration': None, 'dataSets': [ { 'id': 'data_set_1', 'status': 'QUEUED' }, { 'id': 'data_set_2', 'status': 'SUCCEEDED' }, ] }) plot2d_log = data_model.Plot2DLog(id='plot_1', status=data_model.Status.RUNNING, curves={ 'curve_1': data_model.Status.QUEUED, 'curve_2': data_model.Status.SUCCEEDED, }) self.assertEqual( plot2d_log.to_json(), { 'id': 'plot_1', 'status': 'RUNNING', 'exception': None, 'skipReason': None, 'output': None, 'duration': None, 'curves': [ { 'id': 'curve_1', 'status': 'QUEUED' }, { 'id': 'curve_2', 'status': 'SUCCEEDED' }, ] }) plot3d_log = data_model.Plot3DLog(id='plot_2', status=data_model.Status.RUNNING, surfaces={ 'surface_1': data_model.Status.QUEUED, 'surface_2': data_model.Status.SUCCEEDED, }) self.assertEqual( plot3d_log.to_json(), { 'id': 'plot_2', 'status': 'RUNNING', 'exception': None, 'skipReason': None, 'output': None, 'duration': None, 'surfaces': [ { 'id': 'surface_1', 'status': 'QUEUED' }, { 'id': 'surface_2', 'status': 'SUCCEEDED' }, ] }) doc_log = data_model.SedDocumentLog( location='doc_1', status=data_model.Status.RUNNING, tasks={ 'task_1': task_log, }, outputs={ 'report_1': report_log, 'plot_1': plot2d_log, 'plot_2': plot3d_log, }, ) self.assertEqual( doc_log.to_json(), { 'location': 'doc_1', 'status': 'RUNNING', 'exception': None, 'skipReason': None, 'output': None, 'duration': None, 'tasks': [ task_log.to_json(), ], 'outputs': [ report_log.to_json(), plot2d_log.to_json(), plot3d_log.to_json(), ], }) archive_log = data_model.CombineArchiveLog( status=data_model.Status.RUNNING, sed_documents={ 'doc_1': doc_log, }, ) self.assertEqual( archive_log.to_json(), { 'status': 'RUNNING', 'exception': None, 'skipReason': None, 'output': None, 'duration': None, 'sedDocuments': [ doc_log.to_json(), ], }) doc_log.parent = archive_log task_log.parent = doc_log report_log.parent = doc_log plot2d_log.parent = doc_log plot3d_log.parent = doc_log archive_log.out_dir = os.path.join(self.dirname, 'log') archive_log.export() doc_log.export() task_log.export() report_log.export() plot2d_log.export() plot3d_log.export() with open(os.path.join(archive_log.out_dir, get_config().LOG_PATH), 'r') as file: self.assertEqual(yaml.load(file, Loader=yaml.FullLoader), archive_log.to_json())
def _test(simulator_module, example_combine_archive, tmp_dirname): api = get_simulator_api(simulator_module, False) # __version__ if not hasattr(api, '__version__'): raise NotImplementedError( 'API must have a `__version__` attribute whose value is a non-empty string (e.g., 1.0.1)' ) if not isinstance(api.__version__, str): raise ValueError( 'API must have a `__version__` attribute whose value is a non-empty string (e.g., 1.0.1), not `{}`' .format(api.__version__.__class__.__name__)) if api.__version__ == '': raise ValueError( 'API must have a `__version__` attribute whose value is a non-empty string (e.g., 1.0.1), not `{}`' .format(api.__version__)) # get_simulator_version if not hasattr(api, 'get_simulator_version'): raise NotImplementedError( 'API must have a `get_simulator_version` callable that returns a non-empty string (e.g., 1.0.1)' ) if not callable(api.get_simulator_version): raise ValueError( '`get_simulator_version` must be a callable that returns a non-empty string (e.g., 1.0.1), not `{}`' .format(api.get_simulator_version.__class__.__name__)) simulator_version = api.get_simulator_version() if not isinstance(simulator_version, str): raise ValueError( '`get_simulator_version` must return a non-empty string (e.g., 1.0.1), not `{}`' .format(simulator_version.__class__.__name__)) if simulator_version == '': raise ValueError( '`get_simulator_version` must return a non-empty string (e.g., 1.0.1), not `{}`' .format(simulator_version)) # exec_sedml_docs_in_combine_archive if not hasattr(api, 'exec_sedml_docs_in_combine_archive'): raise NotImplementedError( 'API must have a `exec_sedml_docs_in_combine_archive` callable' ) if not callable(api.exec_sedml_docs_in_combine_archive): raise ValueError( '`exec_sedml_docs_in_combine_archive` must be a callable, not `{}`' .format( api.exec_sedml_docs_in_combine_archive.__class__.__name__)) response = requests.get(EXAMPLES_BASE_URL + '/' + example_combine_archive) response.raise_for_status() archive_filename = os.path.join(tmp_dirname, 'archive.omex') with open(archive_filename, 'wb') as file: file.write(response.content) out_dir = os.path.join(tmp_dirname, 'out') config = get_config() config.COLLECT_COMBINE_ARCHIVE_RESULTS = True config.COLLECT_SED_DOCUMENT_RESULTS = True config.DEBUG = True results, log = api.exec_sedml_docs_in_combine_archive(archive_filename, out_dir, config=config) # exec_sed_doc if not hasattr(api, 'exec_sed_doc'): raise NotImplementedError( 'API must have a `exec_sed_doc` callable') if not callable(api.exec_sed_doc): raise ValueError( '`exec_sed_doc` must be a callable, not `{}`'.format( api.exec_sed_doc.__class__.__name__)) # exec_sed_task if not hasattr(api, 'exec_sed_task'): raise NotImplementedError( 'API must have a `exec_sed_task` callable') if not callable(api.exec_sed_task): raise ValueError( '`exec_sed_task` must be a callable, not `{}`'.format( api.exec_sed_task.__class__.__name__)) # preprocess_sed_task if not hasattr(api, 'preprocess_sed_task'): raise NotImplementedError( 'API must have a `preprocess_sed_task` callable') if not callable(api.preprocess_sed_task): raise ValueError( '`preprocess_sed_task` must be a callable, not `{}`'.format( api.preprocess_sed_task.__class__.__name__))
def test_exec_sedml_docs_in_archive_error_handling(self): def exec_sed_doc(task_executer, filename, working_dir, base_out_dir, rel_path, apply_xml_model_changes=False, indent=0, log=None, log_level=None, config=None): return None, None def sed_task_executer(task, variables): pass sed_doc_executer = functools.partial(exec_sed_doc, sed_task_executer) config = get_config() config.DEBUG = True # valid archive archive_filename = os.path.join( os.path.dirname(__file__), '..', 'fixtures', 'Ciliberto-J-Cell-Biol-2003-morphogenesis-checkpoint.omex') exec.exec_sedml_docs_in_archive(sed_doc_executer, archive_filename, self.tmp_dir, config=config) # invalid archive archive_filename = os.path.join( os.path.dirname(__file__), '..', 'fixtures', 'sedml-validation', 'invalid-omex-manifest-missing-attribute.omex') with self.assertRaisesRegex( ValueError, re.compile('is not a valid COMBINE/OMEX archive.\n - ', re.MULTILINE)): exec.exec_sedml_docs_in_archive(sed_doc_executer, archive_filename, self.tmp_dir, config=config) with self.assertRaisesRegex(ValueError, 'must have the required attributes'): exec.exec_sedml_docs_in_archive(sed_doc_executer, archive_filename, self.tmp_dir, config=config) # invalid SED-ML file in archive archive_filename = os.path.join( os.path.dirname(__file__), '..', 'fixtures', 'sedml-validation', 'invalid-sedml-missing-attribute.omex') with self.assertRaisesRegex( ValueError, re.compile('is not a valid COMBINE/OMEX archive.\n - ', re.MULTILINE)): exec.exec_sedml_docs_in_archive(sed_doc_executer, archive_filename, self.tmp_dir, config=config) with self.assertRaisesRegex(ValueError, 'must have the required attributes'): exec.exec_sedml_docs_in_archive(sed_doc_executer, archive_filename, self.tmp_dir, config=config)
def test_exec_sedml_docs_in_archive_without_log(self): archive = CombineArchive(contents=[ CombineArchiveContent( location='sim.sedml', format='http://identifiers.org/combine.specifications/sed-ml', ), CombineArchiveContent( location='model.xml', format='http://identifiers.org/combine.specifications/sbml', ), ], ) sed_doc = SedDocument() model = Model(id='model_1', source='model.xml', language=ModelLanguage.SBML.value) sed_doc.models.append(model) sim = UniformTimeCourseSimulation( id='sim_1', initial_time=0., output_start_time=0., output_end_time=10., number_of_points=10, algorithm=Algorithm(kisao_id='KISAO_0000019')) sed_doc.simulations.append(sim) task = Task(id='task_1', model=model, simulation=sim) sed_doc.tasks.append(task) sed_doc.data_generators.append( DataGenerator( id='data_gen_1', variables=[ Variable( id='var_1', target= "/sbml:sbml/sbml:model/sbml:listOfSpecies/sbml:species[@id='Trim']", target_namespaces={ 'sbml': 'http://www.sbml.org/sbml/level2/version4' }, task=task) ], math='var_1', )) sed_doc.data_generators.append( DataGenerator( id='data_gen_2', variables=[ Variable( id='var_2', target= "/sbml:sbml/sbml:model/sbml:listOfSpecies/sbml:species[@id='Clb']", target_namespaces={ 'sbml': 'http://www.sbml.org/sbml/level2/version4' }, task=task) ], math='var_2', )) report = Report(id='output_1') sed_doc.outputs.append(report) report.data_sets.append( DataSet(id='data_set_1', label='data_set_1', data_generator=sed_doc.data_generators[0])) report.data_sets.append( DataSet(id='data_set_2', label='data_set_2', data_generator=sed_doc.data_generators[1])) archive_dirname = os.path.join(self.tmp_dir, 'archive') os.makedirs(archive_dirname) shutil.copyfile( os.path.join(os.path.dirname(__file__), '..', 'fixtures', 'BIOMD0000000297.xml'), os.path.join(archive_dirname, 'model.xml')) SedmlSimulationWriter().run(sed_doc, os.path.join(archive_dirname, 'sim.sedml')) archive_filename = os.path.join(self.tmp_dir, 'archive.omex') CombineArchiveWriter().run(archive, archive_dirname, archive_filename) def sed_task_executer(task, variables, log=None, config=None): if log: log.algorithm = task.simulation.algorithm.kisao_id log.simulator_details = { 'attrib': 'value', } return VariableResults({ 'var_1': numpy.linspace(0., 10., task.simulation.number_of_points + 1), 'var_2': numpy.linspace(10., 20., task.simulation.number_of_points + 1), }), log def sed_task_executer_error(task, variables, log=None, config=None): raise ValueError('Big error') out_dir = os.path.join(self.tmp_dir, 'outputs') config = get_config() config.REPORT_FORMATS = [] config.VIZ_FORMATS = [] config.COLLECT_COMBINE_ARCHIVE_RESULTS = True config.LOG = True # with log sed_doc_executer = functools.partial(sedml_exec.exec_sed_doc, sed_task_executer) results, log = exec.exec_sedml_docs_in_archive( sed_doc_executer, archive_filename, out_dir, apply_xml_model_changes=False, config=config) self.assertEqual(set(results.keys()), set(['sim.sedml'])) self.assertEqual(set(results['sim.sedml'].keys()), set(['output_1'])) self.assertEqual(set(results['sim.sedml']['output_1'].keys()), set(['data_set_1', 'data_set_2'])) numpy.testing.assert_allclose( results['sim.sedml']['output_1']['data_set_1'], numpy.linspace(0., 10., 11)) numpy.testing.assert_allclose( results['sim.sedml']['output_1']['data_set_2'], numpy.linspace(10., 20., 11)) self.assertEqual(log.exception, None) self.assertEqual( log.sed_documents['sim.sedml'].tasks['task_1'].algorithm, task.simulation.algorithm.kisao_id) self.assertEqual( log.sed_documents['sim.sedml'].tasks['task_1'].simulator_details, {'attrib': 'value'}) sed_doc_executer = functools.partial(sedml_exec.exec_sed_doc, sed_task_executer_error) results, log = exec.exec_sedml_docs_in_archive( sed_doc_executer, archive_filename, out_dir, apply_xml_model_changes=False, config=config) self.assertIsInstance(log.exception, CombineArchiveExecutionError) config.DEBUG = True sed_doc_executer = functools.partial(sedml_exec.exec_sed_doc, sed_task_executer_error) with self.assertRaisesRegex(ValueError, 'Big error'): exec.exec_sedml_docs_in_archive(sed_doc_executer, archive_filename, out_dir, apply_xml_model_changes=False, config=config) # without log config.COLLECT_COMBINE_ARCHIVE_RESULTS = False config.LOG = False config.DEBUG = False sed_doc_executer = functools.partial(sedml_exec.exec_sed_doc, sed_task_executer) results, log = exec.exec_sedml_docs_in_archive( sed_doc_executer, archive_filename, out_dir, apply_xml_model_changes=False, config=config) self.assertEqual(results, None) self.assertEqual(log, None) sed_doc_executer = functools.partial(sedml_exec.exec_sed_doc, sed_task_executer_error) with self.assertRaisesRegex(CombineArchiveExecutionError, 'Big error'): exec.exec_sedml_docs_in_archive(sed_doc_executer, archive_filename, out_dir, apply_xml_model_changes=False, config=config) config.DEBUG = True sed_doc_executer = functools.partial(sedml_exec.exec_sed_doc, sed_task_executer_error) with self.assertRaisesRegex(ValueError, 'Big error'): exec.exec_sedml_docs_in_archive(sed_doc_executer, archive_filename, out_dir, apply_xml_model_changes=False, config=config)
def test_1(self): archive = CombineArchive(contents=[ CombineArchiveContent( location='sim.sedml', format='http://identifiers.org/combine.specifications/sed-ml', ), CombineArchiveContent( location='model.xml', format='http://identifiers.org/combine.specifications/sbml', ), ], ) in_dir = os.path.join(self.tmp_dir, 'archive') archive_filename = os.path.join(self.tmp_dir, 'archive.omex') CombineArchiveWriter().run(archive, in_dir, archive_filename) def sed_task_executer(task, variables): pass out_dir = os.path.join(self.tmp_dir, 'outputs') def exec_sed_doc(task_executer, filename, working_dir, base_out_dir, rel_path, apply_xml_model_changes=False, indent=0, log=None, log_level=None, config=None): out_dir = os.path.join(base_out_dir, rel_path) if not os.path.isdir(out_dir): os.makedirs(out_dir) with open(os.path.join(out_dir, 'report1.csv'), 'w') as file: file.write('ABC') with open(os.path.join(out_dir, 'report2.csv'), 'w') as file: file.write('DEF') with open(os.path.join(base_out_dir, 'reports.h5'), 'w') as file: file.write('DEF') return ReportResults({ 'report1': 'ABC', 'report2': 'DEF', }), None with mock.patch('biosimulators_utils.sedml.exec.exec_sed_doc', side_effect=exec_sed_doc): sed_doc = SedDocument( tasks=[Task(id='task_1')], outputs=[Report(id='output_1')], ) with mock.patch.object(SedmlSimulationReader, 'run', return_value=sed_doc): sed_doc_executer = functools.partial(exec_sed_doc, sed_task_executer) config = get_config() config.REPORT_FORMATS = [ReportFormat.h5, ReportFormat.csv] config.VIZ_FORMATS = [] config.BUNDLE_OUTPUTS = True config.KEEP_INDIVIDUAL_OUTPUTS = True config.COLLECT_COMBINE_ARCHIVE_RESULTS = False results, _ = exec.exec_sedml_docs_in_archive(sed_doc_executer, archive_filename, out_dir, config=config) self.assertEqual(results, None) config.COLLECT_COMBINE_ARCHIVE_RESULTS = True results, _ = exec.exec_sedml_docs_in_archive(sed_doc_executer, archive_filename, out_dir, config=config) self.assertEqual( results, SedDocumentResults({ 'sim.sedml': ReportResults({ 'report1': 'ABC', 'report2': 'DEF', }) })) self.assertEqual( sorted(os.listdir(out_dir)), sorted(['reports.h5', 'reports.zip', 'sim.sedml', 'log.yml'])) self.assertEqual( sorted(os.listdir(os.path.join(out_dir, 'sim.sedml'))), sorted(['report1.csv', 'report2.csv'])) archive.contents[0].format = CombineArchiveContentFormat.TEXT CombineArchiveWriter().run(archive, in_dir, archive_filename) with self.assertRaisesRegex( NoSedmlError, 'does not contain any executing SED-ML files'): with mock.patch('biosimulators_utils.sedml.exec.exec_sed_doc', side_effect=exec_sed_doc): sed_doc_executer = functools.partial(exec_sed_doc, sed_task_executer) config = get_config() config.REPORT_FORMATS = [ReportFormat.h5, ReportFormat.csv] config.VIZ_FORMATS = [] config.BUNDLE_OUTPUTS = True config.KEEP_INDIVIDUAL_OUTPUTS = True config.DEBUG = True exec.exec_sedml_docs_in_archive(sed_doc_executer, archive_filename, out_dir, config=config) archive.contents[ 0].format = 'http://identifiers.org/combine.specifications/sed-ml' CombineArchiveWriter().run(archive, in_dir, archive_filename) out_dir = os.path.join(self.tmp_dir, 'outputs-with-error') def exec_sed_doc(task_executer, filename, working_dir, base_out_dir, rel_path, apply_xml_model_changes=False, indent=0, log=None, log_level=None, config=None): out_dir = os.path.join(base_out_dir, rel_path) if not os.path.isdir(out_dir): os.makedirs(out_dir) with open(os.path.join(out_dir, 'report1.csv'), 'w') as file: file.write('ABC') with open(os.path.join(out_dir, 'report2.csv'), 'w') as file: file.write('DEF') with open(os.path.join(base_out_dir, 'reports.h5'), 'w') as file: file.write('DEF') raise ValueError('An error') sed_doc = SedDocument( tasks=[Task(id='task_1')], outputs=[Report(id='output_1')], ) with mock.patch.object(SedmlSimulationReader, 'run', return_value=sed_doc): sed_doc_executer = functools.partial(exec_sed_doc, sed_task_executer) with self.assertRaisesRegex(CombineArchiveExecutionError, 'An error'): config = get_config() config.REPORT_FORMATS = [ReportFormat.h5, ReportFormat.csv] config.VIZ_FORMATS = [] config.BUNDLE_OUTPUTS = True config.KEEP_INDIVIDUAL_OUTPUTS = True _, log = exec.exec_sedml_docs_in_archive(sed_doc_executer, archive_filename, out_dir, config=config) if log.exception: raise log.exception self.assertEqual( sorted(os.listdir(out_dir)), sorted(['reports.h5', 'reports.zip', 'sim.sedml', 'log.yml'])) self.assertEqual( sorted(os.listdir(os.path.join(out_dir, 'sim.sedml'))), sorted(['report1.csv', 'report2.csv']))