def validate_archive(self, filename): reader = CombineArchiveReader() name = os.path.relpath(filename, EXAMPLES_DIR) temp_dirname = os.path.join(self.temp_dirname, name) if not os.path.isdir(temp_dirname): os.makedirs(temp_dirname) archive = reader.run(filename, temp_dirname) config = Config( OMEX_METADATA_SCHEMA=OmexMetadataSchema.biosimulations, ) error_msgs, warning_msgs = validate( archive, temp_dirname, formats_to_validate=list( CombineArchiveContentFormat.__members__.values()), config=config, ) if warning_msgs: msg = 'The COMBINE/OMEX archive may be invalid.\n {}'.format( flatten_nested_list_of_strings(warning_msgs).replace( '\n', '\n ')) warnings.warn(msg, BioSimulatorsWarning) if error_msgs: msg = 'The COMBINE/OMEX archive is not valid.\n {}'.format( flatten_nested_list_of_strings(error_msgs).replace( '\n', '\n ')) raise ValueError(msg)
def test_build_combine_archive_for_model_xpp_with_plot(self): model_filename = os.path.join(os.path.dirname(__file__), '..', 'fixtures', 'xpp', 'wilson-cowan.ode') archive_filename = os.path.join(self.dir_name, 'archive.omex') model_utils.build_combine_archive_for_model( model_filename, ModelLanguage.XPP, UniformTimeCourseSimulation, archive_filename, ) archive_dirname = os.path.join(self.dir_name, 'archive') archive = CombineArchiveReader().run(archive_filename, archive_dirname) doc = SedmlSimulationReader().run( os.path.join(archive_dirname, 'simulation.sedml')) output = doc.outputs[-1] self.assertIsInstance(output, Plot2D) self.assertEqual(len(output.curves), 1) self.assertEqual(len(output.curves[0].x_data_generator.variables), 1) self.assertEqual(len(output.curves[0].y_data_generator.variables), 1) self.assertEqual(output.curves[0].x_data_generator.variables[0].target, 'U') self.assertEqual(output.curves[0].y_data_generator.variables[0].target, 'V') self.assertEqual(output.curves[0].x_scale, AxisScale.linear) self.assertEqual(output.curves[0].y_scale, AxisScale.linear)
def test_build_combine_archive_for_model_bngl(self): model_filename = os.path.join(os.path.dirname(__file__), '..', 'fixtures', 'smoldyn', 'bounce1.txt') archive_filename = os.path.join(self.dir_name, 'archive.omex') extra_filename = os.path.join(self.dir_name, 'extra.txt') with open(extra_filename, 'w') as file: file.write('extra content') model_utils.build_combine_archive_for_model( model_filename, ModelLanguage.Smoldyn, UniformTimeCourseSimulation, archive_filename, extra_contents={ extra_filename: CombineArchiveContent( location='extra.txt', format=CombineArchiveContentFormat.TEXT, ), }, ) archive_dirname = os.path.join(self.dir_name, 'archive') archive = CombineArchiveReader().run(archive_filename, archive_dirname) doc = SedmlSimulationReader().run( os.path.join(archive_dirname, 'simulation.sedml')) sim = doc.simulations[0] self.assertEqual(sim.initial_time, 0.) self.assertEqual(sim.output_start_time, 0.) self.assertEqual(sim.output_end_time, 100.) self.assertEqual(sim.number_of_steps, 10000) self.assertEqual(sim.algorithm.kisao_id, 'KISAO_0000057')
def test_modify_combine_archive_with_uploaded_model_file_and_download( self): endpoint = '/combine/modify' archive_filename = os.path.abspath( os.path.join(self.FIXTURES_DIR, self.TEST_CASE + '.omex')) with open(archive_filename, 'rb') as file: archive_content = file.read() archive_specs_filename = os.path.join(self.FIXTURES_DIR, 'archive-specs.json') with open(archive_specs_filename, 'rb') as file: archive_specs = json.load(file) file_0_path = os.path.abspath( os.path.join(self.FIXTURES_DIR, 'model.xml')) file_1_path = os.path.abspath( os.path.join(self.FIXTURES_DIR, 'file.txt')) archive_specs['contents'][0]['location'][ 'path'] = './Caravagna2010.xml' archive_specs['contents'][0]['location']['value'][ 'filename'] = file_0_path archive_specs['contents'][1]['location']['value'][ 'filename'] = file_1_path fid_0 = open(file_0_path, 'rb') fid_1 = open(file_1_path, 'rb') data = MultiDict([ ('archive', json.dumps({'url': 'archive.omex'})), ('specs', json.dumps(archive_specs)), ('files', fid_0), ('files', fid_1), ('download', True), ]) with app.app.app.test_client() as client: with mock.patch('requests.get', return_value=mock.Mock( raise_for_status=lambda: None, content=archive_content)): response = client.post(endpoint, data=data, content_type="multipart/form-data") fid_0.close() fid_1.close() self.assertEqual(response.status_code, 200, response.json) self.assertEqual(response.content_type, 'application/zip') downloaded_archive_filename = os.path.join(self.temp_dirname, 'archive-downloaded.omex') with open(downloaded_archive_filename, 'wb') as file: file.write(response.data) contents_dirname = os.path.join(self.temp_dirname, 'archive') archive = CombineArchiveReader().run(downloaded_archive_filename, contents_dirname) self.assertEqual(len(archive.contents), 5)
def gen_sedml_2d_3d(omex_file_path, base_out_path): temp_path = os.path.join(base_out_path, "temp") if not os.path.exists(temp_path): os.mkdir(temp_path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) # defining archive archive = CombineArchiveReader().run( in_file=omex_file_path, out_dir=temp_path, try_reading_as_plain_zip_archive=False) # determine files to execute sedml_contents = get_sedml_contents(archive) for i_content, content in enumerate(sedml_contents): content_filename = os.path.join(temp_path, content.location) sedml_name = content.location.split('/')[1].split('.')[0] doc = SedmlSimulationReader().run(content_filename) for output in doc.outputs: if isinstance(output, (Plot2D, Plot3D)): report = Report(id='__plot__' + output.id, name=output.name) data_generators = {} if isinstance(output, Plot2D): for curve in output.curves: data_generators[ curve.x_data_generator.id] = curve.x_data_generator data_generators[ curve.y_data_generator.id] = curve.y_data_generator elif isinstance(output, Plot3D): for surface in output.surfaces: data_generators[surface.x_data_generator. id] = surface.x_data_generator data_generators[surface.y_data_generator. id] = surface.y_data_generator data_generators[surface.z_data_generator. id] = surface.z_data_generator for data_generator in data_generators.values(): report.data_sets.append( DataSet( id='__data_set__{}_{}'.format( output.id, data_generator.id), name=data_generator.name, label=data_generator.id, data_generator=data_generator, )) report.data_sets.sort(key=lambda data_set: data_set.id) doc.outputs.append(report) filename_with_reports_for_plots = os.path.join( temp_path, f'simulation_{sedml_name}.sedml') SedmlSimulationWriter().run(doc, filename_with_reports_for_plots, validate_models_with_languages=False)
def test_hdf5_response(self): endpoint = '/run/run' archive_filename = os.path.join( self.FIXTURES_DIRNAME, 'Caravagna-J-Theor-Biol-2010-tumor-suppressive-oscillations.omex') data = MultiDict([ ('_type', 'SimulationRun'), ('archiveUrl', 'https://web.site/archive.omex'), ('simulator', 'copasi'), ]) with app.app.app.test_client() as client: with open(archive_filename, 'rb') as archive_file: with mock.patch('requests.get', return_value=mock.Mock( raise_for_status=lambda: None, content=archive_file.read())): response = client.post( endpoint, data=data, content_type="multipart/form-data", headers={'Accept': 'application/x-hdf'}) self.assertEqual(response.status_code, 200, response.json) h5_filename = os.path.join(self.tmp_dirname, 'reports.h5') with open(h5_filename, 'wb') as file: file.write(response.data) self.assertEqual([ 'BIOMD0000000912_sim.sedml/BIOMD0000000912_report', 'BIOMD0000000912_sim.sedml/plot_1' ], ReportReader().get_ids(self.tmp_dirname)) archive_dirname = os.path.join(self.tmp_dirname, 'archive') CombineArchiveReader().run(archive_filename, archive_dirname) sed_doc = SedmlSimulationReader().run( os.path.join(archive_dirname, 'BIOMD0000000912_sim.sedml')) report = next(output for output in sed_doc.outputs if output.id == 'BIOMD0000000912_report') data_set_results = ReportReader().run( report, self.tmp_dirname, 'BIOMD0000000912_sim.sedml/BIOMD0000000912_report') self.assertEqual( set(data_set_results.keys()), set([ 'data_set_time', 'data_set_T', 'data_set_E', 'data_set_I', ]), ) numpy.testing.assert_allclose(data_set_results['data_set_time'], numpy.linspace(0., 1000., 5001)) for values in data_set_results.values(): self.assertEqual(values.shape, (5001, )) self.assertFalse(numpy.any(numpy.isnan(values)))
def handler(url, location): ''' Retrieve a file at a location in a COMBINE/OMEX archive Args: url (:obj:`str`): URL for a COMBINE/OMEX archive location (:obj:`str`): location of the COMBINE/OMEX archive to retrieve Returns: :obj:`werkzeug.wrappers.response.Response`: response with file from COMBINE/OMEX archive ''' # create temporary working directory temp_dirname = get_temp_dir() archive_filename = os.path.join(temp_dirname, 'archive.omex') # get COMBINE/OMEX archive try: response = requests.get(url) response.raise_for_status() except requests.exceptions.RequestException as exception: title = 'COMBINE/OMEX archive could not be loaded from `{}`'.format( url) raise BadRequestException( title=title, instance=exception, ) # save archive to local temporary file with open(archive_filename, 'wb') as file: file.write(response.content) # read archive archive_dirname = os.path.join(temp_dirname, 'archive') try: archive = CombineArchiveReader().run(archive_filename, archive_dirname) except Exception as exception: # return exception raise BadRequestException( title='`{}` is not a valid COMBINE/OMEX archive'.format(url), instance=exception, ) filename = os.path.join(archive_dirname, location) if not os.path.isfile(filename): msg = '`{}` is not a valid location in the COMBINE/OMEX archive.'.format(location) raise BadRequestException( title=msg, instance=ValueError()) mimetype = None for content in archive.contents: if os.path.relpath(content.location, '.') == os.path.relpath(location, '.'): if content.format.startswith('http://purl.org/NET/mediatypes/'): mimetype = content.format[len('http://purl.org/NET/mediatypes/'):] break return flask.send_file(filename, mimetype=mimetype, as_attachment=False, attachment_filename=os.path.basename(location))
def test_validate_no_metadata(self): os.remove(os.path.join(self.tmp_dir, 'thumbnail.png')) config = Config(VALIDATE_OMEX_METADATA=True) archive = CombineArchiveReader().run(os.path.join( self.FIXTURES_DIR, 'no-metadata.omex'), self.tmp_dir, config=config) errors, warnings = validate(archive, self.tmp_dir, config=config) self.assertEqual(errors, []) config = Config(VALIDATE_OMEX_METADATA=False) archive = CombineArchiveReader().run(os.path.join( self.FIXTURES_DIR, 'no-metadata.omex'), self.tmp_dir, config=config) errors, warnings = validate(archive, self.tmp_dir, config=config) self.assertEqual(errors, []) config = Config(VALIDATE_OMEX_METADATA=True) archive = CombineArchiveReader().run(os.path.join( self.FIXTURES_DIR, 'no-metadata.omex'), self.tmp_dir, config=config) errors, warnings = validate( archive, self.tmp_dir, formats_to_validate=list( CombineArchiveContentFormat.__members__.values()), config=config) self.assertNotEqual(errors, []) config = Config(VALIDATE_OMEX_METADATA=False) archive = CombineArchiveReader().run(os.path.join( self.FIXTURES_DIR, 'no-metadata.omex'), self.tmp_dir, config=config) errors, warnings = validate( archive, self.tmp_dir, formats_to_validate=list( CombineArchiveContentFormat.__members__.values()), config=config) self.assertEqual(errors, [])
def exec_plot_output_sed_doc(omex_file_path, base_out_path): archive = CombineArchiveReader().run(in_file=omex_file_path, out_dir=tmp_dir, try_reading_as_plain_zip_archive=True) # determine files to execute sedml_contents = get_sedml_contents(archive) report_results = ReportResults() for i_content, content in enumerate(sedml_contents): content_filename = os.path.join(tmp_dir, content.location) for report_filename in glob.glob( os.path.join(base_out_path, content.location, '*.csv')): if report_filename.find('__plot__') != -1: report_id = os.path.splitext( os.path.basename(report_filename))[0] # read report from CSV file produced by tellurium # data_set_df = pd.read_csv(report_filename).transpose() data_set_df = pd.read_csv(report_filename, header=None).T data_set_df.columns = data_set_df.iloc[0] data_set_df.drop(0, inplace=True) data_set_df.reset_index(inplace=True) data_set_df.drop('index', axis=1, inplace=True) # create pseudo-report for ReportWriter datasets = [] for col in list(data_set_df.columns): datasets.append(DataSet(id=col, label=col, name=col)) #report.data_sets = datasets report = Report(id=report_id, name=report_id, data_sets=datasets) data_set_results = DataSetResults() for col in list(data_set_df.columns): data_set_results[col] = data_set_df[col].values # append to data structure of report results # save file in desired BioSimulators format(s) export_id = report_id.replace('__plot__', '') report.id = export_id rel_path = os.path.join(content.location, report.id) if len(rel_path.split("./")) > 1: rel_path = rel_path.split("./")[1] ReportWriter().run(report, data_set_results, base_out_path, rel_path, format='h5') os.rename(report_filename, report_filename.replace('__plot__', ''))
def test_manifest_in_manifest(self): out_dir = os.path.join(self.tmp_dir, 'out') archive = CombineArchiveReader().run( os.path.join(os.path.dirname(__file__), '..', 'fixtures', 'manifest-in-manifest.omex'), out_dir) errors, warnings = validate(archive, out_dir) self.assertEqual(errors, []) self.assertIn( 'manifests should not contain content entries for themselves', flatten_nested_list_of_strings(warnings)) out_dir = os.path.join(self.tmp_dir, 'out') archive = CombineArchiveReader().run( os.path.join(os.path.dirname(__file__), '..', 'fixtures', 'multiple-manifests.omex'), out_dir) errors, warnings = validate(archive, out_dir) self.assertEqual(errors, []) self.assertIn( 'manifests should not contain content entries for themselves', flatten_nested_list_of_strings(warnings))
def test_create_combine_archive_with_uploaded_model_file_and_download( self): endpoint = '/combine/create' archive_specs_filename = os.path.join(self.FIXTURES_DIR, 'archive-specs.json') with open(archive_specs_filename, 'rb') as file: archive_specs = json.load(file) file_0_path = os.path.abspath( os.path.join(self.FIXTURES_DIR, 'model.xml')) file_1_path = os.path.abspath( os.path.join(self.FIXTURES_DIR, 'file.txt')) archive_specs['contents'][0]['location']['value'][ 'filename'] = file_0_path archive_specs['contents'][1]['location']['value'][ 'filename'] = file_1_path fid_0 = open(file_0_path, 'rb') fid_1 = open(file_1_path, 'rb') data = MultiDict([ ('specs', json.dumps(archive_specs)), ('files', fid_0), ('files', fid_1), ('download', True), ]) with app.app.app.test_client() as client: archive_filename = os.path.join(self.temp_dirname, 'archive.omex') response = client.post(endpoint, data=data, content_type="multipart/form-data") fid_0.close() fid_1.close() self.assertEqual(response.status_code, 200, response.json) self.assertEqual(response.content_type, 'application/zip') downloaded_archive_filename = os.path.join(self.temp_dirname, 'archive-downloaded.omex') with open(downloaded_archive_filename, 'wb') as file: file.write(response.data) contents_dirname = os.path.join(self.temp_dirname, 'archive') archive = CombineArchiveReader().run(downloaded_archive_filename, contents_dirname) self.assertEqual(len(archive.contents), 3)
def test_validate(self): os.remove(os.path.join(self.tmp_dir, 'thumbnail.png')) archive = CombineArchiveReader().run(self.OMEX_FIXTURE, self.tmp_dir) errors, warnings = validate(archive, self.tmp_dir) self.assertEqual(errors, []) self.assertNotEqual(warnings, []) archive2 = copy.deepcopy(archive) for content in archive.contents: archive2.contents.append(content) errors, warnings = validate(archive2, self.tmp_dir) self.assertIn('contains repeated content items', flatten_nested_list_of_strings(errors)) archive2 = copy.deepcopy(archive) archive2.contents = [] errors, warnings = validate(archive2, self.tmp_dir) self.assertIn('does not contain content items', flatten_nested_list_of_strings(errors))
def exec_plot_output_sed_doc(omex_file_path, base_out_path): config = Config(VALIDATE_OMEX_MANIFESTS=False) archive = CombineArchiveReader().run(in_file=omex_file_path, out_dir=tmp_dir, config=config) # determine files to execute sedml_contents = get_sedml_contents(archive) report_results = ReportResults() for i_content, content in enumerate(sedml_contents): content_filename = os.path.join(tmp_dir, content.location) for report_filename in glob.glob( os.path.join(base_out_path, content.location, '*.csv')): if report_filename.find('__plot__') != -1: report_id = os.path.splitext( os.path.basename(report_filename))[0] # read report from CSV file produced by tellurium # data_set_df = pd.read_csv(report_filename).transpose() data_set_df = pd.read_csv(report_filename, header=None).T datasets = [] for col in data_set_df.columns: datasets.append( DataSet(id=data_set_df.loc[0, col], label=data_set_df.loc[1, col], name=data_set_df.loc[2, col])) report = Report(id=report_id, name=report_id, data_sets=datasets) data_set_df.columns = data_set_df.iloc[0] data_set_df.drop(0, inplace=True) data_set_df.drop(1, inplace=True) data_set_df.drop(2, inplace=True) data_set_df.reset_index(inplace=True) data_set_df.drop('index', axis=1, inplace=True) # create pseudo-report for ReportWriter data_set_results = DataSetResults() for col in list(data_set_df.columns): data_set_results[col] = data_set_df[col].to_numpy( dtype='float64') # append to data structure of report results # save file in desired BioSimulators format(s) export_id = report_id.replace('__plot__', '') report.id = export_id rel_path = os.path.join(content.location, report.id) if len(rel_path.split("./")) > 1: rel_path = rel_path.split("./")[1] # print("base: ", base_out_path, file=sys.stdout) # print("rel: ", rel_path, file=sys.stdout) ReportWriter().run(report, data_set_results, base_out_path, rel_path, format='h5', type=Plot2D) os.rename(report_filename, report_filename.replace('__plot__', '')) else: print("report : ", report_filename, file=sys.stdout) report_id = os.path.splitext( os.path.basename(report_filename))[0] data_set_df = pd.read_csv(report_filename, header=None).T datasets = [] for col in data_set_df.columns: datasets.append( DataSet(id=data_set_df.loc[0, col], label=data_set_df.loc[1, col], name="")) report = Report(id=report_id, name=report_id, data_sets=datasets) data_set_df.columns = data_set_df.iloc[0] # use ids data_set_df.drop(0, inplace=True) data_set_df.drop(1, inplace=True) data_set_df.drop(2, inplace=True) data_set_df.reset_index(inplace=True) data_set_df.drop('index', axis=1, inplace=True) data_set_results = DataSetResults() for col in list(data_set_df.columns): data_set_results[col] = data_set_df[col].to_numpy( dtype='float64') rel_path = os.path.join(content.location, report.id) if len(rel_path.split("./")) > 1: rel_path = rel_path.split("./")[1] ReportWriter().run(report, data_set_results, base_out_path, rel_path, format='h5', type=Report)
def exec_sed_doc(omex_file_path, base_out_path): # defining archive config = Config(VALIDATE_OMEX_MANIFESTS=False) archive = CombineArchiveReader().run(in_file=omex_file_path, out_dir=tmp_dir, config=config) # determine files to execute sedml_contents = get_sedml_contents(archive) report_results = ReportResults() for i_content, content in enumerate(sedml_contents): content_filename = os.path.join(tmp_dir, content.location) doc = SedmlSimulationReader().run(content_filename) for report_filename in glob.glob( os.path.join(base_out_path, content.location, '*.csv')): report_id = os.path.splitext(os.path.basename(report_filename))[0] # read report from CSV file produced by VCell data_set_df = pd.read_csv(report_filename).transpose() data_set_df.columns = data_set_df.iloc[0] data_set_df = data_set_df.drop(data_set_df.iloc[0].name) data_set_df = data_set_df.reset_index() data_set_df = data_set_df.rename( columns={'index': data_set_df.columns.name}) data_set_df = data_set_df.transpose() data_set_df.index.name = None report = next(report for report in doc.outputs if report.id == report_id) data_set_results = DataSetResults() # print("report: ", report, file=sys.stderr) # print("report Type: ", type(report), file=sys.stderr) # print("Plot Type: ", Plot2D, file=sys.stderr) if type(report) != Plot2D and type(report) != Plot3D: # Considering the scenario where it has the datasets in sedml for data_set in report.data_sets: data_set_results[data_set.id] = data_set_df.loc[ data_set.label, :].to_numpy(dtype='float64') # print("DF for report: ", data_set_results[data_set.id], file=sys.stderr) # print("df.types: ", data_set_results[data_set.id].dtype, file=sys.stderr) else: data_set_df = pd.read_csv(report_filename, header=None).T data_set_df.columns = data_set_df.iloc[0] data_set_df.drop(0, inplace=True) data_set_df.reset_index(inplace=True) data_set_df.drop('index', axis=1, inplace=True) # print("DF for plot: ", data_set_df, file=sys.stderr) # Considering the scenario where it doesn't have datasets in sedml (pseudo sedml for plots) for col in list(data_set_df.columns): data_set_results[col] = data_set_df[col].values # append to data structure of report results report_results[report_id] = data_set_results # save file in desired BioSimulators format(s) # for report_format in report_formats: # print("HDF report: ", report, file=sys.stderr) # print("HDF dataset results: ", data_set_results, file=sys.stderr) # print("HDF base_out_path: ", base_out_path,file=sys.stderr) # print("HDF path: ", os.path.join(content.location, report.id), file=sys.stderr) rel_path = os.path.join(content.location, report.id) if len(rel_path.split("./")) > 1: rel_path = rel_path.split("./")[1] if type(report) != Plot2D and type(report) != Plot3D: ReportWriter().run(report, data_set_results, base_out_path, rel_path, format='h5') else: datasets = [] for col in list(data_set_df.columns): datasets.append(DataSet(id=col, label=col, name=col)) report.data_sets = datasets ReportWriter().run(report, data_set_results, base_out_path, rel_path, format='h5') # Remove temp directory shutil.rmtree(tmp_dir)
def test_add_file_to_combine_archive_download(self): endpoint = '/combine/file' archive_filename = os.path.join(self.FIXTURES_DIR, self.TEST_CASE + '.omex') new_content_filename = os.path.abspath( os.path.join(self.FIXTURES_DIR, 'file.txt')) new_content = { '_type': 'CombineArchiveContent', 'location': './NewLocation.txt', "filename": new_content_filename, 'format': 'http://purl.org/NET/mediatypes/text/plain', 'master': False, } archive_specs_filename = os.path.join(self.FIXTURES_DIR, 'archive-specs.json') with open(archive_specs_filename, 'rb') as file: archive_specs = json.load(file) archive_fid = open(archive_filename, 'rb') new_content_fid = open(new_content_filename, 'rb') data = MultiDict([ ('archive', json.dumps({"filename": archive_filename})), ('newContent', json.dumps(new_content)), ('files', archive_fid), ('files', new_content_fid), ('download', True), ]) modified_archive_filename = os.path.join(self.temp_dirname, 'archive.omex') with app.app.app.test_client() as client: response = client.post(endpoint, data=data, content_type="multipart/form-data") archive_fid.close() new_content_fid.close() self.assertEqual(response.status_code, 200, response.json) self.assertEqual(response.content_type, 'application/zip') modified_archive_filename = os.path.join(self.temp_dirname, 'archive-downloaded.omex') with open(modified_archive_filename, 'wb') as file: file.write(response.data) modified_archive_dirname = os.path.join(self.temp_dirname, 'archive') archive = CombineArchiveReader().run(modified_archive_filename, modified_archive_dirname) self.assertEqual( set( os.path.relpath(content.location, '.') for content in archive.contents), set([ 'BIOMD0000000912_sim.sedml', 'Caravagna2010.xml', 'metadata.rdf', 'NewLocation.txt', ])) content = next(content for content in archive.contents if content.location == 'NewLocation.txt') self.assertEqual(content.format, 'http://purl.org/NET/mediatypes/text/plain') self.assertEqual(content.master, False) self.assertTrue( os.path.isfile( os.path.join(modified_archive_dirname, 'NewLocation.txt')))
def handler(body, files=None): ''' Add one or more files to a COMBINE/OMEX archive Args: body (:obj:`dict`): dictionary with schema ``CombineArchiveAndAdditionalContent`` with the specifications of the desired additions to the COMBINE/OMEX archive files (:obj:`list` of :obj:`werkzeug.datastructures.FileStorage`, optional): files (e.g., SBML file) Returns: :obj:`werkzeug.wrappers.response.Response` or :obj:`str`: response with COMBINE/OMEX archive or a URL to a COMBINE/OMEX archive ''' archive = body.get('archive') new_content = body.get('newContent') files = connexion.request.files.getlist('files') filename_map = {file.filename: file for file in files} overwrite_locations = body.get('overwriteLocations', True) download = body.get('download', False) # create temporary working directory temp_dirname = get_temp_dir() archive_filename = os.path.join(temp_dirname, 'archive.omex') # get COMBINE/OMEX archive archive_url = archive.get('url', None) if archive_url: try: response = requests.get(archive_url) response.raise_for_status() except requests.exceptions.RequestException as exception: title = 'COMBINE/OMEX archive could not be loaded from `{}`'.format( archive_url) raise BadRequestException( title=title, instance=exception, ) # save archive to local temporary file with open(archive_filename, 'wb') as file: file.write(response.content) else: archive_file = filename_map.get(archive['filename'], None) if archive_file is None: raise BadRequestException( title='Archive file with name `{}` was not uploaded'.format( archive['filename']), instance=ValueError(), ) archive_file.save(archive_filename) # read archive archive_dirname = os.path.join(temp_dirname, 'archive') try: archive = CombineArchiveReader().run(archive_filename, archive_dirname) except Exception as exception: # return exception raise BadRequestException( title='COMBINE/OMEX archive is not valid.', instance=exception, ) location_content_map = { os.path.relpath(content.location, '.'): content for content in archive.contents } new_location = os.path.relpath(new_content['location'], '.') content = location_content_map.get(new_location, None) if content is None: content = CombineArchiveContent(location=new_location) archive.contents.append(content) else: if not overwrite_locations: new_location_parts = os.path.splitext(new_location) i_file = 0 while True: i_file += 1 temp_new_location = new_location_parts[0] + '_' + str( i_file) + new_location_parts[1] if temp_new_location not in location_content_map: new_location = temp_new_location break content = CombineArchiveContent(location=new_location) archive.contents.append(content) content_filename = new_content['filename'] file = filename_map.get(content_filename, None) if file: file.save(os.path.join(archive_dirname, new_location)) else: raise BadRequestException( title='File with name `{}` was not uploaded'.format( content_filename), instance=ValueError(), ) content.format = new_content['format'] content.master = new_content['master'] # package COMBINE/OMEX archive CombineArchiveWriter().run(archive, archive_dirname, archive_filename) if download: return flask.send_file(archive_filename, mimetype='application/zip', as_attachment=True, attachment_filename='archive.omex') else: # save COMBINE/OMEX archive to S3 bucket archive_url = src.s3.save_temporary_combine_archive_to_s3_bucket( archive_filename, public=True) # return URL for archive in S3 bucket return archive_url
def handler(body, file=None): ''' Get the specifications of the SED-ML files in a COMBINE/OMEX arvhive Args: body (:obj:`dict`): dictionary with keys * ``url`` whose value has schema ``Url`` with the URL for a COMBINE/OMEX archive file (:obj:`werkzeug.datastructures.FileStorage`, optional): COMBINE/OMEX archive file Returns: ``CombineArchive``: specifications of the SED-ML files in the COMBINE/OMEX archive ''' archive_file = file archive_url = body.get('url', None) if archive_url and archive_file: raise BadRequestException( title='Only one of `file` or `url` can be used at a time.', instance=ValueError(), ) if not archive_url and not archive_file: raise BadRequestException( title='One of `file` or `url` must be used.', instance=ValueError(), ) # create temporary working directory temp_dirname = get_temp_dir() archive_filename = os.path.join(temp_dirname, 'archive.omex') # get COMBINE/OMEX archive if archive_file: archive_file.save(archive_filename) else: try: response = requests.get(archive_url) response.raise_for_status() except requests.exceptions.RequestException as exception: title = 'COMBINE/OMEX archive could not be loaded from `{}`'.format( archive_url) raise BadRequestException( title=title, instance=exception, ) # save archive to local temporary file with open(archive_filename, 'wb') as file: file.write(response.content) # read archive archive_dirname = os.path.join(temp_dirname, 'archive') try: archive = CombineArchiveReader().run(archive_filename, archive_dirname) except Exception as exception: # return exception raise BadRequestException( title='`{}` is not a valid COMBINE/OMEX archive'.format( archive_url if archive_url else archive_file.filename), instance=exception, ) # get specifications of SED-ML outputs contents_specs = [] sedml_contents = get_sedml_contents(archive) for content in sedml_contents: sed_doc_filename = os.path.join(archive_dirname, content.location) try: sed_doc = SedmlSimulationReader().run( sed_doc_filename, validate_semantics=False, validate_models_with_languages=False) except Exception: traceback.print_exc() continue sed_doc_specs = get_sed_document_specs(sed_doc) content_specs = { '_type': 'CombineArchiveSedDocSpecsContent', 'location': { '_type': 'CombineArchiveSedDocSpecsLocation', 'path': content.location, 'value': sed_doc_specs, }, 'format': content.format, 'master': content.master, } contents_specs.append(content_specs) # format response response = { '_type': 'CombineArchiveSedDocSpecs', 'contents': contents_specs } # return reponse return response
def test_add_file_to_combine_archive_at_existing_location_overwrite(self): endpoint = '/combine/file' archive_filename = os.path.join(self.FIXTURES_DIR, self.TEST_CASE + '.omex') new_content_filename = os.path.abspath( os.path.join(self.FIXTURES_DIR, 'file.txt')) new_content = { '_type': 'CombineArchiveContent', 'location': 'Caravagna2010.xml', "filename": new_content_filename, 'format': 'http://purl.org/NET/mediatypes/text/plain', 'master': False, } archive_specs_filename = os.path.join(self.FIXTURES_DIR, 'archive-specs.json') with open(archive_specs_filename, 'rb') as file: archive_specs = json.load(file) archive_fid = open(archive_filename, 'rb') new_content_fid = open(new_content_filename, 'rb') data = MultiDict([ ('archive', json.dumps({"filename": archive_filename})), ('newContent', json.dumps(new_content)), ('files', archive_fid), ('files', new_content_fid), ('overwriteLocations', True), ]) modified_archive_filename = os.path.join(self.temp_dirname, 'archive.omex') with app.app.app.test_client() as client: def save_temporary_combine_archive_to_s3_bucket( filename, public=True, archive_filename=modified_archive_filename): shutil.copy(filename, archive_filename) return archive_filename with mock.patch( 'src.s3.save_temporary_combine_archive_to_s3_bucket', side_effect=save_temporary_combine_archive_to_s3_bucket): response = client.post(endpoint, data=data, content_type="multipart/form-data") archive_fid.close() new_content_fid.close() self.assertEqual(response.status_code, 200, response.json) self.assertEqual(response.json, modified_archive_filename) modified_archive_dirname = os.path.join(self.temp_dirname, 'archive') archive = CombineArchiveReader().run(modified_archive_filename, modified_archive_dirname) self.assertEqual( set( os.path.relpath(content.location, '.') for content in archive.contents), set([ 'BIOMD0000000912_sim.sedml', 'Caravagna2010.xml', 'metadata.rdf', ])) content = next( content for content in archive.contents if os.path.relpath(content.location, '.') == 'Caravagna2010.xml') self.assertEqual(content.format, 'http://purl.org/NET/mediatypes/text/plain') self.assertEqual(content.master, False) with open(os.path.join(modified_archive_dirname, 'Caravagna2010.xml'), 'r') as file: self.assertEqual(file.read(), 'A text file\n')
def handler(body, file=None): ''' Validate a COMBINE/OMEX archive Args: body (:obj:`dict`): dictionary in schema ``ValidateCombineArchiveFileOrUrl`` with keys * ``url`` whose value has schema ``Url`` with the URL for a COMBINE/OMEX archive * ``omexMetadataFormat`` (:obj:`str`): format of the OMEX Metadata files * ``omexMetadataSchema`` (:obj:`str`): schema for validating the OMEX Metadata files * ``validateOmexManifest`` (:obj:`bool`, optional): Whether to validate the OMEX manifest file in the archive * ``validateSedml`` (:obj:`bool`, optional): Whether to validate the SED-ML files in the archive * ``validateSedmlModels`` (:obj:`bool`, optional): Whether to validate the sources of the models in the SED-ML files in the archive * ``validateOmexMetadata`` (:obj:`bool`, optional): Whether to validate the OMEX metdata files in the archive according to `BioSimulators' conventions <https://docs.biosimulations.org/concepts/conventions/simulation-project-metadata/>`_ * ``validateImages`` (:obj:`bool`, optional): Whether to validate the images (BMP, GIF, JPEG, PNG, TIFF WEBP) files in the archive file (:obj:`werkzeug.datastructures.FileStorage`): COMBINE/OMEX archive file Returns: ``ValidationReport``: information about the validity or lack thereof of a COMBINE/OMEX archive ''' try: omexMetadataInputFormat = OmexMetadataInputFormat( body['omexMetadataFormat']) except ValueError as exception: raise BadRequestException( title='`omexMetadataFormat` must be a recognized format.', exception=exception) try: omexMetadataSchema = OmexMetadataSchema(body['omexMetadataSchema']) except ValueError as exception: raise BadRequestException( title='`omexMetadataSchema` must be a recognized schema.', exception=exception) config = Config( OMEX_METADATA_INPUT_FORMAT=omexMetadataInputFormat, OMEX_METADATA_SCHEMA=omexMetadataSchema, VALIDATE_OMEX_MANIFESTS=body.get('validateOmexManifest', True), VALIDATE_SEDML=body.get('validateSedml', True), VALIDATE_SEDML_MODELS=body.get('validateSedmlModels', True), VALIDATE_OMEX_METADATA=body.get('validateOmexMetadata', True), VALIDATE_IMAGES=body.get('validateImages', True), ) archive_file = file archive_url = body.get('url', None) if archive_url and archive_file: raise BadRequestException( title='Only one of `file` or `url` can be used at a time.', instance=ValueError(), ) if not archive_url and not archive_file: raise BadRequestException( title='One of `file` or `url` must be used.', instance=ValueError(), ) # create temporary working directory temp_dirname = get_temp_dir() archive_filename = os.path.join(temp_dirname, 'archive.omex') # get COMBINE/OMEX archive if archive_file: archive_file.save(archive_filename) else: try: response = requests.get(archive_url) response.raise_for_status() except requests.exceptions.RequestException as exception: title = 'COMBINE/OMEX archive could not be loaded from `{}`'.format( archive_url) raise BadRequestException( title=title, instance=exception, ) # save archive to local temporary file with open(archive_filename, 'wb') as file: file.write(response.content) # read archive archive_dirname = os.path.join(temp_dirname, 'archive') reader = CombineArchiveReader() errors = [] warnings = [] try: archive = reader.run(archive_filename, archive_dirname, config=config) except Exception as exception: errors = [[ 'The file could not be parsed as a COMBINE/OMEX archive.', [[str(exception)]] ]] if not errors: errors, warnings = validate( archive, archive_dirname, formats_to_validate=list( CombineArchiveContentFormat.__members__.values()), config=config, ) return make_validation_report(errors, warnings, filenames=[archive_filename])
def handler(body, files=None): ''' Modify a COMBINE/OMEX archive. Args: body (:obj:`dict`): dictionary with schema ``ModifyCombineArchiveSpecsAndFiles`` with the specifications of the COMBINE/OMEX archive to create files (:obj:`list` of :obj:`werkzeug.datastructures.FileStorage`, optional): files (e.g., SBML file) Returns: :obj:`werkzeug.wrappers.response.Response` or :obj:`str`: response with COMBINE/OMEX archive or a URL to a COMBINE/OMEX archive ''' download = body.get('download', False) archive_filename_or_url = body['archive'] archive_specs = body['specs'] files = connexion.request.files.getlist('files') # build map from model filenames to file objects filename_map = { file.filename: file for file in files } # create temporary working directory temp_dirname = get_temp_dir() archive_filename = os.path.join(temp_dirname, 'archive.omex') # save COMBINE/OMEX archive to local temporary file if 'filename' in archive_filename_or_url and 'url' in archive_filename_or_url: raise BadRequestException( title='Only one of `filename` or `url` can be used at a time.', instance=ValueError(), ) elif 'filename' not in archive_filename_or_url and 'url' not in archive_filename_or_url: raise BadRequestException( title='One of `filename` or `url` must be used.', instance=ValueError(), ) elif 'filename' in archive_filename_or_url: # get COMBINE/OMEX archive archive_file = filename_map[archive_filename_or_url['filename']] # save archive to local temporary file archive_file.save(archive_filename) else: # get COMBINE/OMEX archive archive_url = archive_filename_or_url['url'] try: response = requests.get(archive_url) response.raise_for_status() except requests.exceptions.RequestException as exception: title = 'COMBINE/OMEX archive could not be loaded from `{}`'.format(archive_url) raise BadRequestException( title=title, instance=exception, ) # save archive to local temporary file with open(archive_filename, 'wb') as file: file.write(response.content) # read archive archive_dirname = os.path.join(temp_dirname, 'archive') try: archive = CombineArchiveReader().run(archive_filename, archive_dirname) except Exception as exception: # return exception raise BadRequestException( title='`{}` is not a valid COMBINE/OMEX archive'.format( archive_filename_or_url.get('filename', None) or archive_filename_or_url.get('url', None) ), instance=exception, ) # build map of locations in archive to contents archive_location_to_contents = { os.path.relpath(content.location, '.'): content for content in archive.contents } # add files to archive or modify existing files for content in archive_specs['contents']: content_type = content['location']['value']['_type'] if content_type == 'SedDocument': sed_doc = export_sed_doc(content['location']['value']) # save SED-ML document to file try: SedmlSimulationWriter().run( sed_doc, os.path.join(archive_dirname, content['location']['path']), validate_models_with_languages=False) except ValueError as exception: raise BadRequestException( title='`{}` does not contain a configuration for a valid SED-ML document.'.format( content['location']['value']), instance=exception, ) elif content_type == 'CombineArchiveContentFile': file = filename_map.get( content['location']['value']['filename'], None) if not file: raise BadRequestException( title='File with name `{}` was not uploaded'.format( content['location']['value']['filename']), instance=ValueError(), ) filename = os.path.join(archive_dirname, content['location']['path']) if not os.path.isdir(os.path.dirname(filename)): os.makedirs(os.path.dirname(filename)) file.save(filename) elif content_type == 'CombineArchiveContentUrl': filename = os.path.join(archive_dirname, content['location']['path']) if not os.path.isdir(os.path.dirname(filename)): os.makedirs(os.path.dirname(filename)) content_url = content['location']['value']['url'] try: response = requests.get(content_url) response.raise_for_status() except requests.exceptions.RequestException as exception: title = 'COMBINE/OMEX archive content could not be loaded from `{}`'.format( content_url) raise BadRequestException( title=title, instance=exception, ) with open(filename, 'wb') as file: file.write(response.content) else: raise BadRequestException( title='Content of type `{}` is not supported'.format( content_type), instance=NotImplementedError('Invalid content') ) # pragma: no cover: unreachable due to schema validation combine_archive_content = archive_location_to_contents.get(os.path.relpath(content['location']['path'], '.'), None) if combine_archive_content is None: combine_archive_content = CombineArchiveContent( location=content['location']['path'], format=content['format'], master=content['master'], ) archive.contents.append(combine_archive_content) else: combine_archive_content.format = content['format'] combine_archive_content.master = content['master'] # package COMBINE/OMEX archive CombineArchiveWriter().run(archive, archive_dirname, archive_filename) if download: return flask.send_file(archive_filename, mimetype='application/zip', as_attachment=True, attachment_filename='archive.omex') else: # save COMBINE/OMEX archive to S3 bucket archive_url = src.s3.save_temporary_combine_archive_to_s3_bucket(archive_filename, public=True) # return URL for archive in S3 bucket return archive_url
def test_modify_combine_archive_with_uploaded_model_file(self): endpoint = '/combine/modify' archive_specs_filename = os.path.join(self.FIXTURES_DIR, 'archive-specs.json') with open(archive_specs_filename, 'rb') as file: archive_specs = json.load(file) file_0_path = os.path.abspath( os.path.join(self.FIXTURES_DIR, 'model.xml')) file_1_path = os.path.abspath( os.path.join(self.FIXTURES_DIR, 'file.txt')) file_2_path = os.path.abspath( os.path.join(self.FIXTURES_DIR, self.TEST_CASE + '.omex')) archive_specs['contents'][0]['location']['path'] = 'Caravagna2010.xml' archive_specs['contents'][0]['location']['value'][ 'filename'] = file_0_path archive_specs['contents'][1]['location'][ 'path'] = 'subdir/' + archive_specs['contents'][1]['location'][ 'path'] archive_specs['contents'][1]['location']['value'][ 'filename'] = file_1_path fid_0 = open(file_0_path, 'rb') fid_1 = open(file_1_path, 'rb') fid_2 = open(file_2_path, 'rb') data = MultiDict([ ('archive', json.dumps({'filename': file_2_path})), ('specs', json.dumps(archive_specs)), ('files', fid_0), ('files', fid_1), ('files', fid_2), ]) with app.app.app.test_client() as client: archive_filename = os.path.join(self.temp_dirname, 'archive.omex') def save_temporary_combine_archive_to_s3_bucket( filename, public=True, archive_filename=archive_filename): shutil.copy(filename, archive_filename) return archive_filename with mock.patch( 'src.s3.save_temporary_combine_archive_to_s3_bucket', side_effect=save_temporary_combine_archive_to_s3_bucket): response = client.post(endpoint, data=data, content_type="multipart/form-data") fid_0.close() fid_1.close() fid_2.close() self.assertEqual(response.status_code, 200, response.json) self.assertEqual(response.json, archive_filename) contents_dirname = os.path.join(self.temp_dirname, 'archive') archive = CombineArchiveReader().run(archive_filename, contents_dirname) self.assertEqual(len(archive.contents), 5) for expected_content in archive_specs['contents']: found = False for content in archive.contents: if os.path.relpath(content.location, '.') == os.path.relpath( expected_content['location']['path'], '.'): found = True self.assertEqual(content.format, expected_content['format']) self.assertEqual(content.master, expected_content['master']) break self.assertTrue(found) sed_doc = SedmlSimulationReader().run( os.path.join(contents_dirname, archive_specs['contents'][2]['location']['path']), validate_models_with_languages=False) sed_doc_specs = archive_specs['contents'][2]['location']['value'] self.assertEqual(sed_doc.level, sed_doc_specs['level']) self.assertEqual(sed_doc.version, sed_doc_specs['version']) self.assertEqual(sed_doc.tasks[0].model, sed_doc.models[0]) self.assertEqual(len(sed_doc.models[0].changes), 2) self.assertEqual( sed_doc.models[0].changes[0].target, "/sbml:sbml/sbml:model/sbml:listOfParameters/sbml:parameter[@id='k1']/@value" ) self.assertEqual(sed_doc.models[0].changes[0].new_value, '1.2') self.assertEqual( sed_doc.models[0].changes[0].target_namespaces, { None: 'http://sed-ml.org/sed-ml/level1/version4', 'sbml': 'http://www.sbml.org/sbml/level3/version1/core', 'qual': 'http://www.sbml.org/sbml/level3/version1/qual/version1', }) self.assertEqual( sed_doc.tasks[0].simulation.algorithm.changes[0].kisao_id, 'KISAO_0000488') self.assertEqual( sed_doc.tasks[0].simulation.algorithm.changes[0].new_value, '10') self.assertEqual( sed_doc.outputs[1].curves[0].x_data_generator.variables[0].target, "/sbml:sbml/sbml:model/qual:listOfQualitativeSpecies/qual:qualitativeSpecies[@qual:id='x']" ) self.assertEqual( sed_doc.outputs[1].curves[0].x_data_generator.variables[0]. target_namespaces, { None: 'http://sed-ml.org/sed-ml/level1/version4', "sbml": "http://www.sbml.org/sbml/level3/version1/core", "qual": "http://www.sbml.org/sbml/level3/version1/qual/version1" }, )
def test_create_combine_archive_with_model_at_url(self): endpoint = '/combine/create' archive_specs_filename = os.path.join(self.FIXTURES_DIR, 'archive-specs.json') with open(archive_specs_filename, 'rb') as file: archive_specs = json.load(file) file_0_path = os.path.abspath( os.path.join(self.FIXTURES_DIR, 'model.xml')) file_1_path = os.path.abspath( os.path.join(self.FIXTURES_DIR, 'file.txt')) file_0_url = 'https://models.org/model.xml' file_1_url = 'https://models.org/file.txt' archive_specs['contents'][0]['location']['value'].pop('filename') archive_specs['contents'][1]['location']['value'].pop('filename') archive_specs['contents'][0]['location']['value'][ '_type'] = 'CombineArchiveContentUrl' archive_specs['contents'][1]['location']['value'][ '_type'] = 'CombineArchiveContentUrl' archive_specs['contents'][0]['location']['value']['url'] = file_0_url archive_specs['contents'][1]['location']['value']['url'] = file_1_url fid_0 = open(file_0_path, 'rb') fid_1 = open(file_1_path, 'rb') data = MultiDict([ ('specs', json.dumps(archive_specs)), ]) with app.app.app.test_client() as client: archive_filename = os.path.join(self.temp_dirname, 'archive.omex') def save_temporary_combine_archive_to_s3_bucket( filename, public=True, archive_filename=archive_filename): shutil.copy(filename, archive_filename) return archive_filename with mock.patch( 'src.s3.save_temporary_combine_archive_to_s3_bucket', side_effect=save_temporary_combine_archive_to_s3_bucket): def requests_get(url): assert url in [file_0_url, file_1_url] if url == file_0_url: return mock.Mock(raise_for_status=lambda: None, content=fid_0.read()) else: return mock.Mock(raise_for_status=lambda: None, content=fid_1.read()) with mock.patch('requests.get', side_effect=requests_get): response = client.post(endpoint, data=data, content_type="multipart/form-data") fid_0.close() fid_1.close() self.assertEqual(response.status_code, 200, response.json) self.assertEqual(response.json, archive_filename) contents_dirname = os.path.join(self.temp_dirname, 'archive') archive = CombineArchiveReader().run(archive_filename, contents_dirname) self.assertEqual(len(archive.contents), 3) for content, expected_content in zip(archive.contents, archive_specs['contents']): self.assertEqual(content.location, expected_content['location']['path']) self.assertEqual(content.format, expected_content['format']) self.assertEqual(content.master, expected_content['master']) sed_doc = SedmlSimulationReader().run( os.path.join(contents_dirname, archive_specs['contents'][2]['location']['path']), validate_models_with_languages=False) sed_doc_specs = archive_specs['contents'][2]['location']['value'] self.assertEqual(sed_doc.level, sed_doc_specs['level']) self.assertEqual(sed_doc.version, sed_doc_specs['version']) self.assertEqual( sed_doc.tasks[0].simulation.algorithm.changes[0].kisao_id, 'KISAO_0000488') self.assertEqual( sed_doc.tasks[0].simulation.algorithm.changes[0].new_value, '10') self.assertEqual( sed_doc.outputs[1].curves[0].x_data_generator.variables[0].target, "/sbml:sbml/sbml:model/qual:listOfQualitativeSpecies/qual:qualitativeSpecies[@qual:id='x']" ) self.assertEqual( sed_doc.outputs[1].curves[0].x_data_generator.variables[0]. target_namespaces, { None: 'http://sed-ml.org/sed-ml/level1/version4', "sbml": "http://www.sbml.org/sbml/level3/version1/core", "qual": "http://www.sbml.org/sbml/level3/version1/qual/version1" }, )
def test_create_combine_archive_with_uploaded_model_file(self): endpoint = '/combine/create' archive_specs_filename = os.path.join(self.FIXTURES_DIR, 'archive-specs.json') with open(archive_specs_filename, 'rb') as file: archive_specs = json.load(file) file_0_path = os.path.abspath( os.path.join(self.FIXTURES_DIR, 'model.xml')) file_1_path = os.path.abspath( os.path.join(self.FIXTURES_DIR, 'file.txt')) archive_specs['contents'][0]['location']['value'][ 'filename'] = file_0_path archive_specs['contents'][1]['location']['value'][ 'filename'] = file_1_path fid_0 = open(file_0_path, 'rb') fid_1 = open(file_1_path, 'rb') data = MultiDict([ ('specs', json.dumps(archive_specs)), ('files', fid_0), ('files', fid_1), ]) with app.app.app.test_client() as client: archive_filename = os.path.join(self.temp_dirname, 'archive.omex') def save_temporary_combine_archive_to_s3_bucket( filename, public=True, archive_filename=archive_filename): shutil.copy(filename, archive_filename) return archive_filename with mock.patch( 'src.s3.save_temporary_combine_archive_to_s3_bucket', side_effect=save_temporary_combine_archive_to_s3_bucket): response = client.post(endpoint, data=data, content_type="multipart/form-data") fid_0.close() fid_1.close() self.assertEqual(response.status_code, 200, response.json) self.assertEqual(response.json, archive_filename) contents_dirname = os.path.join(self.temp_dirname, 'archive') archive = CombineArchiveReader().run(archive_filename, contents_dirname) self.assertEqual(len(archive.contents), 3) for content, expected_content in zip(archive.contents, archive_specs['contents']): self.assertEqual(content.location, expected_content['location']['path']) self.assertEqual(content.format, expected_content['format']) self.assertEqual(content.master, expected_content['master']) with self.assertRaisesRegex(ValueError, 'Missing a required XML attribute'): sed_doc = SedmlSimulationReader().run( os.path.join(contents_dirname, archive_specs['contents'][2]['location']['path']), validate_models_with_languages=True) sed_doc = SedmlSimulationReader().run( os.path.join(contents_dirname, archive_specs['contents'][2]['location']['path']), validate_models_with_languages=False) sed_doc_specs = archive_specs['contents'][2]['location']['value'] self.assertEqual(sed_doc.level, sed_doc_specs['level']) self.assertEqual(sed_doc.version, sed_doc_specs['version']) self.assertEqual(sed_doc.styles[0].id, sed_doc_specs['styles'][0]['id']) self.assertEqual(sed_doc.styles[0].name, sed_doc_specs['styles'][0].get('name', None)) self.assertEqual(sed_doc.styles[0].base, sed_doc_specs['styles'][0].get('base', None)) self.assertEqual( sed_doc.styles[0].line.type.value, sed_doc_specs['styles'][0].get('line', None).get('type', None)) self.assertEqual( sed_doc.styles[0].line.color, sed_doc_specs['styles'][0].get('line', None).get('color', None)) self.assertEqual( sed_doc.styles[0].line.thickness, sed_doc_specs['styles'][0].get('line', None).get('thickness', None)) self.assertEqual( sed_doc.styles[0].marker.type.value, sed_doc_specs['styles'][0].get('marker', None).get('type', None)) self.assertEqual( sed_doc.styles[0].marker.size, sed_doc_specs['styles'][0].get('marker', None).get('size', None)) self.assertEqual( sed_doc.styles[0].marker.line_color, sed_doc_specs['styles'][0].get('marker', None).get('lineColor', None)) self.assertEqual( sed_doc.styles[0].marker.line_thickness, sed_doc_specs['styles'][0].get('marker', None).get('lineThickness', None)) self.assertEqual( sed_doc.styles[0].marker.fill_color, sed_doc_specs['styles'][0].get('marker', None).get('fillColor', None)) self.assertEqual( sed_doc.styles[0].fill.color, sed_doc_specs['styles'][0].get('fill', None).get('color', None)) self.assertEqual(sed_doc.styles[1].id, sed_doc_specs['styles'][1]['id']) self.assertEqual(sed_doc.styles[1].name, sed_doc_specs['styles'][1].get('name', None)) self.assertEqual(sed_doc.styles[1].base.id, sed_doc_specs['styles'][1].get('base', None)) self.assertEqual(sed_doc.styles[1].line, sed_doc_specs['styles'][1].get('line', None)) self.assertEqual( sed_doc.styles[1].marker.type.value, sed_doc_specs['styles'][1].get('marker', None).get('type', None)) self.assertEqual( sed_doc.styles[1].marker.size, sed_doc_specs['styles'][1].get('marker', None).get('size', None)) self.assertEqual( sed_doc.styles[1].marker.line_color, sed_doc_specs['styles'][1].get('marker', None).get('lineColor', None)) self.assertEqual( sed_doc.styles[1].marker.line_thickness, sed_doc_specs['styles'][1].get('marker', None).get('lineThickness', None)) self.assertEqual( sed_doc.styles[1].marker.fill_color, sed_doc_specs['styles'][1].get('marker', None).get('fillColor', None)) self.assertEqual( sed_doc.styles[1].fill.color, sed_doc_specs['styles'][1].get('fill', None).get('color', None)) self.assertEqual(sed_doc.outputs[1].curves[0].style.id, sed_doc_specs['outputs'][1]['curves'][0]['style']) self.assertEqual(sed_doc.outputs[2].surfaces[0].style.id, sed_doc_specs['outputs'][2]['surfaces'][0]['style']) self.assertEqual(sed_doc.tasks[0].model, sed_doc.models[0]) self.assertEqual(len(sed_doc.models[0].changes), 2) self.assertEqual( sed_doc.models[0].changes[0].target, "/sbml:sbml/sbml:model/sbml:listOfParameters/sbml:parameter[@id='k1']/@value" ) self.assertEqual(sed_doc.models[0].changes[0].new_value, '1.2') self.assertEqual( sed_doc.models[0].changes[0].target_namespaces, { None: 'http://sed-ml.org/sed-ml/level1/version4', 'sbml': 'http://www.sbml.org/sbml/level3/version1/core', 'qual': 'http://www.sbml.org/sbml/level3/version1/qual/version1', }) self.assertEqual( sed_doc.tasks[0].simulation.algorithm.changes[0].kisao_id, 'KISAO_0000488') self.assertEqual( sed_doc.tasks[0].simulation.algorithm.changes[0].new_value, '10') self.assertEqual( sed_doc.outputs[1].curves[0].x_data_generator.variables[0].target, "/sbml:sbml/sbml:model/qual:listOfQualitativeSpecies/qual:qualitativeSpecies[@qual:id='x']" ) self.assertEqual( sed_doc.outputs[1].curves[0].x_data_generator.variables[0]. target_namespaces, { None: 'http://sed-ml.org/sed-ml/level1/version4', "sbml": "http://www.sbml.org/sbml/level3/version1/core", "qual": "http://www.sbml.org/sbml/level3/version1/qual/version1" }, )
def handler(body, file=None, omexMetadataSchema=OmexMetadataSchema.biosimulations.value): ''' Get the metadata about a COMBINE/OMEX archive and its contents Args: dictionary in schema ``GetCombineArchiveMetadataFileOrUrl`` with keys * ``url`` whose value has schema ``Url`` with the URL for a COMBINE/OMEX archive * ``omexMetadataFormat` whose value is a value of :obj:`OmexMetadataInputFormat` file (:obj:`werkzeug.datastructures.FileStorage`, optional): COMBINE/OMEX archive file omexMetadataSchema (:obj:`str`, optional): schema for validating the OMEX Metadata files Returns: :obj:`list` of ``BioSimulationsCombineArchiveLocationMetadata`` or ``RdfTriple``: metadata about a COMBINE/OMEX archive and its contents ''' try: omexMetadataInputFormat = OmexMetadataInputFormat(body.get('omexMetadataFormat', 'rdfxml')) except ValueError as exception: raise BadRequestException(title='`omexMetadataFormat` must be a recognized format.', exception=exception) try: omexMetadataSchema = OmexMetadataSchema(omexMetadataSchema) except ValueError as exception: raise BadRequestException(title='`omexMetadataSchema` must be a recognized schema.', exception=exception) archive_file = file archive_url = body.get('url', None) if archive_url and archive_file: raise BadRequestException( title='Only one of `file` or `url` can be used at a time.', instance=ValueError(), ) if not archive_url and not archive_file: raise BadRequestException( title='One of `file` or `url` must be used.', instance=ValueError(), ) # create temporary working directory temp_dirname = get_temp_dir() archive_filename = os.path.join(temp_dirname, 'archive.omex') # get COMBINE/OMEX archive if archive_file: archive_file.save(archive_filename) else: try: response = requests.get(archive_url) response.raise_for_status() except requests.exceptions.RequestException as exception: title = 'COMBINE/OMEX archive could not be loaded from `{}`'.format( archive_url) raise BadRequestException( title=title, instance=exception, ) # save archive to local temporary file with open(archive_filename, 'wb') as file: file.write(response.content) # read archive archive_dirname = os.path.join(temp_dirname, 'archive') try: archive = CombineArchiveReader().run(archive_filename, archive_dirname) except Exception as exception: # return exception raise BadRequestException( title='`{}` is not a valid COMBINE/OMEX archive'.format(archive_url if archive_url else archive_file.filename), instance=exception, ) config = Config( OMEX_METADATA_INPUT_FORMAT=omexMetadataInputFormat, OMEX_METADATA_SCHEMA=omexMetadataSchema, ) metadata, errors, warnings = read_omex_meta_files_for_archive(archive, archive_dirname, config=config) shutil.rmtree(archive_dirname) if errors: raise BadRequestException( title='The metadata for the COMBINE/OMEX archive is not valid.', instance=ValueError(), validation_report=make_validation_report(errors, warnings, filenames=[archive_filename]), ) # return response return metadata
def handler(body, file=None): ''' Get the manifest of a COMBINE/OMEX archive Args: body (:obj:`dict`): dictionary with keys * ``url`` whose value has schema ``Url`` with the URL for a COMBINE/OMEX archive or a manifest for a COMBINE/OMEX archive file (:obj:`werkzeug.datastructures.FileStorage`, optional): COMBINE/OMEX archive or OMEX manifest file Returns: ``CombineArchive``: manifest of the COMBINE/OMEX archive ''' archive_or_manifest_file = file archive_or_manifest_url = body.get('url', None) if archive_or_manifest_url and archive_or_manifest_file: raise BadRequestException( title='Only one of `file` or `url` can be used at a time.', instance=ValueError(), ) if not archive_or_manifest_url and not archive_or_manifest_file: raise BadRequestException( title='One of `file` or `url` must be used.', instance=ValueError(), ) # create temporary working directory temp_dirname = get_temp_dir() archive_or_manifest_filename = os.path.join(temp_dirname, 'archive.omex') manifest_filename = os.path.join(temp_dirname, 'manifest.xml') # get COMBINE/OMEX archive or manifest if archive_or_manifest_file: archive_or_manifest_file.save(archive_or_manifest_filename) else: try: response = requests.get(archive_or_manifest_url) response.raise_for_status() except requests.exceptions.RequestException as exception: title = 'File could not be loaded from `{}`'.format( archive_or_manifest_url) raise BadRequestException( title=title, instance=exception, ) # save archive to local temporary file with open(archive_or_manifest_filename, 'wb') as file: file.write(response.content) # read archive is_archive = False try: with zipfile.ZipFile(archive_or_manifest_filename, 'r') as zip_file: is_archive = True zip_file.extract('manifest.xml', temp_dirname) except zipfile.BadZipFile: pass except KeyError as exception: raise BadRequestException( title='COMBINE/OMEX archive does not contain a manifest.', instance=exception) if not is_archive: manifest_filename = archive_or_manifest_filename reader = CombineArchiveReader() contents = reader.read_manifest(manifest_filename, archive_or_manifest_filename) if reader.errors: raise BadRequestException( title= 'File is not a valid manifest or a COMBINE/OMEX which contains a valid manifest.\n {}' .format( flatten_nested_list_of_strings(reader.errors).replace( '\n', '\n ')), instance=ValueError()) contents_specs = [] for content in contents: content_specs = { '_type': 'CombineArchiveManifestContent', 'location': { '_type': 'CombineArchiveManifestLocation', 'path': content.location, 'value': { '_type': 'CombineArchiveContentFile', 'filename': os.path.relpath(content.location, '.'), }, }, 'format': content.format, 'master': content.master, } contents_specs.append(content_specs) # format response response = {'_type': 'CombineArchiveManifest', 'contents': contents_specs} # return reponse return response