def test_save_empty_matrix(self): """Save empty Matrix to file, read it and compare the results.""" reference = _create_empty_tensor(2) print(reference) xml.save(reference, self.f) test_data = xml.load(self.f) assert np.array_equal(test_data, reference)
def _load_griddedfield(self, dim): gf = griddedfield.GriddedField(dim) gf.grids = [np.arange(2)] * dim gf.data = _create_tensor(dim) xml.save(gf, self.f) test_data = xml.load(self.f) assert np.array_equal(gf.data, test_data.data)
def calc_average_coeffs(species, outdir, **_): """Calculate averaged coefficients""" averaged_coeffs_xml_file = os.path.join(outdir, "cfc_averaged_coeffs.xml") averaged_species_xml_file = os.path.join(outdir, "cfc_averaged_species.xml") all_species = [] for s in species: rms_file = os.path.join(outdir, s, "xsec_rms.json") try: data = load_rms_data(rms_file) except FileNotFoundError: logger.warning(f"No RMS file found for species {s}, ignoring") else: all_species.extend(data) logger.info(f"Added {s}") plotfile = os.path.join(outdir, "xsec_avg_scatter.pdf") fig = plt.figure() scatter_plot_by_pressure_difference(all_species, species=",".join(species), outliers=False) plt.savefig(plotfile) plt.close(fig) logger.info(f"Wrote {plotfile}") fwhm, pressure_diff = calc_fwhm_and_pressure_difference(all_species) avg_coeffs, _, _ = do_rms_fit(fwhm, pressure_diff) axml.save(avg_coeffs, averaged_coeffs_xml_file) logger.info(f"Wrote {averaged_coeffs_xml_file}") axml.save(species, averaged_species_xml_file) logger.info(f"Wrote {averaged_species_xml_file}")
def test_save_binary_gzip(self): """Check for exception when attempting to write zipped binary file.""" f = self.f + '.gz' ref = np.arange(10) with pytest.raises(Exception): xml.save(ref, f, format='binary')
def test_save_gzip(self): """Test writing/reading of gzipped files.""" f = self.f + '.gz' ref = np.arange(10) xml.save(ref, f) assert np.array_equal(ref, xml.load(f))
def test_write_load_griddedfield(self, dim): gf = _get_griddedfield_type(dim)() gf.grids = [np.arange(2)] * dim gf.data = _create_tensor(dim) xml.save(gf, self.f) test_data = xml.load(self.f) assert np.array_equal(gf.data, test_data.data)
def _save_tensor(self, n, format='ascii'): """Save tensor of dimension n to file, read it and compare data to reference. Args: n (int): number of dimensions """ reference = _create_tensor(n) xml.save(reference, self.f, format=format) test_data = xml.load(self.f) assert np.array_equal(test_data, reference)
def test_save_empty_tensor(self, n): """Save empty tensor of dimension n to file, read it and compare data to reference. Args: n (int): number of dimensions """ reference = _create_empty_tensor(n) xml.save(reference, self.f) test_data = xml.load(self.f) assert np.array_equal(test_data, reference)
def _save_empty_tensor(self, n): """Save empty tensor of dimension n to file, read it and compare data to reference. Args: n (int): number of dimensions """ reference = _create_empty_tensor(n) xml.save(reference, self.f) test_data = xml.load(self.f) assert np.array_equal(test_data, reference)
def set_variable_by_xml(ws, variable, data): """ Set a workspace variable by writing and loading an xml file with binary data. :param ws: The Workspace :param variable: The variable (ws.something) :param data: The data, must be writeable by `typhon.arts.xml.save()`. """ if sparse.issparse(data): data = Sparse(data) with TemporaryDirectory() as path: fn = os.path.join(path, 'data.xml') xml.save(data, fn, format='binary') ws.ReadXML(variable, fn)
def test_xml_io(self): save(self.covmat, self.f) covmat2 = load(self.f) def compare_matrices(args): b1, b2 = args m1 = b1.matrix m2 = b2.matrix if isinstance(m1, sp.sparse.spmatrix): m1 = m1.todense() m2 = m2.todense() print(m1) return np.allclose(m1, m2) assert(all(map(compare_matrices, zip(self.covmat.blocks, covmat2.blocks))))
def from_typhon(self, var): """ Set the value of this WSV in the associated workspace to the given typhon type. This function writes the value in ASCII format to a temporary file and reads it into the workspace Args: var: The value to which this WSV should be set in the associated workspace. """ if not self.ws: raise Exception("Cannot set the value of a variable without " + " associated Workspace.") tmp = tempfile.NamedTemporaryFile() save(var, tmp.name) self.ws.ReadXML(self, tmp.name)
def combine_xsec(outfile): cfcs = [] for species in ( ("CFC11", "output_cfc11_full"), ("CFC12", "output_cfc12_full"), ("HFC134a", "output_hfc134a_full"), ("HCFC22", "output_hcfc22_full"), ): cfcs.extend(axml.load(os.path.join(species[1], species[0] + ".xml"))) axml.save(cfcs, outfile + ".xml", format="binary") avg_coeffs = numpy.sum(numpy.array([x.coeffs for x in cfcs]), axis=0) / len(cfcs) for x in cfcs: x.coeffs = avg_coeffs print(f"Average coeffs: {avg_coeffs}") axml.save(cfcs, outfile + ".avg.xml", format="binary")
def from_typhon(self, var): """ Set the value of this WSV in the associated workspace to the given typhon type. This function writes the value in ASCII format to a temporary file and reads it into the workspace Args: var: The value to which this WSV should be set in the associated workspace. """ from typhon.arts.xml import save if not self.ws: raise Exception("Cannot set the value of a variable without " + " associated Workspace.") with tempfile.TemporaryDirectory() as tmpdir: tfile = os.path.join(tmpdir, 'wsv.xml') save(var, tfile, format='binary') self.ws.ReadXML(self, tfile)
def from_typhon(self, val): if not self.ws: raise Exception("Variable needs associated workspace to be set from typhon object.") if not val.__class__.__name__ == self.group: raise Exception("Passed object does not match group of this variable.") name = str(uuid.uuid4()) xml_file = os.mkfifo(name + ".xml") bin_file = os.mkfifo(name + ".xml.bin") p = Process(target = self.ws.ReadXML, kwargs={'out':self, 'filename':name + ".xml"}) pr1 = Process(target = open, args=(name + ".xml",), kwargs={'mode':'rt', 'encoding':'UTF-8'}) pr2 = Process(target = open, args=(name + ".xml.bin",), kwargs={'mode':'rb'}) p.start() pr2.start() xml.save(val, name + ".xml", format='binary') os.remove(name + ".xml") os.remove(name + ".xml.bin")
def combine_data_for_arts(species, outdir, **_): # FIXME: How to handle the active flag? # active_species = {k: v for k, v in XSEC_SPECIES_INFO.items() # if k in species # and (('active' in v and v[ # 'active']) or 'active' not in v)} combined_xml_file = os.path.join(outdir, "cfc_combined.xml") all_species = [] for s in species: cfc_file = os.path.join(outdir, s, "cfc.xml") try: data = axml.load(cfc_file) except FileNotFoundError: logger.warning(f"No xml file found for species {s}, ignoring") else: all_species.append(data) logger.info(f"Added {s}") axml.save(list(itertools.chain(*all_species)), combined_xml_file, format="binary") logger.info(f"Wrote {combined_xml_file}")
def test_sparse(self, fileformat): """Save Sparse to file, read it and compare the result.""" reference = _create_sparse(10) xml.save(reference, self.f, format=fileformat) test_data = xml.load(self.f) assert np.array_equal(test_data.toarray(), reference.toarray())
def calc_broadening(species, xscdir, outdir, ignore_rms=False, rms_plots=False, averaged=False, **_): output_dir = os.path.join(outdir, species) xfi = prepare_data(xscdir, output_dir, species) if not xfi.files: logger.warning(f"No input files found for {species}.") return averaged_coeffs_xml_file = os.path.join(outdir, "cfc_averaged_coeffs.xml") if os.path.exists(averaged_coeffs_xml_file): avg_coeffs = axml.load(averaged_coeffs_xml_file) else: avg_coeffs = None # Scatter plot of available cross section data files plotfile = os.path.join(output_dir, "xsec_datasets.pdf") fig = plt.figure() plot_available_xsecs(xfi, title=species) plt.savefig(plotfile) plt.close(fig) logger.info(f"Wrote {plotfile}") rms_file = os.path.join(output_dir, "xsec_rms.json") rms_result = None if os.path.exists(rms_file) and not ignore_rms: logger.info(f"Reading precalculated RMS values form {rms_file}.") rms_result = load_rms_data(rms_file) elif not averaged: rms_result = [x for x in optimize_xsec_multi(xfi) if x] if rms_result: save_rms_data(rms_file, rms_result) logger.info(f"Wrote {rms_file}") else: logger.warning(f"No results for {species}") # Plot of best FWHM vs. pressure difference and the fit if rms_result and rms_plots: for r in rms_result: generate_rms_and_spectrum_plots(xfi, title=species, xsec_result=r, outdir=output_dir) # Load temperature fit if available try: tfit_file = os.path.join(output_dir, "xsec_tfit.json") tfit_result = load_rms_data(tfit_file) logger.info(f"Loaded temperature fit data for {species}") except FileNotFoundError: logger.info(f"No temperature fit data for {species}") tfit_result = None try: xsec_records = (gen_arts(xfi, rms_result, tfit_result, averaged_coeffs=avg_coeffs), ) except XsecError as e: logger.warning(str(e)) logger.warning(f"No RMS calculation possible for {species}") return xml_file = os.path.join(output_dir, "cfc.xml") axml.save(xsec_records, xml_file) logger.info(f"Wrote {xml_file}") plotfile = os.path.join(output_dir, "xsec_bands.pdf") fig = plt.figure() plot_xsec_records(xsec_records) plt.savefig(plotfile) plt.close(fig) logger.info(f"Wrote {plotfile}") if rms_result: plotfile = os.path.join(output_dir, "xsec_scatter.pdf") fig = plt.figure() scatter_plot_by_pressure_difference_per_band(xfi, rms_result, outliers=False) plt.savefig(plotfile) plt.close(fig) logger.info(f"Wrote {plotfile}") plotfile = os.path.join(output_dir, "xsec_scatter_temp.pdf") fig = plt.figure() scatter_plot_by_temperature(xfi, rms_result) plt.savefig(plotfile) plt.close(fig) logger.info(f"Wrote {plotfile}")
def test_save_arrayofvector(self): """Save ArrayOfIndex to file, read it and compare the results.""" reference = [np.arange(1), np.arange(1)] xml.save(reference, self.f) test_data = xml.load(self.f) assert np.array_equal(test_data, reference)
def test_xml_io(self): save(self.covmat, self.f) covmat2 = load(self.f) assert (all([ b1 == b2 for (b1, b2) in zip(self.covmat.blocks, covmat2.blocks) ]))
def test_save_arrayofindex_binary(self): """Save ArrayOfIndex to binary file, read it and compare the result.""" reference = [1., 2., 3.] xml.save(reference, self.f, format='binary') test_data = xml.load(self.f) assert np.array_equal(test_data, reference)
def test_save_arrayofstring(self): """Save ArrayOfString to file, read it and compare the results.""" reference = ['a', 'bb', 'ccc'] xml.save(reference, self.f) test_data = xml.load(self.f) assert np.array_equal(test_data, reference)
def test_save_empty_vector(self): """Save empty Vector to file, read it and compare the results.""" reference = _create_empty_tensor(1) xml.save(reference, self.f) test_data = xml.load(self.f) assert np.array_equal(test_data, reference)
def test_save_vector_binary(self): """Save Vector to binary file, read it and compare the results.""" reference = _create_tensor(1) xml.save(reference, self.f, format='binary') test_data = xml.load(self.f) assert np.array_equal(test_data, reference)
def test_save_index(self): """Save Index to file, read it and compare the results.""" reference = 0 xml.save(reference, self.f) test_data = xml.load(self.f) assert test_data == reference
def test_save_complex_matrix_binary(self): """Save complex Matrix to file, read it and compare the results.""" reference = _create_complex_tensor(2) xml.save(reference, self.f, format='binary') test_data = xml.load(self.f) assert np.array_equal(test_data, reference)