def test_save_empty_matrix(self):
"""Save empty Matrix to file, read it and compare the results."""
reference = _create_empty_tensor(2)
print(reference)
xml.save(reference, self.f)
test_data = xml.load(self.f)
assert np.array_equal(test_data, reference)
def _load_griddedfield(self, dim):
gf = griddedfield.GriddedField(dim)
gf.grids = [np.arange(2)] * dim
gf.data = _create_tensor(dim)
xml.save(gf, self.f)
test_data = xml.load(self.f)
assert np.array_equal(gf.data, test_data.data)
def calc_average_coeffs(species, outdir, **_):
"""Calculate averaged coefficients"""
averaged_coeffs_xml_file = os.path.join(outdir, "cfc_averaged_coeffs.xml")
averaged_species_xml_file = os.path.join(outdir,
"cfc_averaged_species.xml")
all_species = []
for s in species:
rms_file = os.path.join(outdir, s, "xsec_rms.json")
try:
data = load_rms_data(rms_file)
except FileNotFoundError:
logger.warning(f"No RMS file found for species {s}, ignoring")
else:
all_species.extend(data)
logger.info(f"Added {s}")
plotfile = os.path.join(outdir, "xsec_avg_scatter.pdf")
fig = plt.figure()
scatter_plot_by_pressure_difference(all_species,
species=",".join(species),
outliers=False)
plt.savefig(plotfile)
plt.close(fig)
logger.info(f"Wrote {plotfile}")
fwhm, pressure_diff = calc_fwhm_and_pressure_difference(all_species)
avg_coeffs, _, _ = do_rms_fit(fwhm, pressure_diff)
axml.save(avg_coeffs, averaged_coeffs_xml_file)
logger.info(f"Wrote {averaged_coeffs_xml_file}")
axml.save(species, averaged_species_xml_file)
logger.info(f"Wrote {averaged_species_xml_file}")
def test_save_empty_matrix(self):
"""Save empty Matrix to file, read it and compare the results."""
reference = _create_empty_tensor(2)
print(reference)
xml.save(reference, self.f)
test_data = xml.load(self.f)
assert np.array_equal(test_data, reference)
def _load_griddedfield(self, dim):
gf = griddedfield.GriddedField(dim)
gf.grids = [np.arange(2)] * dim
gf.data = _create_tensor(dim)
xml.save(gf, self.f)
test_data = xml.load(self.f)
assert np.array_equal(gf.data, test_data.data)
def test_save_binary_gzip(self):
"""Check for exception when attempting to write zipped binary file."""
f = self.f + '.gz'
ref = np.arange(10)
with pytest.raises(Exception):
xml.save(ref, f, format='binary')
def test_save_gzip(self):
"""Test writing/reading of gzipped files."""
f = self.f + '.gz'
ref = np.arange(10)
xml.save(ref, f)
assert np.array_equal(ref, xml.load(f))
def test_write_load_griddedfield(self, dim):
gf = _get_griddedfield_type(dim)()
gf.grids = [np.arange(2)] * dim
gf.data = _create_tensor(dim)
xml.save(gf, self.f)
test_data = xml.load(self.f)
assert np.array_equal(gf.data, test_data.data)
def _save_tensor(self, n, format='ascii'):
"""Save tensor of dimension n to file, read it and compare data to
reference.
Args:
n (int): number of dimensions
"""
reference = _create_tensor(n)
xml.save(reference, self.f, format=format)
test_data = xml.load(self.f)
assert np.array_equal(test_data, reference)
def test_save_empty_tensor(self, n):
"""Save empty tensor of dimension n to file, read it and compare data
to reference.
Args:
n (int): number of dimensions
"""
reference = _create_empty_tensor(n)
xml.save(reference, self.f)
test_data = xml.load(self.f)
assert np.array_equal(test_data, reference)
def _save_empty_tensor(self, n):
"""Save empty tensor of dimension n to file, read it and compare data to
reference.
Args:
n (int): number of dimensions
"""
reference = _create_empty_tensor(n)
xml.save(reference, self.f)
test_data = xml.load(self.f)
assert np.array_equal(test_data, reference)
def set_variable_by_xml(ws, variable, data):
"""
Set a workspace variable by writing and loading an xml file with binary data.
:param ws: The Workspace
:param variable: The variable (ws.something)
:param data: The data, must be writeable by `typhon.arts.xml.save()`.
"""
if sparse.issparse(data):
data = Sparse(data)
with TemporaryDirectory() as path:
fn = os.path.join(path, 'data.xml')
xml.save(data, fn, format='binary')
ws.ReadXML(variable, fn)
def test_xml_io(self):
save(self.covmat, self.f)
covmat2 = load(self.f)
def compare_matrices(args):
b1, b2 = args
m1 = b1.matrix
m2 = b2.matrix
if isinstance(m1, sp.sparse.spmatrix):
m1 = m1.todense()
m2 = m2.todense()
print(m1)
return np.allclose(m1, m2)
assert(all(map(compare_matrices, zip(self.covmat.blocks, covmat2.blocks))))
def from_typhon(self, var):
"""
Set the value of this WSV in the associated workspace to the given
typhon type. This function writes the value in ASCII format to a
temporary file and reads it into the workspace
Args:
var: The value to which this WSV should be set in the associated
workspace.
"""
if not self.ws:
raise Exception("Cannot set the value of a variable without "
+ " associated Workspace.")
tmp = tempfile.NamedTemporaryFile()
save(var, tmp.name)
self.ws.ReadXML(self, tmp.name)
def combine_xsec(outfile):
cfcs = []
for species in (
("CFC11", "output_cfc11_full"),
("CFC12", "output_cfc12_full"),
("HFC134a", "output_hfc134a_full"),
("HCFC22", "output_hcfc22_full"),
):
cfcs.extend(axml.load(os.path.join(species[1], species[0] + ".xml")))
axml.save(cfcs, outfile + ".xml", format="binary")
avg_coeffs = numpy.sum(numpy.array([x.coeffs for x in cfcs]), axis=0) / len(cfcs)
for x in cfcs:
x.coeffs = avg_coeffs
print(f"Average coeffs: {avg_coeffs}")
axml.save(cfcs, outfile + ".avg.xml", format="binary")
def from_typhon(self, var):
"""
Set the value of this WSV in the associated workspace to the given
typhon type. This function writes the value in ASCII format to a
temporary file and reads it into the workspace
Args:
var: The value to which this WSV should be set in the associated
workspace.
"""
from typhon.arts.xml import save
if not self.ws:
raise Exception("Cannot set the value of a variable without " +
" associated Workspace.")
with tempfile.TemporaryDirectory() as tmpdir:
tfile = os.path.join(tmpdir, 'wsv.xml')
save(var, tfile, format='binary')
self.ws.ReadXML(self, tfile)
def from_typhon(self, val):
if not self.ws:
raise Exception("Variable needs associated workspace to be set from typhon object.")
if not val.__class__.__name__ == self.group:
raise Exception("Passed object does not match group of this variable.")
name = str(uuid.uuid4())
xml_file = os.mkfifo(name + ".xml")
bin_file = os.mkfifo(name + ".xml.bin")
p = Process(target = self.ws.ReadXML, kwargs={'out':self, 'filename':name + ".xml"})
pr1 = Process(target = open, args=(name + ".xml",), kwargs={'mode':'rt', 'encoding':'UTF-8'})
pr2 = Process(target = open, args=(name + ".xml.bin",), kwargs={'mode':'rb'})
p.start()
pr2.start()
xml.save(val, name + ".xml", format='binary')
os.remove(name + ".xml")
os.remove(name + ".xml.bin")
def combine_data_for_arts(species, outdir, **_):
# FIXME: How to handle the active flag?
# active_species = {k: v for k, v in XSEC_SPECIES_INFO.items()
# if k in species
# and (('active' in v and v[
# 'active']) or 'active' not in v)}
combined_xml_file = os.path.join(outdir, "cfc_combined.xml")
all_species = []
for s in species:
cfc_file = os.path.join(outdir, s, "cfc.xml")
try:
data = axml.load(cfc_file)
except FileNotFoundError:
logger.warning(f"No xml file found for species {s}, ignoring")
else:
all_species.append(data)
logger.info(f"Added {s}")
axml.save(list(itertools.chain(*all_species)),
combined_xml_file,
format="binary")
logger.info(f"Wrote {combined_xml_file}")
def test_sparse(self, fileformat):
"""Save Sparse to file, read it and compare the result."""
reference = _create_sparse(10)
xml.save(reference, self.f, format=fileformat)
test_data = xml.load(self.f)
assert np.array_equal(test_data.toarray(), reference.toarray())
def calc_broadening(species,
xscdir,
outdir,
ignore_rms=False,
rms_plots=False,
averaged=False,
**_):
output_dir = os.path.join(outdir, species)
xfi = prepare_data(xscdir, output_dir, species)
if not xfi.files:
logger.warning(f"No input files found for {species}.")
return
averaged_coeffs_xml_file = os.path.join(outdir, "cfc_averaged_coeffs.xml")
if os.path.exists(averaged_coeffs_xml_file):
avg_coeffs = axml.load(averaged_coeffs_xml_file)
else:
avg_coeffs = None
# Scatter plot of available cross section data files
plotfile = os.path.join(output_dir, "xsec_datasets.pdf")
fig = plt.figure()
plot_available_xsecs(xfi, title=species)
plt.savefig(plotfile)
plt.close(fig)
logger.info(f"Wrote {plotfile}")
rms_file = os.path.join(output_dir, "xsec_rms.json")
rms_result = None
if os.path.exists(rms_file) and not ignore_rms:
logger.info(f"Reading precalculated RMS values form {rms_file}.")
rms_result = load_rms_data(rms_file)
elif not averaged:
rms_result = [x for x in optimize_xsec_multi(xfi) if x]
if rms_result:
save_rms_data(rms_file, rms_result)
logger.info(f"Wrote {rms_file}")
else:
logger.warning(f"No results for {species}")
# Plot of best FWHM vs. pressure difference and the fit
if rms_result and rms_plots:
for r in rms_result:
generate_rms_and_spectrum_plots(xfi,
title=species,
xsec_result=r,
outdir=output_dir)
# Load temperature fit if available
try:
tfit_file = os.path.join(output_dir, "xsec_tfit.json")
tfit_result = load_rms_data(tfit_file)
logger.info(f"Loaded temperature fit data for {species}")
except FileNotFoundError:
logger.info(f"No temperature fit data for {species}")
tfit_result = None
try:
xsec_records = (gen_arts(xfi,
rms_result,
tfit_result,
averaged_coeffs=avg_coeffs), )
except XsecError as e:
logger.warning(str(e))
logger.warning(f"No RMS calculation possible for {species}")
return
xml_file = os.path.join(output_dir, "cfc.xml")
axml.save(xsec_records, xml_file)
logger.info(f"Wrote {xml_file}")
plotfile = os.path.join(output_dir, "xsec_bands.pdf")
fig = plt.figure()
plot_xsec_records(xsec_records)
plt.savefig(plotfile)
plt.close(fig)
logger.info(f"Wrote {plotfile}")
if rms_result:
plotfile = os.path.join(output_dir, "xsec_scatter.pdf")
fig = plt.figure()
scatter_plot_by_pressure_difference_per_band(xfi,
rms_result,
outliers=False)
plt.savefig(plotfile)
plt.close(fig)
logger.info(f"Wrote {plotfile}")
plotfile = os.path.join(output_dir, "xsec_scatter_temp.pdf")
fig = plt.figure()
scatter_plot_by_temperature(xfi, rms_result)
plt.savefig(plotfile)
plt.close(fig)
logger.info(f"Wrote {plotfile}")
def test_save_arrayofvector(self):
"""Save ArrayOfIndex to file, read it and compare the results."""
reference = [np.arange(1), np.arange(1)]
xml.save(reference, self.f)
test_data = xml.load(self.f)
assert np.array_equal(test_data, reference)
def test_xml_io(self):
save(self.covmat, self.f)
covmat2 = load(self.f)
assert (all([
b1 == b2 for (b1, b2) in zip(self.covmat.blocks, covmat2.blocks)
]))
def test_save_arrayofindex_binary(self):
"""Save ArrayOfIndex to binary file, read it and compare the result."""
reference = [1., 2., 3.]
xml.save(reference, self.f, format='binary')
test_data = xml.load(self.f)
assert np.array_equal(test_data, reference)
def test_save_arrayofstring(self):
"""Save ArrayOfString to file, read it and compare the results."""
reference = ['a', 'bb', 'ccc']
xml.save(reference, self.f)
test_data = xml.load(self.f)
assert np.array_equal(test_data, reference)
def test_save_arrayofindex_binary(self):
"""Save ArrayOfIndex to binary file, read it and compare the result."""
reference = [1., 2., 3.]
xml.save(reference, self.f, format='binary')
test_data = xml.load(self.f)
assert np.array_equal(test_data, reference)
def test_save_arrayofvector(self):
"""Save ArrayOfIndex to file, read it and compare the results."""
reference = [np.arange(1), np.arange(1)]
xml.save(reference, self.f)
test_data = xml.load(self.f)
assert np.array_equal(test_data, reference)
def test_save_empty_vector(self):
"""Save empty Vector to file, read it and compare the results."""
reference = _create_empty_tensor(1)
xml.save(reference, self.f)
test_data = xml.load(self.f)
assert np.array_equal(test_data, reference)
def test_save_vector_binary(self):
"""Save Vector to binary file, read it and compare the results."""
reference = _create_tensor(1)
xml.save(reference, self.f, format='binary')
test_data = xml.load(self.f)
assert np.array_equal(test_data, reference)
def test_save_index(self):
"""Save Index to file, read it and compare the results."""
reference = 0
xml.save(reference, self.f)
test_data = xml.load(self.f)
assert test_data == reference
def test_save_vector_binary(self):
"""Save Vector to binary file, read it and compare the results."""
reference = _create_tensor(1)
xml.save(reference, self.f, format='binary')
test_data = xml.load(self.f)
assert np.array_equal(test_data, reference)
def test_save_index(self):
"""Save Index to file, read it and compare the results."""
reference = 0
xml.save(reference, self.f)
test_data = xml.load(self.f)
assert test_data == reference
def test_save_complex_matrix_binary(self):
"""Save complex Matrix to file, read it and compare the results."""
reference = _create_complex_tensor(2)
xml.save(reference, self.f, format='binary')
test_data = xml.load(self.f)
assert np.array_equal(test_data, reference)
def test_save_arrayofstring(self):
"""Save ArrayOfString to file, read it and compare the results."""
reference = ['a', 'bb', 'ccc']
xml.save(reference, self.f)
test_data = xml.load(self.f)
assert np.array_equal(test_data, reference)
