def test_add(self):
d = Dataset.new('FLYSPEC')
d1 = Dataset.open(os.path.join(self.data_dir,
'2012_02_29_1340_CHILE.txt'),
format='FLYSPEC')
d += d1
r = d.retrievals[3]
s1 = r.spectra_id.get_referred_object()
angle = s1.angle[r.slice]
id_max = np.argmax(r.sca)
np.testing.assert_almost_equal(angle[id_max], 168.04, 2)
self.assertEqual(len(d.retrievals), 36)
d1 = Dataset.open(os.path.join(self.data_dir,
'2016_06_11_0830_TOFP04.txt'),
format='FLYSPEC', timeshift=12.0)
d2 = Dataset.open(os.path.join(self.data_dir,
'2016_06_11_0900_TOFP04.txt'),
format='FLYSPEC', timeshift=12.0)
d3 = d1 + d2
self.assertEqual(len(d3.retrievals), 25)
d0 = Dataset.new('FLYSPEC')
d0 += d1
d0 += d2
self.assertEqual(len(d0.retrievals), 25)
def test_open(self):
d = Dataset.open(os.path.join(self.data_dir,
'2012_02_29_1340_CHILE.txt'),
format='FLYSPEC')
s = d.spectra[0]
self.assertEqual(s.time.shape, (4600,))
self.assertEqual(s.angle[0], 135.140)
r = d.retrievals[3]
s1 = r.spectra_id.get_referred_object()
angle = s1.angle[r.slice]
id_max = np.argmax(r.sca)
np.testing.assert_almost_equal(angle[id_max], 168.04, 2)
self.assertEqual(len(d.retrievals), 36)
np.testing.assert_array_almost_equal(s1.position[0, :],
[-67.8047, -23.3565, 3927.], 2)
# dicretize all retrievals onto a grid to show a daily plot
bins = np.arange(0, 180, 1.0)
nretrieval = len(d.retrievals)
m = np.zeros((nretrieval, bins.size - 1))
for i, _r in enumerate(d.retrievals):
_s = _r.spectra_id.get_referred_object()
_angle = _s.angle[_r.slice]
_so2 = _r.sca
_so2_binned = binned_statistic(_angle, _so2, 'mean', bins)
m[i, :] = _so2_binned.statistic
ids = np.argmax(np.ma.masked_invalid(m), axis=1)
maxima = np.array([166., 167., 167., 167., 168., 167., 168., 167.,
167., 167., 167., 167., 168., 167., 167., 167.,
167., 166., 167., 166., 166., 167., 165., 165.,
165., 164., 165., 163., 163., 164., 163., 165.,
164., 164., 164., 161.])
np.testing.assert_array_almost_equal(maxima, bins[ids], 2)
d1 = Dataset.open(os.path.join(self.data_dir,
'2016_06_11_0830_TOFP04.txt'),
format='FLYSPEC', timeshift=12.0)
nretrieval = len(d1.retrievals)
m = np.zeros((nretrieval, bins.size - 1))
for i, _r in enumerate(d1.retrievals):
_s = _r.spectra_id.get_referred_object()
_angle = _s.angle[_r.slice]
_so2 = _r.sca
_so2_binned = binned_statistic(_angle, _so2, 'mean', bins)
m[i, :] = _so2_binned.statistic
ids = np.argmax(np.ma.masked_invalid(m), axis=1)
maxima = np.array([147., 25., 27., 86., 29., 31., 27., 27., 28., 137.,
34., 34.])
np.testing.assert_array_almost_equal(maxima, bins[ids], 2)
def verify_flux(filename, perror_thresh=0.5):
"""
Verify that the recomputed flux values are close to the
flux values stored in FITS.
"""
d = Dataset.open(filename)
pf_ah = None
pf_ch = None
for instrument in ['WI301', 'WI302']:
for _pf in d.elements['PreferredFlux']:
gfm = _pf.fluxes[0].gasflow.methods[0].name[:]
sid = _pf.fluxes[0].concentration.rawdata[0].instrument.name[:]
if gfm == 'WS2PV' and sid == instrument:
pf_ah = _pf
elif gfm == 'WS2PVT' and sid == instrument:
pf_ch = _pf
try:
if pf_ah is not None:
flux_ah(pf_ah, perror_thresh=perror_thresh)
if pf_ch is not None:
flux_ch(pf_ch, pf_ah, perror_thresh=perror_thresh)
except MDOASException as e:
msg = str(e)
msg += "StationID: {}\n".format(instrument)
msg += "File: {}\n".format(filename)
raise MDOASException(msg)
d.close()
def test_plot(self):
import matplotlib.image
d = Dataset.open(os.path.join(self.data_dir,
'2012_02_29_1340_CHILE.txt'),
format='FLYSPEC', timeshift=12.0)
with tempfile.TemporaryFile() as fd:
d.plot(savefig=fd, timeshift=12.0)
expected_image = matplotlib.image.imread(
os.path.join(self.data_dir, 'chile_retrievals_overview.png'),
format='png')
fd.seek(0)
actual_image = matplotlib.image.imread(fd, format='png')
# Set the "color" of fully transparent pixels to white. This avoids
# the issue of different "colors" for transparent pixels.
expected_image[expected_image[..., 3] <= 0.0035] = \
[1.0, 1.0, 1.0, 0.0]
actual_image[actual_image[..., 3] <= 0.0035] = \
[1.0, 1.0, 1.0, 0.0]
# This deviates a bit from the matplotlib version and just
# calculates the root mean square error of all pixel values without
# any other fancy considerations. It also uses the alpha channel of
# the images. Scaled by 255.
rms = np.sqrt(
np.sum((255.0 * (expected_image - actual_image)) ** 2) /
float(expected_image.size))
self.assertTrue(rms <= 0.001)
def test_read(self):
"""
Test reading of HDF5 files.
"""
fn = tempfile.mktemp()
d = Dataset(fn, 'w')
tb = TargetBuffer(tags=['WI001'], name='White Island main vent',
position=(177.2, -37.5, 50),
position_error=(0.2, 0.2, 20),
description='Main vent in January 2017')
d.register_tags(['WI001', 'MD01', 'measurement'])
t = d.new(tb)
ib = InstrumentBuffer(tags=['MD01'], sensor_id='F00975',
location='West rim',
no_bits=16, type='DOAS',
description='GeoNet permanent instrument')
i = d.new(ib)
rdtb = RawDataTypeBuffer(tags=['measurement'],
name='1st round measurements',
acquisition='stationary')
rdt = d.new(rdtb)
rb = RawDataBuffer(target=t, instrument=i, type=rdt,
d_var=np.zeros((1, 2048)), ind_var=np.arange(2048),
datetime=['2017-01-10T15:23:00'])
d.new(rb)
d.close()
d1 = Dataset.open(fn)
r1 = d1.elements['RawData'][0]
self.assertEqual(r1.target.name, 'White Island main vent')
self.assertEqual(list(r1.instrument.tags)[0], 'MD01')
def test_not_enough_data(self):
with self.assertRaises(FlySpecPluginException):
d1 = Dataset.open(os.path.join(self.data_dir,
'2015_05_03_1630_TOFP04.txt'),
format='FLYSPEC', timeshift=12.0)