load_dotenv(dotenv_path='./.env')
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.colors import Normalize
from typhon.arts.workspace import Workspace, arts_agenda
ws = Workspace()
ws.execute_controlfile("general/general.arts")
ws.execute_controlfile("general/agendas.arts")
ws.execute_controlfile("general/continua.arts")
ws.execute_controlfile("general/planet_earth.arts")
ws.Copy(ws.abs_xsec_agenda, ws.abs_xsec_agenda__noCIA)
ws.Copy(ws.ppath_agenda, ws.ppath_agenda__FollowSensorLosPath)
ws.Copy(ws.ppath_step_agenda, ws.ppath_step_agenda__GeometricPath)
ws.Copy(ws.iy_space_agenda, ws.iy_space_agenda__CosmicBackground)
ws.Copy(ws.iy_surface_agenda, ws.iy_surface_agenda__UseSurfaceRtprop)
ws.Copy(ws.iy_main_agenda, ws.iy_main_agenda__Emission)
ws.Copy(ws.propmat_clearsky_agenda, ws.propmat_clearsky_agenda__OnTheFly)
# General Settings
# For the wind retrievals, the forward model calculations are performed on a 3D atmosphere grid.
# Radiation is assumed to be unpolarized.
ws.atmosphere_dim = 3
ws.stokes_dim = 1
ws.iy_unit = "RJBT"
# Absorption
class oem_retrieval():
"""
AUTHOR:
Hayden Smotherman
DESCRIPTION:
This class is meant to run an OEM retrieval on data from the radiometers at MPI Solar System.
The hope is that it simplifies the process of using the typhon OEM retreival for the specific
use-case of Paul Hartogh's group at MPI Solar System.
It can load in the data, attempt to fit and subtract-out sine curves from the noise,
and filter high-frequency oscillations that are present in some data.
It can also run a retrieval using the typhon OEM retrieval and plot the results,
along with relevant statistical metrics.
"""
def __init__(self):
"""
AUTHOR:
Hayden Smotherman
DESCRIPTION:
This function initializes the oem_retrieval class
INPUTS:
NONE
OUTPUTS:
NONE
"""
self.signal = np.array(
[]) # This array will hold the brightness temperature data
self.noise = np.array(
[]) # This array will hold the noise for the radiometer
self.freq = np.array(
[]) # This array will hold the frequencies of "signal" and "noise"
def load_data(self,
data_type='radiometer',
use_data=0,
filename='',
signal=[],
noise=[],
freq=[]):
"""
AUTHOR:
Hayden Smotherman
DESCRIPTION:
This function loads in radiometer data to the oem_retrival class.
It accepts data in a number of different formats.
INPUTS:
data_type - A string denoting which format the input data will be in.
VALUES: 'numpy' - Regular numpy arrays passed in to "signal", "noise", and "freq"
REQUIRES: signal, noise, freq
'radiometer' - A .raw file generated by a radiometer at MPI Solar System.
REQUIRES: filename
use_data - Integer value denoting how much data to use when running a mean OEM retrieval.
This value should be zero (use all data) or negative (use the last x signals)
signal - An array of the brightness temperatures of an O3-666 line.
ONLY USED WITH data_type='numpy'
noise - An array of the noise of the radiometer
ONLY USED WITH data_type='numpy'
freq - An array of the central frequency of each datapoint in "signal" and "noise"
ONLY USED WITH data_type='numpy'
OUTPUTS:
"""
eform = '>i16f7500f100f28f28f'
dform = '>i16f4092f100f28f28f'
aform = '>i16f16384f128f28f28f'
xform = '>i16f1019f128f28f28f'
if data_type == 'radiometer':
# Load data from a .raw file from the 210MHz radiometer
# Load in the raw data
if filename[0] == 'a':
data_format = aform
elif filename[0] == 'd':
data_format = dform
elif filename[0] == 'e':
data_format = eform
elif filename[0] == 'x':
data_format = xform
else:
raise ImportError(
'Could not understand the data format based on the file name.'
)
Calibrated = calibration(format=data_format)
Calibrated.load(filename)
Calibrated.calibrate()
# Pre-allocate signal and noise arrays
self.signal = np.array(Calibrated._signal[use_data:])
self.noise = np.array(Calibrated._noise[use_data:])
# Generate the frequency array based on the data format
if filename[0] == 'a':
# First we split the data, since two signals are embedded in the ._signal array
# NOTE: Currently this only keeps the first half of the data. It should keep all of it.
self.signal = Calibrated._signal[use_data:]
self.noise = Calibrated._noise[use_data:]
Half_Length = int(len(Calibrated._noise[0]) /
2) # Data is set as one long array
for i in range(len(self.signal)):
self.signal[i] = np.copy(
Calibrated._signal[i][0:Half_Length])
self.noise[i] = np.copy(
Calibrated._noise[i][0:Half_Length])
self.signal = np.array(self.signal)
self.noise = np.array(self.noise)
# Width of data is 1.5GHz
# Central frequency is data number 14 in _raw array, requested frequency
Min_Freq = Calibrated._raw[0][14] - 1.50 / 2
Max_Freq = Calibrated._raw[0][14] + 1.50 / 2
self.freq = np.linspace(Min_Freq, Max_Freq, Half_Length)
elif filename[0] == 'd':
# Width of data is 40MHz
# Central frequency is data number 14 in _raw array, requested frequency
Min_Freq = Calibrated._raw[0][14] - .040 / 2
Max_Freq = Calibrated._raw[0][14] + .040 / 2
self.freq = np.linspace(Min_Freq, Max_Freq,
len(Calibrated._noise[0]))
elif filename[0] == 'e':
# Width of data is 210MHz
# Central frequency is data number 14 in _raw array, requested frequency
Min_Freq = Calibrated._raw[0][14] - .210 / 2
Max_Freq = Calibrated._raw[0][14] + .210 / 2
self.freq = np.linspace(Min_Freq, Max_Freq,
len(Calibrated._noise[0]))
elif filename[0] == 'x':
# Width of data is 4.4GHz
# Central frequency is data number 14 in _raw array, requested frequency
Min_Freq = Calibrated._raw[0][14] - 4.4 / 2
Max_Freq = Calibrated._raw[0][14] + 4.4 / 2
self.freq = np.linspace(Min_Freq, Max_Freq,
len(Calibrated._noise[0]))
else:
raise ImportError(
'Could not generate frequency array. Unknown file format.')
Calibrated = None
elif data_type == 'numpy':
# Load in arbitrary data that is already in a numpy array
self.signal = np.array(signal)
self.noise = np.array(noise)
self.freq = np.array(freq)
# Define fit_ values here in case subtract_sinewave is never called
self.fit_freq = np.copy(self.freq)
self.fit_signal = np.copy(self.signal)
self.fit_noise = np.copy(self.noise)
def process_data(self,
lims=[21, 23],
shift_freq=False,
fit_sine=False,
plot_example=True,
initial_guess=[3, 1 / 1000, 0],
central_freq=1.42175037e+2):
"""
AUTHOR:
Hayden Smotherman
DESCRIPTION:
This function clips data from the front and back of the dataset based on front_lim and
back_lim. If "fit_sine=True", then it also subtracts out sine waves from the data.
INPUTS:
lims - Optional frequency limits (in GHz) that determine the range of data to use.
shift_freq - Boolean that determines whether or not to shift the frequency such that the
max of the averaged data aligns with the peak of the O3 signal.
fit_sine - Boolean that determines whether or not to fit a sine wave to the noise then
subtract it out of the signal.
plot_example - Boolean that determines whether or not to plot an example fit. Only used if
fit_sine=True
initial_guess - This is the initial guess for the sine parameters used by lsqfit. Only used
if fit_sine=True
central_freq - This is the frequency center of the measurements. Only used if
shift_freq=True
OUTPUTS:
NONE
NOTES:
- This function works best for data in dform and eform. Datasets with longer frequency
baselines are not fit as well and may give bad results.
- By default, "process_data" clips the first 25 data points regardless of what is set in "lims".
This is done because the first ~20 data points are often severe outliers.
"""
# Calculate the index corresponding to the minimum frequency limit
if shift_freq:
if len(self.signal.shape) > 1:
Average_Signal = np.mean(self.signal, axis=0)
else:
Average_Signal = np.copy(self.signal)
Freq_Peak = self.freq[Average_Signal == np.max(Average_Signal)]
Freq_Difference = Freq_Peak - central_freq
self.freq -= Freq_Difference
front_lim = np.where(
self.freq == np.min(self.freq[self.freq > lims[0]]))
front_lim = int(front_lim[0])
if (front_lim < 25):
# Set front_lim minimum to 25, since the first ~25 data points are very often bad
front_lim = 25
# Calculate the index corresponding to the maximum frequency limit
back_lim = np.where(
self.freq == np.max(self.freq[self.freq < lims[1]]))
back_lim = int(back_lim[0])
Signal_Length = self.signal.shape[1] # Length of the signal array
Signal_Number = self.signal.shape[
0] # Number of signals in the data file
X_Data = np.linspace(
0, Signal_Length - 1,
Signal_Length) # Array of indicies for the fitting
X_Data = X_Data[front_lim:back_lim]
# Preallocate the memory for the fitted values of signal and noise
self.fit_signal = np.zeros([
self.signal.shape[0],
np.size(self.signal[0][front_lim:back_lim])
])
self.fit_noise = np.zeros(
[self.noise.shape[0],
np.size(self.noise[0][front_lim:back_lim])])
self.fit_freq = np.copy(self.freq[front_lim:back_lim])
if fit_sine:
for i in range(Signal_Number):
# Iterate over all signals and subtract out the best-fit sine curve
# Load in noise locally for ease of use
Noise_0 = self.noise[i]
# Limit the data based on front_lim and back_lim and subtract the mean
Noise_0 = Noise_0[front_lim:back_lim]
Noise_Median = np.median(Noise_0)
Base_Noise = Noise_0 - Noise_Median # Subtract out the noise mean for fitting
Sine_Params = initial_guess # Amplitude, freq, phase
# Sine function
optimize_func = lambda x: x[0] * np.sin(x[1] * X_Data + x[2]
) - Base_Noise
# Perform least square fit to the sine function
est_amp, est_freq, est_phase = leastsq(optimize_func,
Sine_Params)[0]
# Save the fit
Fit = est_amp * np.sin(est_freq * X_Data + est_phase)
# Load in signal locally
Signal_0 = self.signal[i]
Signal_0 = Signal_0[front_lim:back_lim]
# Subtract the fit from the signal and the noise and save it in new variables
#print(np.size(Signal_0), np.size(Fit))
self.fit_signal[i] = Signal_0 - Fit
self.fit_noise[i] = Noise_0 - Fit
if plot_example:
# Plot the last "noise" value along with the fit sine curve
plt.figure(figsize=[12, 8])
plt.plot(self.fit_freq, Noise_0)
plt.plot(self.fit_freq, Fit + Noise_Median)
plt.legend(['Unfitted Noise', 'Best Fit Sine Curve'],
fontsize=20)
plt.xlabel('Frequency [GHz]', fontsize=20)
plt.ylabel('Brightness Temperature [K]', fontsize=20)
plt.title('Raw Noise and Best Fit Curve', fontsize=24)
# Plot the fitted noise over the raw noise
plt.figure(figsize=[12, 8])
plt.plot(self.fit_freq, self.noise[-1][front_lim:back_lim])
plt.plot(self.fit_freq, self.fit_noise[-1])
plt.legend(['Unfitted Noise', 'Fitted Noise'], fontsize=20)
plt.xlabel('Frequency [GHz]', fontsize=20)
plt.ylabel('Brightness Temperature [K]', fontsize=20)
plt.title('Unfitted Noise and Fitted Noise', fontsize=24)
# Plot the fitted signal over the raw signal
plt.figure(figsize=[12, 8])
plt.plot(self.fit_freq, self.signal[-1][front_lim:back_lim])
plt.plot(self.fit_freq, self.fit_signal[-1])
plt.legend(['Unfitted Signal', 'Fitted Signal'], fontsize=20)
plt.xlabel('Frequency [GHz]', fontsize=20)
plt.ylabel('Brightness Temperature [K]', fontsize=20)
plt.title('Unfitted Signal and Fitted Signal', fontsize=24)
else:
for i in range(Signal_Number):
self.fit_signal[i] = self.signal[i][front_lim:back_lim]
self.fit_noise[i] = self.noise[i][front_lim:back_lim]
def mean_retrieval(self,
use_data=0,
filtered=False,
shift_freq=False,
sigma=None,
central_freq=1.42175037e+2):
"""
AUTHOR:
Hayden Smotherman
DESCRIPTION:
This function uses the Typhon OEM retrieval package to recover atmospheric O3 levels
given the brightness temperature signal of the O3-666 line. If self.signal is a 2D
matrix, then this function will run the retrieval on the mean of this data.
INPUTS:
filtered - Boolean that determines whether or not to use the scipy.signal.filtfilt
function on the mean signal value in order to potentially improve the signal
shift_freq - Boolean that determines whether or not to shift the frequency such that the
max of the averaged data aligns with the peak of the O3 signal.
central_freq - This is the frequency center of the measurements. Only used if
shift_freq=True
OUTPUTS:
NONE
"""
if len(self.signal.shape) > 1:
self.average_signal = np.mean(self.fit_signal, axis=0)
self.average_noise = np.mean(self.fit_noise, axis=0)
else:
self.average_signal = np.copy(self.fit_signal)
self.average_noise = np.copy(self.fit_noise)
if filtered:
n = 4 # the larger n is, the smoother the curve will be
b = [1.0 / n] * n
a = 1
self.average_signal = sg.filtfilt(b, a, self.average_signal)
self.average_noise = sg.filtfilt(b, a, self.average_noise)
if shift_freq:
Freq_Peak = self.fit_freq[self.average_signal == np.max(
self.average_signal)]
Freq_Difference = Freq_Peak - central_freq
self.fit_freq -= Freq_Difference
self._initialize_arts_workspace()
if sigma is None:
self.sigma = np.sqrt(
np.sum(
np.abs(self.average_noise - np.mean(self.average_noise))) /
len(self.average_noise))
else:
self.sigma = sigma
self._initialize_covmat()
self._run_retrieval()
def plot_oem(self, basename='', plot_results=True, plot_statistics=True):
"""
AUTHOR:
Hayden Smotherman
DESCRIPTION:
This function uses the Typhon OEM retrieval package to recover atmospheric O3 levels
given the brightness temperature signal of the O3-666 line. If self.signal is a 2D
matrix, then this function will run the retrieval on the mean of this data.
INPUTS:
plot_results - Boolean that determines whether or not to plot the results of the retrieval.
plot_statistics - Boolean that determines whether or not to plot relevant statistics
from the retrieval.
OUTPUTS:
NONE
"""
if plot_results:
# Plot the retrieved signal and the retrieval values
# First plot the averaged signal and the retrieved signal
plt.figure(1, figsize=[12, 8])
plt.clf()
plt.plot(self.fit_freq, self.average_signal)
plt.plot(self.arts.f_grid.value / 1e9, self.arts.yf.value, 'r')
plt.xlabel('Frequency [GHz]', fontsize=20)
plt.ylabel('Brightness Temperature [K]', fontsize=20)
plt.legend(['Data', 'Retrieval'], fontsize=20)
plt.title('Average Signal and Retrieved Signal', fontsize=24)
plt.savefig(basename + 'fig1.png')
# Now plot the actual retrieved Ozone VMR
Altitude = self.arts.z_field.value.flatten()
plt.figure(2, figsize=[8, 12])
plt.clf()
plt.plot(10**self.arts.xa.value[:-1] * 1e6, Altitude)
plt.plot(10**self.arts.x.value[:-1] * 1e6, Altitude)
plt.legend(['Prior', 'OEM Retrieval'], fontsize=20)
plt.ylabel('Altitude [m]', fontsize=20)
plt.xlabel('O3 [ppmv]', fontsize=20)
plt.title('Altitude vs. O3 VMR', fontsize=24)
plt.savefig(basename + 'fig2.png')
plt.figure(1, figsize=[12, 8])
plt.clf()
plt.plot(self.fit_freq, self.average_signal - self.arts.yf.value,
'r')
plt.xlabel('Frequency [GHz]', fontsize=20)
plt.ylabel('Residual [K]', fontsize=20)
plt.title('Residual Signal', fontsize=24)
plt.savefig(basename + 'fig5.png')
if plot_statistics:
# Plot some relevant statistics of the OEM retrieval
Altitude = self.arts.z_field.value.flatten()
averaging_kernel = self.arts.dxdy.value @ self.arts.jacobian.value
measurement_response = averaging_kernel @ np.ones(
averaging_kernel.shape[1])
plt.figure(3, figsize=[12, 8])
plt.clf()
[plt.plot(kernel[:-1], Altitude) for kernel in averaging_kernel.T]
plt.title('Averaging kernel for the OEM retrieval', fontsize=24)
plt.ylabel('Altitude [m]', fontsize=20)
plt.savefig(basename + 'fig3.png')
plt.figure(4, figsize=[12, 8])
plt.clf()
plt.plot(measurement_response[:-1], Altitude)
plt.title('Measurement response for the OEM retrieval',
fontsize=24)
plt.ylabel('Alititude [m]', fontsize=20)
plt.savefig(basename + 'fig4.png')
def _initialize_arts_workspace(self):
"""
AUTHOR:
Richard Larsson, Hayden Smotherman
DESCRIPTION:
This function is meant to be run interally to this class. It initializes the arts workspace.
INPUTS:
NONE
OUTPUTS:
NONE
"""
# -*- coding: utf-8 -*-
"""
Created on Thu Jul 5 15:53:02 2018
@author: larsson
"""
xmls = typhon.environ.get('ARTS_DATA_PATH')
self.arts = Workspace(0)
@arts_agenda
def water_psat_agenda(ws):
ws.water_p_eq_fieldMK05()
@arts_agenda
def propmat_clearsky_agenda_zeeman(ws):
ws.propmat_clearskyInit()
ws.propmat_clearskyAddOnTheFly()
ws.Ignore(ws.rtp_mag)
ws.Ignore(ws.rtp_los)
@arts_agenda
def ppath_agenda_step_by_step(ws):
ws.Ignore(ws.rte_pos2)
ws.ppathStepByStep()
@arts_agenda
def iy_main_agenda_emission(ws):
ws.Ignore(ws.iy_id)
ws.ppathCalc()
ws.iyEmissionStandard()
@arts_agenda
def iy_space_agenda_cosmic_background(ws):
ws.Ignore(ws.rtp_pos)
ws.Ignore(ws.rtp_los)
ws.MatrixCBR(ws.iy, ws.stokes_dim, ws.f_grid)
@arts_agenda
def iy_surface_agenda(ws):
ws.SurfaceDummy()
ws.iySurfaceRtpropAgenda()
@arts_agenda
def ppath_step_agenda_geometric(ws):
ws.Ignore(ws.t_field)
ws.Ignore(ws.vmr_field)
ws.Ignore(ws.f_grid)
ws.Ignore(ws.ppath_lraytrace)
ws.ppath_stepGeometric()
@arts_agenda
def abs_xsec_agenda_lines(ws):
ws.abs_xsec_per_speciesInit()
ws.abs_xsec_per_speciesAddLines2()
ws.abs_xsec_per_speciesAddConts()
@arts_agenda
def surface_rtprop_agenda(ws):
ws.InterpSurfaceFieldToPosition(out=ws.surface_skin_t,
field=ws.t_surface)
ws.surfaceBlackbody()
@arts_agenda
def geo_pos_agenda(ws):
ws.Ignore(ws.ppath)
ws.VectorSet(ws.geo_pos, np.array([]))
@arts_agenda
def sensor_response_agenda(ws):
ws.AntennaOff()
ws.sensorOff()
ws.Ignore(ws.f_backend)
# Set some agendas
self.arts.Copy(self.arts.surface_rtprop_agenda, surface_rtprop_agenda)
self.arts.Copy(self.arts.abs_xsec_agenda, abs_xsec_agenda_lines)
self.arts.Copy(self.arts.ppath_step_agenda,
ppath_step_agenda_geometric)
self.arts.Copy(self.arts.propmat_clearsky_agenda,
propmat_clearsky_agenda_zeeman)
self.arts.Copy(self.arts.iy_main_agenda, iy_main_agenda_emission)
self.arts.Copy(self.arts.iy_space_agenda,
iy_space_agenda_cosmic_background)
self.arts.Copy(self.arts.ppath_agenda, ppath_agenda_step_by_step)
self.arts.Copy(self.arts.iy_surface_agenda, iy_surface_agenda)
self.arts.Copy(self.arts.geo_pos_agenda, geo_pos_agenda)
self.arts.Copy(self.arts.water_p_eq_agenda, water_psat_agenda)
self.arts.Copy(self.arts.sensor_response_agenda,
sensor_response_agenda)
# Set some quantities that are unused because you do not need them (for now)
self.arts.Touch(self.arts.surface_props_data)
self.arts.Touch(self.arts.surface_props_names)
self.arts.Touch(self.arts.mag_u_field)
self.arts.Touch(self.arts.mag_v_field)
self.arts.Touch(self.arts.mag_w_field)
self.arts.Touch(self.arts.wind_u_field)
self.arts.Touch(self.arts.wind_v_field)
self.arts.Touch(self.arts.wind_w_field)
self.arts.Touch(self.arts.transmitter_pos)
self.arts.Touch(self.arts.iy_aux_vars)
self.arts.Touch(self.arts.mblock_dlos_grid)
self.arts.Touch(self.arts.scat_species)
# Ozone line and continua
self.arts.abs_cont_descriptionInit()
self.arts.abs_cont_descriptionAppend(tagname="H2O-PWR98",
model="Rosenkranz")
self.arts.abs_cont_descriptionAppend(tagname="O2-PWR98",
model="Rosenkranz")
self.arts.abs_cont_descriptionAppend(tagname="O2-PWR98",
model="Rosenkranz")
self.arts.abs_cont_descriptionAppend(tagname="N2-CIArotCKDMT252",
model="CKDMT252")
self.arts.abs_cont_descriptionAppend(tagname="N2-CIAfunCKDMT252",
model="CKDMT252")
self.arts.abs_speciesSet(species=[
'O2-PWR98', 'H2O-PWR98', 'N2-CIAfunCKDMT252, N2-CIArotCKDMT252'
])
self.arts.abs_linesReadFromSplitArtscat(
basename=os.path.join(xmls, 'spectroscopy/Perrin/'),
fmin=np.min(self.fit_freq) * 1e9 - 1e8,
fmax=np.max(self.fit_freq) * 1e9 + 1e8)
self.arts.abs_lines_per_speciesCreateFromLines()
# Set builtin Earth-viable isotopologue values and partition functions
self.arts.isotopologue_ratiosInitFromBuiltin()
self.arts.partition_functionsInitFromBuiltin()
self.arts.nlteOff() # LTE
self.arts.stokes_dim = 1 # No polarization
self.arts.xsec_speedup_switch = 0 # No speedup (experimental feature)
self.arts.rte_alonglos_v = 0. # No movement of satellite or rotation of planet
self.arts.lm_p_lim = 0. # Just do line mixing if available (it is not)
self.arts.abs_f_interp_order = 1 # Interpolation in frequency if you add a sensor
self.arts.ppath_lmax = 10000. # Maximum path length (Original)
self.arts.ppath_lraytrace = 10000. # Maximum path trace (Original)
self.arts.refellipsoidEarth(model="Sphere") # Europa average radius
self.arts.iy_unit = "PlanckBT" # Output results in Planck Brightess Temperature
# Set the size of the problem (change to your own numbers)
NP = 201 # Number of pressure levels
NF = len(self.fit_freq)
self.arts.lon_true = np.array([0])
self.arts.lat_true = np.array([0])
self.arts.AtmosphereSet1D()
self.arts.p_grid = np.logspace(5.04, -1.2, NP)
self.arts.z_surface = np.zeros((1, 1))
self.arts.t_surface = np.full((1, 1), 295.)
self.arts.f_grid = np.linspace(
np.min(self.fit_freq) * 1e9,
np.max(self.fit_freq) * 1e9, NF)
self.arts.sensorOff() # No sensor simulations
# Read the atmosphere... folder should contain:
# "H2O.xml"
# "t.xml"
# "z.xml"
# The files can be in binary format
self.arts.AtmRawRead(
basename=xmls +
'planets/Earth/Fascod/subarctic-summer/subarctic-summer')
self.arts.AtmFieldsCalc()
# Set observation geometry... You can make more positions and los
self.arts.sensor_pos = np.array([[10000]
]) # [[ALT, LAT, LON]] (Original)
self.arts.sensor_los = np.array([[63]]) # [[ZENITH, AZIMUTH]]
# Temperature and Ozone VMR Jacobian
self.arts.jacobianInit()
self.arts.jacobianAddTemperature(g1=self.arts.p_grid.value,
g2=np.array([]),
g3=np.array([]))
self.arts.jacobianAddAbsSpecies(g1=self.arts.p_grid.value,
g2=np.array([]),
g3=np.array([]),
species='H2O-PWR98')
self.arts.jacobianClose()
self.arts.cloudboxOff() # No Clouds
# Check that the input looks OK
self.arts.atmgeom_checkedCalc()
self.arts.atmfields_checkedCalc()
self.arts.cloudbox_checkedCalc()
self.arts.sensor_checkedCalc()
self.arts.propmat_clearsky_agenda_checkedCalc()
self.arts.abs_xsec_agenda_checkedCalc()
# Perform the calculations!
self.arts.yCalc()
def _initialize_covmat(self):
"""
AUTHOR:
Simon Pfreundschuh, Hayden Smotherman
DESCRIPTION:
This function is meant to be run interally to this class. It initializes the covariance matricies.
INPUTS:
NONE
OUTPUTS:
NONE
"""
z_grid = self.arts.z_field.value.flatten()
n_p = self.arts.p_grid.value.size
self.arts.retrievalDefInit()
# Kernel panic if 'grid_1' and 'sigma_1' are not the same size
self.arts.covmat1D(
self.arts.covmat_block,
grid_1=z_grid,
sigma_1=1e-7 * np.ones(n_p), # Relative uncertainty
cls_1=1.0e3 * np.ones(n_p), # Correlation length [m]
fname="exp")
self.arts.retrievalAddAbsSpecies(species="H2O-PWR98",
unit="vmr",
atmosphere_dim=1,
g1=self.arts.p_grid,
g2=np.array([]),
g3=np.array([]))
self.arts.jacobianSetFuncTransformation(transformation_func="none")
self.arts.covmatDiagonal(out=self.arts.covmat_block,
out_inverse=self.arts.covmat_inv_block,
vars=100.0 * np.ones(1))
self.arts.retrievalAddPolyfit(poly_order=0)
# self.arts.covmatDiagonal(out = self.arts.covmat_block,
# out_inverse = self.arts.covmat_inv_block,
# vars = 100.0 * np.ones(8*2))
# self.arts.retrievalAddSinefit(period_lengths = np.array([10e3, 20e3, 1e6, 2e6, 5e6, 10e6, 20e6, 1e9]))
self.arts.retrievalDefClose()
# More uncertainty measurements
self.arts.covmatDiagonal(self.arts.covmat_block,
self.arts.covmat_inv_block,
vars=self.sigma**2 *
np.ones(self.arts.y.value.shape))
self.arts.covmat_seSet(self.arts.covmat_block)
self.arts.jacobianAdjustAndTransform
# Kernel panic if 'arts.Ignore(arts.inversion_iteration_counter)' is not included
@arts_agenda
def inversion_iterate_agenda(arts):
arts.Ignore(arts.inversion_iteration_counter)
arts.x2artsAtmAndSurf()
arts.Copy(arts.f_backend, arts.f_grid)
arts.x2artsSensor()
arts.atmfields_checkedCalc(
negative_vmr_ok=1
) # negative_vmr_ok added to avoid error with negative vmr values
arts.atmgeom_checkedCalc()
arts.yCalc()
arts.jacobianAdjustAndTransform()
arts.VectorAddVector(arts.yf, arts.y, arts.y_baseline)
self.arts.Copy(self.arts.inversion_iterate_agenda,
inversion_iterate_agenda)
def _run_retrieval(self):
"""
AUTHOR:
Simon Pfreundschuh, Hayden Smotherman
DESCRIPTION:
This function is meant to be run interally to this class. It runs the actual retrieval.
INPUTS:
NONE
OUTPUTS:
NONE
"""
# Run the OEM retrieval
self.arts.Touch(self.arts.particle_bulkprop_field)
self.arts.Touch(self.arts.particle_bulkprop_names)
self.arts.VectorSetConstant(self.arts.sensor_time, 1, 0.)
self.arts.xaStandard()
self.arts.x = np.zeros(0)
self.arts.y.value[:] = self.average_signal
self.arts.jacobian = np.zeros((0, 0))
self.arts.OEM(method="li",
max_iter=20,
display_progress=1,
lm_ga_settings=np.array(
[100.0, 5.0, 2.0, 10.0, 1.0, 1.0]))
self.arts.avkCalc()
class TestWorkspace:
def setup_method(self):
"""This ensures a new Workspace for every test."""
self.dir = os.path.dirname(os.path.realpath(__file__))
self.ws = Workspace()
self.setup_workspace()
def setup_workspace(self):
ws = self.ws
ws.atmosphere_dim = 1
ws.p_grid = np.linspace(1e5, 1e3, 21)
ws.Touch(ws.lat_grid)
ws.Touch(ws.lon_grid)
ws.f_grid = 183.0e9 * np.ones(1)
ws.stokes_dim = 1
ws.sensor_los = 180.0 * np.ones((1, 1))
ws.sensor_pos = 830e3 * np.ones((1, 1))
ws.sensorOff()
def test_index_transfer(self):
self.ws.IndexCreate("index_variable")
i = np.random.randint(0, 100)
self.ws.index_variable = i
assert self.ws.index_variable.value == i
def test_array_of_index_transfer(self):
self.ws.ArrayOfIndexCreate("array_of_index_variable")
i = [np.random.randint(0, 100) for j in range(10)]
self.ws.array_of_index_variable = i
assert self.ws.array_of_index_variable.value == i
self.ws.array_of_index_variable = []
assert self.ws.array_of_index_variable.value == []
def test_array_of_vector_transfer(self):
self.ws.ArrayOfVectorCreate("array_of_vector_variable")
aov = typhon.arts.xml.load(
os.path.join(self.dir, "xml/reference/arrayofvector.xml"))
self.ws.array_of_vector_variable = aov
assert self.ws.array_of_vector_variable.value == aov
def test_string_transfer(self):
self.ws.StringCreate("string_variable")
s = "some random string."
self.ws.string_variable = s
assert self.ws.string_variable.value == s
def test_vector_transfer(self):
self.ws.VectorCreate("vector_variable")
v = np.random.rand(10)
self.ws.vector_variable = v
assert all(self.ws.vector_variable.value == v)
def test_matrix_transfer(self):
self.ws.MatrixCreate("matrix_variable")
m = np.random.rand(10, 10)
self.ws.matrix_variable = m
assert all(self.ws.matrix_variable.value.ravel() == m.ravel())
def test_sparse_transfer(self):
sparse_formats = ["csc", "csr", "bsr", "lil", "dok", "coo", "dia"]
for f in sparse_formats:
i = sp.sparse.identity(11)
self.ws.sensor_response = i
print(self.ws.sensor_response.value)
assert np.all(
i.todense() == self.ws.sensor_response.value.todense())
def test_supergeneric_overload_resolution(self):
self.ws.ArrayOfIndexCreate("array_of_index")
self.ws.ArrayOfArrayOfIndexCreate("array_of_array_of_index")
self.ws.array_of_index = [1, 2, 3]
self.ws.Append(self.ws.array_of_array_of_index, self.ws.array_of_index)
self.ws.Append(self.ws.array_of_array_of_index, self.ws.array_of_index)
def test_creation(self):
self.ws.ArrayOfIndexCreate("array_of_index")
self.ws.ArrayOfIndexCreate("array_of_index")
with pytest.raises(Exception):
self.ws.VectorCreate("array_of_index")
def test_wsm_error(self):
with pytest.raises(Exception):
self.ws.yCalc()
def test_doc(self):
repr(self.ws.yCalc)
def test_agenda(self):
self.ws.atmosphere_dim = 1
@arts_agenda
def add_1(ws):
ws.IndexAdd(ws.atmosphere_dim, ws.atmosphere_dim, 1)
add_1.execute(self.ws)
assert self.ws.atmosphere_dim.value == 2
add_1.append(add_1)
add_1.execute(self.ws)
assert self.ws.atmosphere_dim.value == 4
args = [self.ws.atmosphere_dim, self.ws.atmosphere_dim, 1]
@arts_agenda
def add_2(ws):
ws.IndexAdd(*args)
add_2.execute(self.ws)
assert self.ws.atmosphere_dim.value == 5
def test_execute_controlfile(self):
dir = os.path.dirname(os.path.realpath(__file__))
test_dir = os.path.join(dir, "test_files")
self.ws.WriteXML("ascii", np.array([1.0]),
os.path.join(test_dir, "vector.xml"))
os.chdir(test_dir)
self.ws.execute_controlfile("controlfile.arts")
os.remove(os.path.join(test_dir, "vector.xml"))
def test_supergeneric_overload_failure(self):
with pytest.raises(Exception):
self.ws.NumericCreate("numeric_wsv")
self.ws.StringCreate("string_wsv")
self.ws.Copy(self.ws.string_wsv, self.ws.numeric_wsv)
def test_tensor_3(self):
t_0 = np.random.rand(*([3] * 3))
self.ws.Tensor3Create("tensor_3")
self.ws.tensor_3 = t_0
assert np.all(t_0 == self.ws.tensor_3.value)
def test_tensor_4(self):
t_0 = np.random.rand(*([3] * 4))
t_1 = self.ws.Tensor4Create("tensor_4")
self.ws.tensor_4 = t_0
assert np.all(t_0 == self.ws.tensor_4.value)
def test_tensor_5(self):
t_0 = np.random.rand(*([3] * 5))
t_1 = self.ws.Tensor5Create("tensor_5")
self.ws.tensor_5 = t_0
assert np.all(t_0 == self.ws.tensor_5.value)
def test_tensor_6(self):
t_0 = np.random.rand(*([3] * 6))
t_1 = self.ws.Tensor6Create("tensor_6")
self.ws.tensor_6 = t_0
assert np.all(t_0 == self.ws.tensor_6.value)
def test_tensor_7(self):
t_0 = np.random.rand(*([3] * 7))
self.ws.Tensor7Create("tensor_7")
self.ws.tensor_7 = t_0
assert np.all(t_0 == self.ws.tensor_7.value)
def test_execute_controlfile(self):
dir = os.path.dirname(os.path.realpath(__file__))
test_dir = os.path.join(dir, "test_files")
self.ws.WriteXML("ascii", np.array([1.0]),
os.path.join(test_dir, "vector.xml"))
os.chdir(test_dir)
agenda = self.ws.execute_controlfile("controlfile.arts")
self.ws.foo = "not bar"
@arts_agenda
def execute(ws):
ws.FlagOff(ws.jacobian_do)
ws.StringSet(ws.foo, "still not bar")
INCLUDE("controlfile.arts")
INCLUDE(agenda)
self.ws.execute_agenda(execute)
assert self.ws.foo.value == "bar"
os.remove(os.path.join(test_dir, "vector.xml"))
def test_covariance_matrix(self):
ws = self.ws
ws.jacobianInit()
ws.jacobianAddAbsSpecies(species="O3",
g1=ws.p_grid,
g2=ws.lat_grid,
g3=ws.lon_grid)
ws.jacobianAddAbsSpecies(species="H2O",
g1=ws.p_grid,
g2=ws.lat_grid,
g3=ws.lon_grid)
ws.jacobianClose()
ws.covmatDiagonal(out=ws.covmat_block,
out_inverse=ws.covmat_block,
vars=10.0 * np.ones(ws.p_grid.value.size))
ws.covmat_sxAddBlock(block=ws.covmat_block)
ws.covmatDiagonal(out=ws.covmat_block,
out_inverse=ws.covmat_block,
vars=20.0 * np.ones(ws.p_grid.value.size))
ws.covmat_sxAddBlock(block=ws.covmat_block)
def test_variable_set_empty(self):
self.ws.f_grid = np.array([94e9])
self.ws.f_grid = []
assert self.ws.f_grid.value.size == 0
def test_variable_creation(self):
# Unnamed variable
wsv = self.ws.create_variable("Matrix", None)
self.ws.__setattr__(wsv.name, np.eye(5))
assert np.all(
np.isclose(np.eye(5),
self.ws.__getattr__(wsv.name).value))
# Named variable
wsv = self.ws.create_variable("Matrix", "matrix_wsv")
self.ws.matrix_wsv = np.eye(5)
assert np.all(np.isclose(np.eye(5), self.ws.matrix_wsv.value))
def test_wsv_setattr(self):
wsv = self.ws.atmosphere_dim
wsv.value = 12
assert self.ws.atmosphere_dim.value == 12
def test_convert(self):
v = WorkspaceVariable.convert("Index", 1.2)
assert (v == 1)
v = WorkspaceVariable.convert("String", "string")
assert (v == "string")
v = WorkspaceVariable.convert("Numeric", 1)
assert (type(v) == np.float64)
v = WorkspaceVariable.convert("Vector", 1.0)
assert (v.shape == (1, ))
v = WorkspaceVariable.convert("Matrix", 1.0)
assert (v.shape == (1, 1))
v = WorkspaceVariable.convert("Tensor3", 1.0)
assert (v.shape == (1, 1, 1))
v = WorkspaceVariable.convert("Tensor6", 1.0)
assert (v.shape == (1, 1, 1, 1, 1, 1))
v = WorkspaceVariable.convert("ArrayOfArrayOfIndex", 1.0)
assert (type(v) == list)
assert (type(v[0]) == list)
assert (type(v[0][0]) == int)
def test_wind_3d_demo():
ws = Workspace()
ws.execute_controlfile("general/general.arts")
ws.verbositySet(0, 0, 0, 0)
ws.execute_controlfile("general/agendas.arts")
ws.execute_controlfile("general/continua.arts")
ws.execute_controlfile("general/planet_earth.arts")
ws.Copy(ws.abs_xsec_agenda, ws.abs_xsec_agenda__noCIA)
ws.Copy(ws.ppath_agenda, ws.ppath_agenda__FollowSensorLosPath)
ws.Copy(ws.ppath_step_agenda, ws.ppath_step_agenda__GeometricPath)
ws.Copy(ws.iy_space_agenda, ws.iy_space_agenda__CosmicBackground)
ws.Copy(ws.iy_surface_agenda, ws.iy_surface_agenda__UseSurfaceRtprop)
ws.Copy(ws.iy_main_agenda, ws.iy_main_agenda__Emission)
ws.Copy(ws.propmat_clearsky_agenda, ws.propmat_clearsky_agenda__OnTheFly)
# General Settings
# For the wind retrievals, the forward model calculations are performed on a 3D atmosphere grid.
# Radiation is assumed to be unpolarized.
ws.atmosphere_dim = 3
ws.stokes_dim = 1
ws.iy_unit = "RJBT"
# Absorption
# We only consider absorption from ozone in this example. The lineshape data is available from
# the ARTS testdata available in `controlfiles/testdata`.
ws.abs_speciesSet(["O3", "H2O-PWR98"])
ws.abs_lineshapeDefine("Voigt_Kuntz6", "VVH", 750e9)
ws.ReadXML(ws.abs_lines, "testdata/ozone_line.xml")
ws.abs_lines_per_speciesCreateFromLines()
# Atmosphere (A Priori)
# We create a pressure grid using the `PFromZSimple` function to create a grid of approximate pressure levels
# corresponding to altitudes in the range
# z = 0.0, 2000.0, ..., 94000.0
z_toa = 95e3
z_surf = 1e3
z_grid = np.arange(z_surf - 1e3, z_toa, 2e3)
ws.PFromZSimple(ws.p_grid, z_grid)
ws.lat_grid = np.arange(-40.0, 1.0, 40.0)
ws.lon_grid = np.arange(40.0, 61.0, 20.0)
ws.z_surface = z_surf * np.ones(
(np.asarray(ws.lat_grid).size, np.asarray(ws.lon_grid).size))
# For the a priori state we read data from the Fascod climatology that is part of the ARTS xml data.
ws.AtmRawRead(basename="planets/Earth/Fascod/tropical/tropical")
ws.AtmFieldsCalcExpand1D()
# Adding Wind
# Wind in ARTS is represented by the `wind_u_field` and `wind_v_field` WSVs, which hold the horizontal components
# of the wind at each grid point of the atmosphere model. For this example, a constant wind is assumed.
u_wind = 60.0
v_wind = -40.0
ws.wind_u_field = u_wind * np.ones(
(ws.p_grid.value.size, ws.lat_grid.value.size, ws.lon_grid.value.size))
ws.wind_v_field = v_wind * np.ones(
(ws.p_grid.value.size, ws.lat_grid.value.size, ws.lon_grid.value.size))
ws.wind_w_field = np.zeros((0, 0, 0))
# Frequency Grid and Sensor
# The frequency grid for the simulation consists of 119 grid points between 110.516 and 111.156 GHz.
# The frequencies are given by a degree-10 polynomial that has been obtained from a fit to the data from
# the original `qpack` example. This is obscure but also kind of cool.
coeffs = np.array([
5.06312189e-08, -2.68851772e-05, 6.20655463e-03, -8.16344090e-01,
6.75337174e+01, -3.66786505e+03, 1.32578167e+05, -3.14514304e+06,
4.57491354e+07, 1.10516484e+11
])
ws.f_grid = np.poly1d(coeffs)(np.arange(119))
# For the sensor we assume a channel width and channel spacing of 50 kHz. We also call AntennaOff to compute
# only one pencilbeam along the line of sight of the sensor.
df = 50e3
f_backend = np.arange(ws.f_grid.value.min() + 2.0 * df,
ws.f_grid.value.max() - 2.0 * df, df)
ws.backend_channel_responseGaussian(np.array([df]), np.array([2.0]))
ws.AntennaOff()
ws.sensor_norm = 1
ws.sensor_time = np.zeros(1)
ws.sensor_responseInit()
# Sensor Position and Viewing Geometry
# 5 Measurements are performed, one straight up, and four with zenith angle 70∘70∘ in directions SW, NW, NE, SE.
# In ARTS the measurement directions are given by a two-column matrix, where the first column contains the zenith
# angle and the second column the azimuth angle.
ws.sensor_los = np.array([[
0.0,
0.0,
], [70.0, -135.0], [70.0, -45.0], [70.0, 45.0], [70.0, 135.0]])
ws.sensor_pos = np.array([[2000.0, -21.1, 55.6]] * 5)
# Reference Measurement
# Before we can calculate `y`, our setup needs to pass the following tests:
ws.abs_f_interp_order = 3
ws.propmat_clearsky_agenda_checkedCalc()
ws.sensor_checkedCalc()
ws.atmgeom_checkedCalc()
ws.atmfields_checkedCalc()
ws.abs_xsec_agenda_checkedCalc()
ws.jacobianOff()
ws.cloudboxOff()
ws.cloudbox_checkedCalc()
ws.yCalc()
y = np.copy(ws.y.value)
# Setting up the Retrieval
# In this example, we retrieve ozone and the horizontal and vertical components of the wind velocities.
# The state space covariance matrix in ARTS is represented by the **covmat_sa** WSV.
# It belongs to the CovarianceMatrix group, which is used to represent block diagonal matrices.
# For each retrieval quantity that is added to the retrieval, a corresponding block must be added to **covmat_sa**.
# This is usually done by the corresponding **retrievalAdd...** call, which looks for this block
# in the **covmat_block** WSV.
# In short the general workflow for adding a retrieval quantity is as follows:
# - Create the covariance matrix for the retrieval quantity either calling one of the **covmat...** WSV or
# by loading your own matrix
# - Write the matrix block into **covmat_block**
# - Call the **retrievalAdd...** method to add the retrieval quantity and the covariance matrix block
# to **covmat_sa**
lat_ret_grid = np.array([np.mean(ws.lat_grid)])
lon_ret_grid = np.array([np.mean(ws.lon_grid)])
n_p = ws.p_grid.value.size
ws.retrievalDefInit()
ws.covmat1D(
ws.covmat_block,
grid_1=z_grid,
sigma_1=0.1 * np.ones(n_p), # Relative uncertainty
cls_1=10e3 * np.ones(n_p), # 10km correlation length
fname="lin")
ws.retrievalAddAbsSpecies(species="O3",
unit="rel",
g1=ws.p_grid,
g2=lat_ret_grid,
g3=lon_ret_grid)
# Wind u-component
ws.covmat1D(
ws.covmat_block,
grid_1=z_grid[::2],
sigma_1=100.0 * np.ones(n_p // 2), # Relative uncertainty
cls_1=10e3 * np.ones(n_p // 2), # 10km correlation length
fname="lin")
ws.retrievalAddWind(g1=ws.p_grid.value[::2],
g2=np.array([np.mean(ws.lat_grid)]),
g3=np.array([np.mean(ws.lon_grid)]),
component="u")
# Wind v-component
ws.covmat1D(
ws.covmat_block,
grid_1=z_grid[::2],
sigma_1=100.0 * np.ones(n_p // 2), # Relative uncertainty
cls_1=10e3 * np.ones(n_p // 2), # 10km correlation length
fname="lin")
ws.retrievalAddWind(g1=ws.p_grid.value[::2],
g2=np.array([np.mean(ws.lat_grid)]),
g3=np.array([np.mean(ws.lon_grid)]),
component="v")
ws.retrievalDefClose()
ws.covmatDiagonal(ws.covmat_block,
ws.covmat_inv_block,
vars=0.0001 * np.ones(ws.y.value.shape))
ws.covmat_seSet(ws.covmat_block)
@arts_agenda
def inversion_iterate_agenda(ws):
ws.x2artsStandard()
ws.atmfields_checkedCalc()
ws.atmgeom_checkedCalc()
ws.yCalc()
ws.Print(ws.y)
ws.Print(ws.jacobian)
ws.VectorAddVector(ws.yf, ws.y, ws.y_baseline)
ws.IndexAdd(ws.inversion_iteration_counter,
ws.inversion_iteration_counter, 1)
ws.Copy(ws.inversion_iterate_agenda, inversion_iterate_agenda)
# A Priori State
# For the a priori state we assume zero wind in any direction. The a priori vector for the OEM is created by
# the `xaStandard` WSM, which computes $x_a$ from the current atmospheric state.
ws.wind_u_field.value[:] = 0.0
ws.wind_v_field.value[:] = 0.0
ws.xaStandard()
# The OEM Calculation
ws.x = np.zeros(0)
ws.jacobian = np.zeros((0, 0))
ws.y.value[:] = y
ws.OEM(method="lm",
max_iter=20,
display_progress=1,
lm_ga_settings=np.array([100.0, 2.0, 2.0, 10.0, 1.0, 1.0]))
ws.x2artsStandard()
z = ws.z_field.value[:, 0, 0].ravel()
wind_u = ws.wind_u_field.value[z > 40e3, 0, 0]
wind_v = ws.wind_v_field.value[z > 40e3, 0, 0]
assert np.allclose(wind_u, u_wind, atol=1)
assert np.allclose(wind_v, v_wind, atol=1)