class ArtsController():
"""A not so high level interface to ARTS."""
def __init__(self, verbosity=0, agenda_verbosity=0):
self.ws = Workspace(verbosity=verbosity, agenda_verbosity=agenda_verbosity)
self.retrieval_quantities = []
self._sensor = None
self._observations = []
def setup(self, atmosphere_dim=1, iy_unit='RJBT', ppath_lmax=-1, stokes_dim=1):
"""
Run boilerplate (includes, agendas) and set basic variables.
:param atmosphere_dim:
:param iy_unit:
:param ppath_lmax:
:param stokes_dim:
"""
boilerplate.include_general(self.ws)
boilerplate.copy_agendas(self.ws)
boilerplate.set_basics(self.ws, atmosphere_dim, iy_unit, ppath_lmax, stokes_dim)
# Deactivate some stuff (can be activated later)
self.ws.jacobianOff()
self.ws.cloudboxOff()
def checked_calc(self, negative_vmr_ok=False, bad_partition_functions_ok=False):
"""Run checked calculations."""
boilerplate.run_checks(self.ws, negative_vmr_ok, bad_partition_functions_ok)
def set_spectroscopy(self, abs_lines, abs_species, line_shape=None, abs_f_interp_order=3):
"""
Setup absorption species and spectroscopy data.
:param ws: The workspace.
:param abs_lines: Absoption lines.
:param abs_species: List of abs species tags.
:param line_shape: Line shape definition. Default: ['Voigt_Kuntz6', 'VVH', 750e9]
:param f_abs_interp_order: No effect for OnTheFly propmat. Default: 3
:type abs_lines: typhon.arts.catalogues.ArrayOfLineRecord
"""
boilerplate.setup_spectroscopy(self.ws, abs_lines, abs_species, line_shape)
self.ws.abs_f_interp_order = abs_f_interp_order # no effect for OnTheFly propmat
def set_spectroscopy_from_file(self, abs_lines_file, abs_species, format='Arts', line_shape=None, abs_f_interp_order=3):
"""
Setup absorption species and spectroscopy data from XML file.
:param ws: The workspace.
:param abs_lines_file: Path to an XML file.
:param abs_species: List of abs species tags.
:param format: One of 'Arts', 'Jpl', 'Hitran' (and others for which a WSM `abs_linesReadFrom...` exists)
:param line_shape: Line shape definition. Default: ['Voigt_Kuntz6', 'VVH', 750e9]
:param f_abs_interp_order: No effect for OnTheFly propmat. Default: 3
"""
ws = self.ws
if line_shape is None:
line_shape = ['Voigt_Kuntz6', 'VVH', 750e9]
ws.abs_speciesSet(abs_species)
ws.abs_lineshapeDefine(*line_shape)
#ws.ReadXML(ws.abs_lines, abs_lines_file)
read_fn = getattr(ws, 'abs_linesReadFrom' + format)
read_fn(filename=abs_lines_file, fmin=float(0), fmax=float(10e12))
ws.abs_lines_per_speciesCreateFromLines()
ws.abs_f_interp_order = abs_f_interp_order
def set_grids(self, f_grid, p_grid, lat_grid=None, lon_grid=None):
"""
Set the forward model grids. Basic checks are performed, depending on dimensionality of atmosphere.
:param f_grid:
:param p_grid:
:param lat_grid:
:param lon_grid:
"""
if lat_grid is None:
lat_grid = np.array([0])
if lon_grid is None:
lon_grid = np.array([0])
if not self.ws.atmosphere_dim.initialized:
raise Exception('atmosphere_dim must be initialized before assigning grids.')
if not _is_asc(f_grid):
raise ValueError('Values of f_grid must be strictly increasing.')
if not _is_desc(p_grid) or not np.all(p_grid > 0):
raise ValueError('Values of p_grid must be strictly decreasing and positive.')
if self.ws.atmosphere_dim.value == 1:
if lat_grid.size > 1 or lon_grid.size > 1:
raise ValueError('For 1D atmosphere, lat_grid and lon_grid shall be of length 1.')
elif self.ws.atmosphere_dim.value == 2:
if lon_grid is not None or len(lat_grid):
raise ValueError('For 2D atmosphere, lon_grid shall be empty.')
if lat_grid is None or len(lat_grid) == 0:
raise ValueError('For 2D atmosphere, lat_grid must be set.')
elif self.ws.atmosphere_dim.value == 3:
if lat_grid is None or len(lat_grid) < 2 or lon_grid is None or len(lon_grid) < 2:
raise ValueError('For 3D atmosphere, length of lat_grid and lon_grid must be >= 2.')
if max(abs(lon_grid)) > 360:
raise ValueError('Values of lon_grid must be in the range [-360,360].')
if max(abs(lat_grid)) > 90:
raise ValueError('Values of lat_grid must be in the range [-90,90].')
if lat_grid is not None and not _is_asc(lat_grid):
raise ValueError('Values of lat_grid must be strictly increasing.')
if lon_grid is not None and not _is_asc(lon_grid):
raise ValueError('Values of lon_grid must be strictly increasing.')
self.ws.f_grid = f_grid
self.ws.p_grid = p_grid
if self.atmosphere_dim > 1:
self.ws.lat_grid = lat_grid
self.ws.lon_grid = lon_grid
else:
self.ws.lat_grid = []
self.ws.lon_grid = []
self.ws.lat_true = lat_grid
self.ws.lon_true = lon_grid
self.set_surface(0)
def set_surface(self, altitude: float):
"""
Set surface altitude.
:param float altitude:
.. note:: Currently, only constant altitudes are supported.
"""
shape = max((1, 1), (self.n_lat, self.n_lat))
self.ws.z_surface = altitude * np.ones(shape)
def set_atmosphere(self, atmosphere, vmr_zeropadding=False):
"""
Set the atmospheric state.
:param atmosphere: Atmosphere with Temperature, Altitude and VMR Fields.
:type atmosphere: retrievals.arts.atmosphere.Atmosphere
:param vmr_zeropadding: Allow VMR zero padding wind fields are always zero
padded. Default: False.
.. note:: Currently only supports 1D atmospheres that is then expanded to a
multi-dimensional homogeneous atmosphere.
"""
vmr_zeropadding = 1 if vmr_zeropadding else 0
self.ws.t_field_raw = atmosphere.t_field
self.ws.z_field_raw = atmosphere.z_field
self.ws.vmr_field_raw = [atmosphere.vmr_field(mt) for mt in self.abs_species_maintags]
self.ws.nlte_field_raw = None
for c in ('u', 'v', 'w'):
try:
raw_field = atmosphere.wind_field(c)
field = p_interpolate(
self.p_grid, raw_field.grids[0], raw_field.data[:, 0, 0], fill=0
)
except KeyError:
field = np.zeros_like(self.p_grid)
field = np.tile(
field[:, np.newaxis, np.newaxis], (1, self.n_lat, self.n_lon)
)
field_name = 'wind_{}_field'.format(c)
setattr(self.ws, field_name, field)
if self.atmosphere_dim == 1:
self.ws.AtmFieldsCalc(vmr_zeropadding=vmr_zeropadding)
else:
self.ws.AtmFieldsCalcExpand1D(vmr_zeropadding=vmr_zeropadding)
def apply_hse(self, p_hse=100e2, z_hse_accuracy=0.5):
"""
Calculate z field from hydrostatic equilibrium. See :arts:method:`z_fieldFromHSE`.
:param p_hse: See :arts:variable:`p_hse`.
:param z_hse_accuracy: See :arts:variable:`z_hse_accuracy`.
"""
ws = self.ws
ws.p_hse = p_hse
ws.z_hse_accuracy = z_hse_accuracy
ws.atmfields_checkedCalc()
ws.z_fieldFromHSE()
def set_wind(self, wind_u=None, wind_v=None, wind_w=None):
"""
Set the wind fields to constant values.
"""
ws = self.ws
n_p, n_lat, n_lon = self.n_p, self.n_lat, self.n_lon
if wind_u is not None:
ws.Tensor3SetConstant(ws.wind_u_field, n_p, n_lat, n_lon, float(wind_u))
if wind_v is not None:
ws.Tensor3SetConstant(ws.wind_v_field, n_p, n_lat, n_lon, float(wind_v))
if wind_w is not None:
ws.Tensor3SetConstant(ws.wind_w_field, n_p, n_lat, n_lon, float(wind_w))
def _check_set_wind(self):
"""Set uninitialized wind fields to 0."""
ws = self.ws
if ws.wind_u_field.value.size == 0:
self.set_wind(wind_u=0)
if ws.wind_v_field.value.size == 0:
self.set_wind(wind_v=0)
if ws.wind_w_field.value.size == 0:
self.set_wind(wind_w=0)
def set_observations(self, observations):
"""
Set the geometry of the observations made.
:param observations:
:type observations: Iterable[retrievals.arts.interface.Observation]
"""
sensor_los = np.array([[obs.za, obs.aa] for obs in observations])
sensor_pos = np.array([[obs.alt, obs.lat, obs.lon] for obs in observations])
self.ws.sensor_los = sensor_los[:, :max(self.atmosphere_dim - 1, 1)]
self.ws.sensor_pos = sensor_pos[:, :self.atmosphere_dim]
self.ws.sensor_time = np.array([obs.time for obs in observations])
self._observations = observations
def set_y(self, ys):
"""
Set the observations.
:param ys: List with a spectrum for every observation.
:return:
"""
y = np.concatenate(ys)
self.ws.y = y
def set_sensor(self, sensor):
"""
Set the sensor.
:param sensor:
:type sensor: retrievals.arts.sensors.AbstractSensor
"""
self._sensor = sensor
sensor.apply(self.ws)
def y_calc(self, jacobian_do=False):
"""
Run the forward model.
:param jacobian_do: Not implemented yet.
:return: The measurements as list with length according to observations.
"""
if jacobian_do:
raise NotImplementedError('Jacobian not implemented yet.')
# self.ws.jacobian_do = 1
self.ws.yCalc()
return self.y
def define_retrieval(self, retrieval_quantities, y_vars):
"""
Define the retrieval quantities.
:param retrieval_quantities: Iterable of retrieval quantities `retrievals.arts.retrieval.RetrievalQuantity`.
:param y_vars: List or variance vectors according.
"""
ws = self.ws
if isinstance(y_vars, (list, tuple)):
y_vars = np.concatenate(y_vars)
if len(y_vars) != self.n_y:
raise ValueError('Variance vector y_vars must have same length as y.')
ws.retrievalDefInit()
# Retrieval quantities
self.retrieval_quantities = retrieval_quantities
for rq in retrieval_quantities:
rq.apply(ws)
# Se and its inverse
covmat_block = sparse.diags(y_vars, format='csr')
boilerplate.set_variable_by_xml(ws, ws.covmat_block, covmat_block)
ws.covmat_seAddBlock(block=ws.covmat_block)
covmat_block = sparse.diags(1/y_vars, format='csr')
boilerplate.set_variable_by_xml(ws, ws.covmat_block, covmat_block)
ws.covmat_seAddInverseBlock(block=ws.covmat_block)
ws.retrievalDefClose()
def oem(self, method='li', max_iter=10, stop_dx=0.01, lm_ga_settings=None, display_progress=True,
inversion_iterate_agenda=None):
"""
Run the optimal estimation. See Arts documentation for details.
:param method:
:param max_iter:
:param stop_dx:
:param lm_ga_settings: Default: [10, 2, 2, 100, 1, 99]
:param display_progress:
:param inversion_iterate_agenda: If set to None, a simple default agenda is used.
:return:
"""
ws = self.ws
if lm_ga_settings is None:
lm_ga_settings = [100.0, 2.0, 2.0, 10.0, 1.0, 1.0]
lm_ga_settings = np.array(lm_ga_settings)
# x, jacobian and yf must be initialised
ws.x = np.array([])
ws.yf = np.array([])
ws.jacobian = np.array([[]])
if inversion_iterate_agenda is None:
inversion_iterate_agenda = boilerplate.inversion_iterate_agenda
ws.Copy(ws.inversion_iterate_agenda, inversion_iterate_agenda)
ws.AgendaExecute(ws.sensor_response_agenda)
# a priori values
self._check_set_wind()
ws.xaStandard()
xa = ws.xa.value
for rq in self.retrieval_quantities:
rq.extract_apriori(xa)
# Run inversion
try:
ws.OEM(method=method, max_iter=max_iter, stop_dx=stop_dx, lm_ga_settings=lm_ga_settings,
display_progress=1 if display_progress else 0)
except Exception as e:
raise OemException(e)
if not self.oem_converged: # Just checks if dxdy is initialized
return False
ws.x2artsAtmAndSurf()
ws.x2artsSensor()
ws.avkCalc()
ws.covmat_ssCalc()
ws.covmat_soCalc()
ws.retrievalErrorsExtract()
x = ws.x.value
avk = ws.avk.value
eo = ws.retrieval_eo.value
es = ws.retrieval_ss.value
for rq in self.retrieval_quantities:
rq.extract_result(x, avk, eo, es)
return True
def get_level2_xarray(self):
ds = xr.merge([rq.to_xarray() for rq in self.retrieval_quantities])
# Spectra
f_backend = self._sensor.f_backend if self._sensor.f_backend is not None else self.f_grid
ds['f'] = ('f', f_backend)
ds['y'] = (('observation', 'f'), np.stack(self.y))
ds['yf'] = (('observation', 'f'), np.stack(self.yf))
ds['oem_diagnostics'] = ('oem_diagnostics_idx', self.oem_diagnostics)
y_baseline = self.y_baseline
if y_baseline is not None:
ds['y_baseline'] = (('observation', 'f'), np.stack(self.y_baseline))
# Observations
for f in Observation._fields:
values = np.array([getattr(obs, f) for obs in self._observations], dtype=np.float)
ds['obs_'+f] = ('observation', values)
# Global attributes
ds.attrs['arts_version'] = self.arts_version
ds.attrs['uuid'] = str(uuid.uuid4())
ds.attrs['date_created'] = datetime.datetime.utcnow().isoformat()
ds.attrs['nodename'] = os.uname().nodename
return ds
@property
def p_grid(self):
return self.ws.p_grid.value
@property
def lat_grid(self):
return self.ws.lat_grid.value
@property
def lon_grid(self):
return self.ws.lon_grid.value
@property
def f_grid(self):
return self.ws.f_grid.value
@property
def n_p(self):
return len(self.ws.p_grid.value)
@property
def n_lat(self):
if self.atmosphere_dim == 1:
return 1
return len(self.ws.lat_grid.value)
@property
def n_lon(self):
if self.atmosphere_dim == 1:
return 1
return len(self.ws.lat_grid.value)
@property
def n_y(self):
return len(self.ws.y.value)
@property
def y(self):
y = np.copy(self.ws.y.value)
return np.split(y, self.n_obs)
@property
def yf(self):
yf = np.copy(self.ws.yf.value)
return np.split(yf, self.n_obs)
@property
def y_baseline(self):
if self.ws.y_baseline.initialized:
bl = np.copy(self.ws.y_baseline.value)
if bl.size == 1 and bl[0] == 0:
return None
return np.split(bl, self.n_obs)
else:
return None
@property
def n_obs(self):
if self.ws.sensor_time.initialized:
return len(self.ws.sensor_time.value)
else:
return 0
@property
def abs_species_maintags(self):
abs_species = self.ws.abs_species.value
maintags = [st[0].split('-')[0] for st in abs_species]
return maintags
@property
def atmosphere_dim(self):
return self.ws.atmosphere_dim.value
@property
def oem_converged(self):
return self.ws.oem_diagnostics.value[0] == 0
@property
def oem_diagnostics(self):
return self.ws.oem_diagnostics.value
@property
def oem_errors(self):
return self.ws.oem_errors.value
@property
def arts_version(self):
cmd = os.path.join(os.environ['ARTS_BUILD_PATH'], 'src/arts')
output = os.popen(cmd + ' --version').read().splitlines()
return output[0]
class oem_retrieval():
"""
AUTHOR:
Hayden Smotherman
DESCRIPTION:
This class is meant to run an OEM retrieval on data from the radiometers at MPI Solar System.
The hope is that it simplifies the process of using the typhon OEM retreival for the specific
use-case of Paul Hartogh's group at MPI Solar System.
It can load in the data, attempt to fit and subtract-out sine curves from the noise,
and filter high-frequency oscillations that are present in some data.
It can also run a retrieval using the typhon OEM retrieval and plot the results,
along with relevant statistical metrics.
"""
def __init__(self):
"""
AUTHOR:
Hayden Smotherman
DESCRIPTION:
This function initializes the oem_retrieval class
INPUTS:
NONE
OUTPUTS:
NONE
"""
self.signal = np.array(
[]) # This array will hold the brightness temperature data
self.noise = np.array(
[]) # This array will hold the noise for the radiometer
self.freq = np.array(
[]) # This array will hold the frequencies of "signal" and "noise"
def load_data(self,
data_type='radiometer',
use_data=0,
filename='',
signal=[],
noise=[],
freq=[]):
"""
AUTHOR:
Hayden Smotherman
DESCRIPTION:
This function loads in radiometer data to the oem_retrival class.
It accepts data in a number of different formats.
INPUTS:
data_type - A string denoting which format the input data will be in.
VALUES: 'numpy' - Regular numpy arrays passed in to "signal", "noise", and "freq"
REQUIRES: signal, noise, freq
'radiometer' - A .raw file generated by a radiometer at MPI Solar System.
REQUIRES: filename
use_data - Integer value denoting how much data to use when running a mean OEM retrieval.
This value should be zero (use all data) or negative (use the last x signals)
signal - An array of the brightness temperatures of an O3-666 line.
ONLY USED WITH data_type='numpy'
noise - An array of the noise of the radiometer
ONLY USED WITH data_type='numpy'
freq - An array of the central frequency of each datapoint in "signal" and "noise"
ONLY USED WITH data_type='numpy'
OUTPUTS:
"""
eform = '>i16f7500f100f28f28f'
dform = '>i16f4092f100f28f28f'
aform = '>i16f16384f128f28f28f'
xform = '>i16f1019f128f28f28f'
if data_type == 'radiometer':
# Load data from a .raw file from the 210MHz radiometer
# Load in the raw data
if filename[0] == 'a':
data_format = aform
elif filename[0] == 'd':
data_format = dform
elif filename[0] == 'e':
data_format = eform
elif filename[0] == 'x':
data_format = xform
else:
raise ImportError(
'Could not understand the data format based on the file name.'
)
Calibrated = calibration(format=data_format)
Calibrated.load(filename)
Calibrated.calibrate()
# Pre-allocate signal and noise arrays
self.signal = np.array(Calibrated._signal[use_data:])
self.noise = np.array(Calibrated._noise[use_data:])
# Generate the frequency array based on the data format
if filename[0] == 'a':
# First we split the data, since two signals are embedded in the ._signal array
# NOTE: Currently this only keeps the first half of the data. It should keep all of it.
self.signal = Calibrated._signal[use_data:]
self.noise = Calibrated._noise[use_data:]
Half_Length = int(len(Calibrated._noise[0]) /
2) # Data is set as one long array
for i in range(len(self.signal)):
self.signal[i] = np.copy(
Calibrated._signal[i][0:Half_Length])
self.noise[i] = np.copy(
Calibrated._noise[i][0:Half_Length])
self.signal = np.array(self.signal)
self.noise = np.array(self.noise)
# Width of data is 1.5GHz
# Central frequency is data number 14 in _raw array, requested frequency
Min_Freq = Calibrated._raw[0][14] - 1.50 / 2
Max_Freq = Calibrated._raw[0][14] + 1.50 / 2
self.freq = np.linspace(Min_Freq, Max_Freq, Half_Length)
elif filename[0] == 'd':
# Width of data is 40MHz
# Central frequency is data number 14 in _raw array, requested frequency
Min_Freq = Calibrated._raw[0][14] - .040 / 2
Max_Freq = Calibrated._raw[0][14] + .040 / 2
self.freq = np.linspace(Min_Freq, Max_Freq,
len(Calibrated._noise[0]))
elif filename[0] == 'e':
# Width of data is 210MHz
# Central frequency is data number 14 in _raw array, requested frequency
Min_Freq = Calibrated._raw[0][14] - .210 / 2
Max_Freq = Calibrated._raw[0][14] + .210 / 2
self.freq = np.linspace(Min_Freq, Max_Freq,
len(Calibrated._noise[0]))
elif filename[0] == 'x':
# Width of data is 4.4GHz
# Central frequency is data number 14 in _raw array, requested frequency
Min_Freq = Calibrated._raw[0][14] - 4.4 / 2
Max_Freq = Calibrated._raw[0][14] + 4.4 / 2
self.freq = np.linspace(Min_Freq, Max_Freq,
len(Calibrated._noise[0]))
else:
raise ImportError(
'Could not generate frequency array. Unknown file format.')
Calibrated = None
elif data_type == 'numpy':
# Load in arbitrary data that is already in a numpy array
self.signal = np.array(signal)
self.noise = np.array(noise)
self.freq = np.array(freq)
# Define fit_ values here in case subtract_sinewave is never called
self.fit_freq = np.copy(self.freq)
self.fit_signal = np.copy(self.signal)
self.fit_noise = np.copy(self.noise)
def process_data(self,
lims=[21, 23],
shift_freq=False,
fit_sine=False,
plot_example=True,
initial_guess=[3, 1 / 1000, 0],
central_freq=1.42175037e+2):
"""
AUTHOR:
Hayden Smotherman
DESCRIPTION:
This function clips data from the front and back of the dataset based on front_lim and
back_lim. If "fit_sine=True", then it also subtracts out sine waves from the data.
INPUTS:
lims - Optional frequency limits (in GHz) that determine the range of data to use.
shift_freq - Boolean that determines whether or not to shift the frequency such that the
max of the averaged data aligns with the peak of the O3 signal.
fit_sine - Boolean that determines whether or not to fit a sine wave to the noise then
subtract it out of the signal.
plot_example - Boolean that determines whether or not to plot an example fit. Only used if
fit_sine=True
initial_guess - This is the initial guess for the sine parameters used by lsqfit. Only used
if fit_sine=True
central_freq - This is the frequency center of the measurements. Only used if
shift_freq=True
OUTPUTS:
NONE
NOTES:
- This function works best for data in dform and eform. Datasets with longer frequency
baselines are not fit as well and may give bad results.
- By default, "process_data" clips the first 25 data points regardless of what is set in "lims".
This is done because the first ~20 data points are often severe outliers.
"""
# Calculate the index corresponding to the minimum frequency limit
if shift_freq:
if len(self.signal.shape) > 1:
Average_Signal = np.mean(self.signal, axis=0)
else:
Average_Signal = np.copy(self.signal)
Freq_Peak = self.freq[Average_Signal == np.max(Average_Signal)]
Freq_Difference = Freq_Peak - central_freq
self.freq -= Freq_Difference
front_lim = np.where(
self.freq == np.min(self.freq[self.freq > lims[0]]))
front_lim = int(front_lim[0])
if (front_lim < 25):
# Set front_lim minimum to 25, since the first ~25 data points are very often bad
front_lim = 25
# Calculate the index corresponding to the maximum frequency limit
back_lim = np.where(
self.freq == np.max(self.freq[self.freq < lims[1]]))
back_lim = int(back_lim[0])
Signal_Length = self.signal.shape[1] # Length of the signal array
Signal_Number = self.signal.shape[
0] # Number of signals in the data file
X_Data = np.linspace(
0, Signal_Length - 1,
Signal_Length) # Array of indicies for the fitting
X_Data = X_Data[front_lim:back_lim]
# Preallocate the memory for the fitted values of signal and noise
self.fit_signal = np.zeros([
self.signal.shape[0],
np.size(self.signal[0][front_lim:back_lim])
])
self.fit_noise = np.zeros(
[self.noise.shape[0],
np.size(self.noise[0][front_lim:back_lim])])
self.fit_freq = np.copy(self.freq[front_lim:back_lim])
if fit_sine:
for i in range(Signal_Number):
# Iterate over all signals and subtract out the best-fit sine curve
# Load in noise locally for ease of use
Noise_0 = self.noise[i]
# Limit the data based on front_lim and back_lim and subtract the mean
Noise_0 = Noise_0[front_lim:back_lim]
Noise_Median = np.median(Noise_0)
Base_Noise = Noise_0 - Noise_Median # Subtract out the noise mean for fitting
Sine_Params = initial_guess # Amplitude, freq, phase
# Sine function
optimize_func = lambda x: x[0] * np.sin(x[1] * X_Data + x[2]
) - Base_Noise
# Perform least square fit to the sine function
est_amp, est_freq, est_phase = leastsq(optimize_func,
Sine_Params)[0]
# Save the fit
Fit = est_amp * np.sin(est_freq * X_Data + est_phase)
# Load in signal locally
Signal_0 = self.signal[i]
Signal_0 = Signal_0[front_lim:back_lim]
# Subtract the fit from the signal and the noise and save it in new variables
#print(np.size(Signal_0), np.size(Fit))
self.fit_signal[i] = Signal_0 - Fit
self.fit_noise[i] = Noise_0 - Fit
if plot_example:
# Plot the last "noise" value along with the fit sine curve
plt.figure(figsize=[12, 8])
plt.plot(self.fit_freq, Noise_0)
plt.plot(self.fit_freq, Fit + Noise_Median)
plt.legend(['Unfitted Noise', 'Best Fit Sine Curve'],
fontsize=20)
plt.xlabel('Frequency [GHz]', fontsize=20)
plt.ylabel('Brightness Temperature [K]', fontsize=20)
plt.title('Raw Noise and Best Fit Curve', fontsize=24)
# Plot the fitted noise over the raw noise
plt.figure(figsize=[12, 8])
plt.plot(self.fit_freq, self.noise[-1][front_lim:back_lim])
plt.plot(self.fit_freq, self.fit_noise[-1])
plt.legend(['Unfitted Noise', 'Fitted Noise'], fontsize=20)
plt.xlabel('Frequency [GHz]', fontsize=20)
plt.ylabel('Brightness Temperature [K]', fontsize=20)
plt.title('Unfitted Noise and Fitted Noise', fontsize=24)
# Plot the fitted signal over the raw signal
plt.figure(figsize=[12, 8])
plt.plot(self.fit_freq, self.signal[-1][front_lim:back_lim])
plt.plot(self.fit_freq, self.fit_signal[-1])
plt.legend(['Unfitted Signal', 'Fitted Signal'], fontsize=20)
plt.xlabel('Frequency [GHz]', fontsize=20)
plt.ylabel('Brightness Temperature [K]', fontsize=20)
plt.title('Unfitted Signal and Fitted Signal', fontsize=24)
else:
for i in range(Signal_Number):
self.fit_signal[i] = self.signal[i][front_lim:back_lim]
self.fit_noise[i] = self.noise[i][front_lim:back_lim]
def mean_retrieval(self,
use_data=0,
filtered=False,
shift_freq=False,
sigma=None,
central_freq=1.42175037e+2):
"""
AUTHOR:
Hayden Smotherman
DESCRIPTION:
This function uses the Typhon OEM retrieval package to recover atmospheric O3 levels
given the brightness temperature signal of the O3-666 line. If self.signal is a 2D
matrix, then this function will run the retrieval on the mean of this data.
INPUTS:
filtered - Boolean that determines whether or not to use the scipy.signal.filtfilt
function on the mean signal value in order to potentially improve the signal
shift_freq - Boolean that determines whether or not to shift the frequency such that the
max of the averaged data aligns with the peak of the O3 signal.
central_freq - This is the frequency center of the measurements. Only used if
shift_freq=True
OUTPUTS:
NONE
"""
if len(self.signal.shape) > 1:
self.average_signal = np.mean(self.fit_signal, axis=0)
self.average_noise = np.mean(self.fit_noise, axis=0)
else:
self.average_signal = np.copy(self.fit_signal)
self.average_noise = np.copy(self.fit_noise)
if filtered:
n = 4 # the larger n is, the smoother the curve will be
b = [1.0 / n] * n
a = 1
self.average_signal = sg.filtfilt(b, a, self.average_signal)
self.average_noise = sg.filtfilt(b, a, self.average_noise)
if shift_freq:
Freq_Peak = self.fit_freq[self.average_signal == np.max(
self.average_signal)]
Freq_Difference = Freq_Peak - central_freq
self.fit_freq -= Freq_Difference
self._initialize_arts_workspace()
if sigma is None:
self.sigma = np.sqrt(
np.sum(
np.abs(self.average_noise - np.mean(self.average_noise))) /
len(self.average_noise))
else:
self.sigma = sigma
self._initialize_covmat()
self._run_retrieval()
def plot_oem(self, basename='', plot_results=True, plot_statistics=True):
"""
AUTHOR:
Hayden Smotherman
DESCRIPTION:
This function uses the Typhon OEM retrieval package to recover atmospheric O3 levels
given the brightness temperature signal of the O3-666 line. If self.signal is a 2D
matrix, then this function will run the retrieval on the mean of this data.
INPUTS:
plot_results - Boolean that determines whether or not to plot the results of the retrieval.
plot_statistics - Boolean that determines whether or not to plot relevant statistics
from the retrieval.
OUTPUTS:
NONE
"""
if plot_results:
# Plot the retrieved signal and the retrieval values
# First plot the averaged signal and the retrieved signal
plt.figure(1, figsize=[12, 8])
plt.clf()
plt.plot(self.fit_freq, self.average_signal)
plt.plot(self.arts.f_grid.value / 1e9, self.arts.yf.value, 'r')
plt.xlabel('Frequency [GHz]', fontsize=20)
plt.ylabel('Brightness Temperature [K]', fontsize=20)
plt.legend(['Data', 'Retrieval'], fontsize=20)
plt.title('Average Signal and Retrieved Signal', fontsize=24)
plt.savefig(basename + 'fig1.png')
# Now plot the actual retrieved Ozone VMR
Altitude = self.arts.z_field.value.flatten()
plt.figure(2, figsize=[8, 12])
plt.clf()
plt.plot(10**self.arts.xa.value[:-1] * 1e6, Altitude)
plt.plot(10**self.arts.x.value[:-1] * 1e6, Altitude)
plt.legend(['Prior', 'OEM Retrieval'], fontsize=20)
plt.ylabel('Altitude [m]', fontsize=20)
plt.xlabel('O3 [ppmv]', fontsize=20)
plt.title('Altitude vs. O3 VMR', fontsize=24)
plt.savefig(basename + 'fig2.png')
plt.figure(1, figsize=[12, 8])
plt.clf()
plt.plot(self.fit_freq, self.average_signal - self.arts.yf.value,
'r')
plt.xlabel('Frequency [GHz]', fontsize=20)
plt.ylabel('Residual [K]', fontsize=20)
plt.title('Residual Signal', fontsize=24)
plt.savefig(basename + 'fig5.png')
if plot_statistics:
# Plot some relevant statistics of the OEM retrieval
Altitude = self.arts.z_field.value.flatten()
averaging_kernel = self.arts.dxdy.value @ self.arts.jacobian.value
measurement_response = averaging_kernel @ np.ones(
averaging_kernel.shape[1])
plt.figure(3, figsize=[12, 8])
plt.clf()
[plt.plot(kernel[:-1], Altitude) for kernel in averaging_kernel.T]
plt.title('Averaging kernel for the OEM retrieval', fontsize=24)
plt.ylabel('Altitude [m]', fontsize=20)
plt.savefig(basename + 'fig3.png')
plt.figure(4, figsize=[12, 8])
plt.clf()
plt.plot(measurement_response[:-1], Altitude)
plt.title('Measurement response for the OEM retrieval',
fontsize=24)
plt.ylabel('Alititude [m]', fontsize=20)
plt.savefig(basename + 'fig4.png')
def _initialize_arts_workspace(self):
"""
AUTHOR:
Richard Larsson, Hayden Smotherman
DESCRIPTION:
This function is meant to be run interally to this class. It initializes the arts workspace.
INPUTS:
NONE
OUTPUTS:
NONE
"""
# -*- coding: utf-8 -*-
"""
Created on Thu Jul 5 15:53:02 2018
@author: larsson
"""
xmls = typhon.environ.get('ARTS_DATA_PATH')
self.arts = Workspace(0)
@arts_agenda
def water_psat_agenda(ws):
ws.water_p_eq_fieldMK05()
@arts_agenda
def propmat_clearsky_agenda_zeeman(ws):
ws.propmat_clearskyInit()
ws.propmat_clearskyAddOnTheFly()
ws.Ignore(ws.rtp_mag)
ws.Ignore(ws.rtp_los)
@arts_agenda
def ppath_agenda_step_by_step(ws):
ws.Ignore(ws.rte_pos2)
ws.ppathStepByStep()
@arts_agenda
def iy_main_agenda_emission(ws):
ws.Ignore(ws.iy_id)
ws.ppathCalc()
ws.iyEmissionStandard()
@arts_agenda
def iy_space_agenda_cosmic_background(ws):
ws.Ignore(ws.rtp_pos)
ws.Ignore(ws.rtp_los)
ws.MatrixCBR(ws.iy, ws.stokes_dim, ws.f_grid)
@arts_agenda
def iy_surface_agenda(ws):
ws.SurfaceDummy()
ws.iySurfaceRtpropAgenda()
@arts_agenda
def ppath_step_agenda_geometric(ws):
ws.Ignore(ws.t_field)
ws.Ignore(ws.vmr_field)
ws.Ignore(ws.f_grid)
ws.Ignore(ws.ppath_lraytrace)
ws.ppath_stepGeometric()
@arts_agenda
def abs_xsec_agenda_lines(ws):
ws.abs_xsec_per_speciesInit()
ws.abs_xsec_per_speciesAddLines2()
ws.abs_xsec_per_speciesAddConts()
@arts_agenda
def surface_rtprop_agenda(ws):
ws.InterpSurfaceFieldToPosition(out=ws.surface_skin_t,
field=ws.t_surface)
ws.surfaceBlackbody()
@arts_agenda
def geo_pos_agenda(ws):
ws.Ignore(ws.ppath)
ws.VectorSet(ws.geo_pos, np.array([]))
@arts_agenda
def sensor_response_agenda(ws):
ws.AntennaOff()
ws.sensorOff()
ws.Ignore(ws.f_backend)
# Set some agendas
self.arts.Copy(self.arts.surface_rtprop_agenda, surface_rtprop_agenda)
self.arts.Copy(self.arts.abs_xsec_agenda, abs_xsec_agenda_lines)
self.arts.Copy(self.arts.ppath_step_agenda,
ppath_step_agenda_geometric)
self.arts.Copy(self.arts.propmat_clearsky_agenda,
propmat_clearsky_agenda_zeeman)
self.arts.Copy(self.arts.iy_main_agenda, iy_main_agenda_emission)
self.arts.Copy(self.arts.iy_space_agenda,
iy_space_agenda_cosmic_background)
self.arts.Copy(self.arts.ppath_agenda, ppath_agenda_step_by_step)
self.arts.Copy(self.arts.iy_surface_agenda, iy_surface_agenda)
self.arts.Copy(self.arts.geo_pos_agenda, geo_pos_agenda)
self.arts.Copy(self.arts.water_p_eq_agenda, water_psat_agenda)
self.arts.Copy(self.arts.sensor_response_agenda,
sensor_response_agenda)
# Set some quantities that are unused because you do not need them (for now)
self.arts.Touch(self.arts.surface_props_data)
self.arts.Touch(self.arts.surface_props_names)
self.arts.Touch(self.arts.mag_u_field)
self.arts.Touch(self.arts.mag_v_field)
self.arts.Touch(self.arts.mag_w_field)
self.arts.Touch(self.arts.wind_u_field)
self.arts.Touch(self.arts.wind_v_field)
self.arts.Touch(self.arts.wind_w_field)
self.arts.Touch(self.arts.transmitter_pos)
self.arts.Touch(self.arts.iy_aux_vars)
self.arts.Touch(self.arts.mblock_dlos_grid)
self.arts.Touch(self.arts.scat_species)
# Ozone line and continua
self.arts.abs_cont_descriptionInit()
self.arts.abs_cont_descriptionAppend(tagname="H2O-PWR98",
model="Rosenkranz")
self.arts.abs_cont_descriptionAppend(tagname="O2-PWR98",
model="Rosenkranz")
self.arts.abs_cont_descriptionAppend(tagname="O2-PWR98",
model="Rosenkranz")
self.arts.abs_cont_descriptionAppend(tagname="N2-CIArotCKDMT252",
model="CKDMT252")
self.arts.abs_cont_descriptionAppend(tagname="N2-CIAfunCKDMT252",
model="CKDMT252")
self.arts.abs_speciesSet(species=[
'O2-PWR98', 'H2O-PWR98', 'N2-CIAfunCKDMT252, N2-CIArotCKDMT252'
])
self.arts.abs_linesReadFromSplitArtscat(
basename=os.path.join(xmls, 'spectroscopy/Perrin/'),
fmin=np.min(self.fit_freq) * 1e9 - 1e8,
fmax=np.max(self.fit_freq) * 1e9 + 1e8)
self.arts.abs_lines_per_speciesCreateFromLines()
# Set builtin Earth-viable isotopologue values and partition functions
self.arts.isotopologue_ratiosInitFromBuiltin()
self.arts.partition_functionsInitFromBuiltin()
self.arts.nlteOff() # LTE
self.arts.stokes_dim = 1 # No polarization
self.arts.xsec_speedup_switch = 0 # No speedup (experimental feature)
self.arts.rte_alonglos_v = 0. # No movement of satellite or rotation of planet
self.arts.lm_p_lim = 0. # Just do line mixing if available (it is not)
self.arts.abs_f_interp_order = 1 # Interpolation in frequency if you add a sensor
self.arts.ppath_lmax = 10000. # Maximum path length (Original)
self.arts.ppath_lraytrace = 10000. # Maximum path trace (Original)
self.arts.refellipsoidEarth(model="Sphere") # Europa average radius
self.arts.iy_unit = "PlanckBT" # Output results in Planck Brightess Temperature
# Set the size of the problem (change to your own numbers)
NP = 201 # Number of pressure levels
NF = len(self.fit_freq)
self.arts.lon_true = np.array([0])
self.arts.lat_true = np.array([0])
self.arts.AtmosphereSet1D()
self.arts.p_grid = np.logspace(5.04, -1.2, NP)
self.arts.z_surface = np.zeros((1, 1))
self.arts.t_surface = np.full((1, 1), 295.)
self.arts.f_grid = np.linspace(
np.min(self.fit_freq) * 1e9,
np.max(self.fit_freq) * 1e9, NF)
self.arts.sensorOff() # No sensor simulations
# Read the atmosphere... folder should contain:
# "H2O.xml"
# "t.xml"
# "z.xml"
# The files can be in binary format
self.arts.AtmRawRead(
basename=xmls +
'planets/Earth/Fascod/subarctic-summer/subarctic-summer')
self.arts.AtmFieldsCalc()
# Set observation geometry... You can make more positions and los
self.arts.sensor_pos = np.array([[10000]
]) # [[ALT, LAT, LON]] (Original)
self.arts.sensor_los = np.array([[63]]) # [[ZENITH, AZIMUTH]]
# Temperature and Ozone VMR Jacobian
self.arts.jacobianInit()
self.arts.jacobianAddTemperature(g1=self.arts.p_grid.value,
g2=np.array([]),
g3=np.array([]))
self.arts.jacobianAddAbsSpecies(g1=self.arts.p_grid.value,
g2=np.array([]),
g3=np.array([]),
species='H2O-PWR98')
self.arts.jacobianClose()
self.arts.cloudboxOff() # No Clouds
# Check that the input looks OK
self.arts.atmgeom_checkedCalc()
self.arts.atmfields_checkedCalc()
self.arts.cloudbox_checkedCalc()
self.arts.sensor_checkedCalc()
self.arts.propmat_clearsky_agenda_checkedCalc()
self.arts.abs_xsec_agenda_checkedCalc()
# Perform the calculations!
self.arts.yCalc()
def _initialize_covmat(self):
"""
AUTHOR:
Simon Pfreundschuh, Hayden Smotherman
DESCRIPTION:
This function is meant to be run interally to this class. It initializes the covariance matricies.
INPUTS:
NONE
OUTPUTS:
NONE
"""
z_grid = self.arts.z_field.value.flatten()
n_p = self.arts.p_grid.value.size
self.arts.retrievalDefInit()
# Kernel panic if 'grid_1' and 'sigma_1' are not the same size
self.arts.covmat1D(
self.arts.covmat_block,
grid_1=z_grid,
sigma_1=1e-7 * np.ones(n_p), # Relative uncertainty
cls_1=1.0e3 * np.ones(n_p), # Correlation length [m]
fname="exp")
self.arts.retrievalAddAbsSpecies(species="H2O-PWR98",
unit="vmr",
atmosphere_dim=1,
g1=self.arts.p_grid,
g2=np.array([]),
g3=np.array([]))
self.arts.jacobianSetFuncTransformation(transformation_func="none")
self.arts.covmatDiagonal(out=self.arts.covmat_block,
out_inverse=self.arts.covmat_inv_block,
vars=100.0 * np.ones(1))
self.arts.retrievalAddPolyfit(poly_order=0)
# self.arts.covmatDiagonal(out = self.arts.covmat_block,
# out_inverse = self.arts.covmat_inv_block,
# vars = 100.0 * np.ones(8*2))
# self.arts.retrievalAddSinefit(period_lengths = np.array([10e3, 20e3, 1e6, 2e6, 5e6, 10e6, 20e6, 1e9]))
self.arts.retrievalDefClose()
# More uncertainty measurements
self.arts.covmatDiagonal(self.arts.covmat_block,
self.arts.covmat_inv_block,
vars=self.sigma**2 *
np.ones(self.arts.y.value.shape))
self.arts.covmat_seSet(self.arts.covmat_block)
self.arts.jacobianAdjustAndTransform
# Kernel panic if 'arts.Ignore(arts.inversion_iteration_counter)' is not included
@arts_agenda
def inversion_iterate_agenda(arts):
arts.Ignore(arts.inversion_iteration_counter)
arts.x2artsAtmAndSurf()
arts.Copy(arts.f_backend, arts.f_grid)
arts.x2artsSensor()
arts.atmfields_checkedCalc(
negative_vmr_ok=1
) # negative_vmr_ok added to avoid error with negative vmr values
arts.atmgeom_checkedCalc()
arts.yCalc()
arts.jacobianAdjustAndTransform()
arts.VectorAddVector(arts.yf, arts.y, arts.y_baseline)
self.arts.Copy(self.arts.inversion_iterate_agenda,
inversion_iterate_agenda)
def _run_retrieval(self):
"""
AUTHOR:
Simon Pfreundschuh, Hayden Smotherman
DESCRIPTION:
This function is meant to be run interally to this class. It runs the actual retrieval.
INPUTS:
NONE
OUTPUTS:
NONE
"""
# Run the OEM retrieval
self.arts.Touch(self.arts.particle_bulkprop_field)
self.arts.Touch(self.arts.particle_bulkprop_names)
self.arts.VectorSetConstant(self.arts.sensor_time, 1, 0.)
self.arts.xaStandard()
self.arts.x = np.zeros(0)
self.arts.y.value[:] = self.average_signal
self.arts.jacobian = np.zeros((0, 0))
self.arts.OEM(method="li",
max_iter=20,
display_progress=1,
lm_ga_settings=np.array(
[100.0, 5.0, 2.0, 10.0, 1.0, 1.0]))
self.arts.avkCalc()