class ArtsController(): """A not so high level interface to ARTS.""" def __init__(self, verbosity=0, agenda_verbosity=0): self.ws = Workspace(verbosity=verbosity, agenda_verbosity=agenda_verbosity) self.retrieval_quantities = [] self._sensor = None self._observations = [] def setup(self, atmosphere_dim=1, iy_unit='RJBT', ppath_lmax=-1, stokes_dim=1): """ Run boilerplate (includes, agendas) and set basic variables. :param atmosphere_dim: :param iy_unit: :param ppath_lmax: :param stokes_dim: """ boilerplate.include_general(self.ws) boilerplate.copy_agendas(self.ws) boilerplate.set_basics(self.ws, atmosphere_dim, iy_unit, ppath_lmax, stokes_dim) # Deactivate some stuff (can be activated later) self.ws.jacobianOff() self.ws.cloudboxOff() def checked_calc(self, negative_vmr_ok=False, bad_partition_functions_ok=False): """Run checked calculations.""" boilerplate.run_checks(self.ws, negative_vmr_ok, bad_partition_functions_ok) def set_spectroscopy(self, abs_lines, abs_species, line_shape=None, abs_f_interp_order=3): """ Setup absorption species and spectroscopy data. :param ws: The workspace. :param abs_lines: Absoption lines. :param abs_species: List of abs species tags. :param line_shape: Line shape definition. Default: ['Voigt_Kuntz6', 'VVH', 750e9] :param f_abs_interp_order: No effect for OnTheFly propmat. Default: 3 :type abs_lines: typhon.arts.catalogues.ArrayOfLineRecord """ boilerplate.setup_spectroscopy(self.ws, abs_lines, abs_species, line_shape) self.ws.abs_f_interp_order = abs_f_interp_order # no effect for OnTheFly propmat def set_spectroscopy_from_file(self, abs_lines_file, abs_species, format='Arts', line_shape=None, abs_f_interp_order=3): """ Setup absorption species and spectroscopy data from XML file. :param ws: The workspace. :param abs_lines_file: Path to an XML file. :param abs_species: List of abs species tags. :param format: One of 'Arts', 'Jpl', 'Hitran' (and others for which a WSM `abs_linesReadFrom...` exists) :param line_shape: Line shape definition. Default: ['Voigt_Kuntz6', 'VVH', 750e9] :param f_abs_interp_order: No effect for OnTheFly propmat. Default: 3 """ ws = self.ws if line_shape is None: line_shape = ['Voigt_Kuntz6', 'VVH', 750e9] ws.abs_speciesSet(abs_species) ws.abs_lineshapeDefine(*line_shape) #ws.ReadXML(ws.abs_lines, abs_lines_file) read_fn = getattr(ws, 'abs_linesReadFrom' + format) read_fn(filename=abs_lines_file, fmin=float(0), fmax=float(10e12)) ws.abs_lines_per_speciesCreateFromLines() ws.abs_f_interp_order = abs_f_interp_order def set_grids(self, f_grid, p_grid, lat_grid=None, lon_grid=None): """ Set the forward model grids. Basic checks are performed, depending on dimensionality of atmosphere. :param f_grid: :param p_grid: :param lat_grid: :param lon_grid: """ if lat_grid is None: lat_grid = np.array([0]) if lon_grid is None: lon_grid = np.array([0]) if not self.ws.atmosphere_dim.initialized: raise Exception('atmosphere_dim must be initialized before assigning grids.') if not _is_asc(f_grid): raise ValueError('Values of f_grid must be strictly increasing.') if not _is_desc(p_grid) or not np.all(p_grid > 0): raise ValueError('Values of p_grid must be strictly decreasing and positive.') if self.ws.atmosphere_dim.value == 1: if lat_grid.size > 1 or lon_grid.size > 1: raise ValueError('For 1D atmosphere, lat_grid and lon_grid shall be of length 1.') elif self.ws.atmosphere_dim.value == 2: if lon_grid is not None or len(lat_grid): raise ValueError('For 2D atmosphere, lon_grid shall be empty.') if lat_grid is None or len(lat_grid) == 0: raise ValueError('For 2D atmosphere, lat_grid must be set.') elif self.ws.atmosphere_dim.value == 3: if lat_grid is None or len(lat_grid) < 2 or lon_grid is None or len(lon_grid) < 2: raise ValueError('For 3D atmosphere, length of lat_grid and lon_grid must be >= 2.') if max(abs(lon_grid)) > 360: raise ValueError('Values of lon_grid must be in the range [-360,360].') if max(abs(lat_grid)) > 90: raise ValueError('Values of lat_grid must be in the range [-90,90].') if lat_grid is not None and not _is_asc(lat_grid): raise ValueError('Values of lat_grid must be strictly increasing.') if lon_grid is not None and not _is_asc(lon_grid): raise ValueError('Values of lon_grid must be strictly increasing.') self.ws.f_grid = f_grid self.ws.p_grid = p_grid if self.atmosphere_dim > 1: self.ws.lat_grid = lat_grid self.ws.lon_grid = lon_grid else: self.ws.lat_grid = [] self.ws.lon_grid = [] self.ws.lat_true = lat_grid self.ws.lon_true = lon_grid self.set_surface(0) def set_surface(self, altitude: float): """ Set surface altitude. :param float altitude: .. note:: Currently, only constant altitudes are supported. """ shape = max((1, 1), (self.n_lat, self.n_lat)) self.ws.z_surface = altitude * np.ones(shape) def set_atmosphere(self, atmosphere, vmr_zeropadding=False): """ Set the atmospheric state. :param atmosphere: Atmosphere with Temperature, Altitude and VMR Fields. :type atmosphere: retrievals.arts.atmosphere.Atmosphere :param vmr_zeropadding: Allow VMR zero padding wind fields are always zero padded. Default: False. .. note:: Currently only supports 1D atmospheres that is then expanded to a multi-dimensional homogeneous atmosphere. """ vmr_zeropadding = 1 if vmr_zeropadding else 0 self.ws.t_field_raw = atmosphere.t_field self.ws.z_field_raw = atmosphere.z_field self.ws.vmr_field_raw = [atmosphere.vmr_field(mt) for mt in self.abs_species_maintags] self.ws.nlte_field_raw = None for c in ('u', 'v', 'w'): try: raw_field = atmosphere.wind_field(c) field = p_interpolate( self.p_grid, raw_field.grids[0], raw_field.data[:, 0, 0], fill=0 ) except KeyError: field = np.zeros_like(self.p_grid) field = np.tile( field[:, np.newaxis, np.newaxis], (1, self.n_lat, self.n_lon) ) field_name = 'wind_{}_field'.format(c) setattr(self.ws, field_name, field) if self.atmosphere_dim == 1: self.ws.AtmFieldsCalc(vmr_zeropadding=vmr_zeropadding) else: self.ws.AtmFieldsCalcExpand1D(vmr_zeropadding=vmr_zeropadding) def apply_hse(self, p_hse=100e2, z_hse_accuracy=0.5): """ Calculate z field from hydrostatic equilibrium. See :arts:method:`z_fieldFromHSE`. :param p_hse: See :arts:variable:`p_hse`. :param z_hse_accuracy: See :arts:variable:`z_hse_accuracy`. """ ws = self.ws ws.p_hse = p_hse ws.z_hse_accuracy = z_hse_accuracy ws.atmfields_checkedCalc() ws.z_fieldFromHSE() def set_wind(self, wind_u=None, wind_v=None, wind_w=None): """ Set the wind fields to constant values. """ ws = self.ws n_p, n_lat, n_lon = self.n_p, self.n_lat, self.n_lon if wind_u is not None: ws.Tensor3SetConstant(ws.wind_u_field, n_p, n_lat, n_lon, float(wind_u)) if wind_v is not None: ws.Tensor3SetConstant(ws.wind_v_field, n_p, n_lat, n_lon, float(wind_v)) if wind_w is not None: ws.Tensor3SetConstant(ws.wind_w_field, n_p, n_lat, n_lon, float(wind_w)) def _check_set_wind(self): """Set uninitialized wind fields to 0.""" ws = self.ws if ws.wind_u_field.value.size == 0: self.set_wind(wind_u=0) if ws.wind_v_field.value.size == 0: self.set_wind(wind_v=0) if ws.wind_w_field.value.size == 0: self.set_wind(wind_w=0) def set_observations(self, observations): """ Set the geometry of the observations made. :param observations: :type observations: Iterable[retrievals.arts.interface.Observation] """ sensor_los = np.array([[obs.za, obs.aa] for obs in observations]) sensor_pos = np.array([[obs.alt, obs.lat, obs.lon] for obs in observations]) self.ws.sensor_los = sensor_los[:, :max(self.atmosphere_dim - 1, 1)] self.ws.sensor_pos = sensor_pos[:, :self.atmosphere_dim] self.ws.sensor_time = np.array([obs.time for obs in observations]) self._observations = observations def set_y(self, ys): """ Set the observations. :param ys: List with a spectrum for every observation. :return: """ y = np.concatenate(ys) self.ws.y = y def set_sensor(self, sensor): """ Set the sensor. :param sensor: :type sensor: retrievals.arts.sensors.AbstractSensor """ self._sensor = sensor sensor.apply(self.ws) def y_calc(self, jacobian_do=False): """ Run the forward model. :param jacobian_do: Not implemented yet. :return: The measurements as list with length according to observations. """ if jacobian_do: raise NotImplementedError('Jacobian not implemented yet.') # self.ws.jacobian_do = 1 self.ws.yCalc() return self.y def define_retrieval(self, retrieval_quantities, y_vars): """ Define the retrieval quantities. :param retrieval_quantities: Iterable of retrieval quantities `retrievals.arts.retrieval.RetrievalQuantity`. :param y_vars: List or variance vectors according. """ ws = self.ws if isinstance(y_vars, (list, tuple)): y_vars = np.concatenate(y_vars) if len(y_vars) != self.n_y: raise ValueError('Variance vector y_vars must have same length as y.') ws.retrievalDefInit() # Retrieval quantities self.retrieval_quantities = retrieval_quantities for rq in retrieval_quantities: rq.apply(ws) # Se and its inverse covmat_block = sparse.diags(y_vars, format='csr') boilerplate.set_variable_by_xml(ws, ws.covmat_block, covmat_block) ws.covmat_seAddBlock(block=ws.covmat_block) covmat_block = sparse.diags(1/y_vars, format='csr') boilerplate.set_variable_by_xml(ws, ws.covmat_block, covmat_block) ws.covmat_seAddInverseBlock(block=ws.covmat_block) ws.retrievalDefClose() def oem(self, method='li', max_iter=10, stop_dx=0.01, lm_ga_settings=None, display_progress=True, inversion_iterate_agenda=None): """ Run the optimal estimation. See Arts documentation for details. :param method: :param max_iter: :param stop_dx: :param lm_ga_settings: Default: [10, 2, 2, 100, 1, 99] :param display_progress: :param inversion_iterate_agenda: If set to None, a simple default agenda is used. :return: """ ws = self.ws if lm_ga_settings is None: lm_ga_settings = [100.0, 2.0, 2.0, 10.0, 1.0, 1.0] lm_ga_settings = np.array(lm_ga_settings) # x, jacobian and yf must be initialised ws.x = np.array([]) ws.yf = np.array([]) ws.jacobian = np.array([[]]) if inversion_iterate_agenda is None: inversion_iterate_agenda = boilerplate.inversion_iterate_agenda ws.Copy(ws.inversion_iterate_agenda, inversion_iterate_agenda) ws.AgendaExecute(ws.sensor_response_agenda) # a priori values self._check_set_wind() ws.xaStandard() xa = ws.xa.value for rq in self.retrieval_quantities: rq.extract_apriori(xa) # Run inversion try: ws.OEM(method=method, max_iter=max_iter, stop_dx=stop_dx, lm_ga_settings=lm_ga_settings, display_progress=1 if display_progress else 0) except Exception as e: raise OemException(e) if not self.oem_converged: # Just checks if dxdy is initialized return False ws.x2artsAtmAndSurf() ws.x2artsSensor() ws.avkCalc() ws.covmat_ssCalc() ws.covmat_soCalc() ws.retrievalErrorsExtract() x = ws.x.value avk = ws.avk.value eo = ws.retrieval_eo.value es = ws.retrieval_ss.value for rq in self.retrieval_quantities: rq.extract_result(x, avk, eo, es) return True def get_level2_xarray(self): ds = xr.merge([rq.to_xarray() for rq in self.retrieval_quantities]) # Spectra f_backend = self._sensor.f_backend if self._sensor.f_backend is not None else self.f_grid ds['f'] = ('f', f_backend) ds['y'] = (('observation', 'f'), np.stack(self.y)) ds['yf'] = (('observation', 'f'), np.stack(self.yf)) ds['oem_diagnostics'] = ('oem_diagnostics_idx', self.oem_diagnostics) y_baseline = self.y_baseline if y_baseline is not None: ds['y_baseline'] = (('observation', 'f'), np.stack(self.y_baseline)) # Observations for f in Observation._fields: values = np.array([getattr(obs, f) for obs in self._observations], dtype=np.float) ds['obs_'+f] = ('observation', values) # Global attributes ds.attrs['arts_version'] = self.arts_version ds.attrs['uuid'] = str(uuid.uuid4()) ds.attrs['date_created'] = datetime.datetime.utcnow().isoformat() ds.attrs['nodename'] = os.uname().nodename return ds @property def p_grid(self): return self.ws.p_grid.value @property def lat_grid(self): return self.ws.lat_grid.value @property def lon_grid(self): return self.ws.lon_grid.value @property def f_grid(self): return self.ws.f_grid.value @property def n_p(self): return len(self.ws.p_grid.value) @property def n_lat(self): if self.atmosphere_dim == 1: return 1 return len(self.ws.lat_grid.value) @property def n_lon(self): if self.atmosphere_dim == 1: return 1 return len(self.ws.lat_grid.value) @property def n_y(self): return len(self.ws.y.value) @property def y(self): y = np.copy(self.ws.y.value) return np.split(y, self.n_obs) @property def yf(self): yf = np.copy(self.ws.yf.value) return np.split(yf, self.n_obs) @property def y_baseline(self): if self.ws.y_baseline.initialized: bl = np.copy(self.ws.y_baseline.value) if bl.size == 1 and bl[0] == 0: return None return np.split(bl, self.n_obs) else: return None @property def n_obs(self): if self.ws.sensor_time.initialized: return len(self.ws.sensor_time.value) else: return 0 @property def abs_species_maintags(self): abs_species = self.ws.abs_species.value maintags = [st[0].split('-')[0] for st in abs_species] return maintags @property def atmosphere_dim(self): return self.ws.atmosphere_dim.value @property def oem_converged(self): return self.ws.oem_diagnostics.value[0] == 0 @property def oem_diagnostics(self): return self.ws.oem_diagnostics.value @property def oem_errors(self): return self.ws.oem_errors.value @property def arts_version(self): cmd = os.path.join(os.environ['ARTS_BUILD_PATH'], 'src/arts') output = os.popen(cmd + ' --version').read().splitlines() return output[0]
class oem_retrieval(): """ AUTHOR: Hayden Smotherman DESCRIPTION: This class is meant to run an OEM retrieval on data from the radiometers at MPI Solar System. The hope is that it simplifies the process of using the typhon OEM retreival for the specific use-case of Paul Hartogh's group at MPI Solar System. It can load in the data, attempt to fit and subtract-out sine curves from the noise, and filter high-frequency oscillations that are present in some data. It can also run a retrieval using the typhon OEM retrieval and plot the results, along with relevant statistical metrics. """ def __init__(self): """ AUTHOR: Hayden Smotherman DESCRIPTION: This function initializes the oem_retrieval class INPUTS: NONE OUTPUTS: NONE """ self.signal = np.array( []) # This array will hold the brightness temperature data self.noise = np.array( []) # This array will hold the noise for the radiometer self.freq = np.array( []) # This array will hold the frequencies of "signal" and "noise" def load_data(self, data_type='radiometer', use_data=0, filename='', signal=[], noise=[], freq=[]): """ AUTHOR: Hayden Smotherman DESCRIPTION: This function loads in radiometer data to the oem_retrival class. It accepts data in a number of different formats. INPUTS: data_type - A string denoting which format the input data will be in. VALUES: 'numpy' - Regular numpy arrays passed in to "signal", "noise", and "freq" REQUIRES: signal, noise, freq 'radiometer' - A .raw file generated by a radiometer at MPI Solar System. REQUIRES: filename use_data - Integer value denoting how much data to use when running a mean OEM retrieval. This value should be zero (use all data) or negative (use the last x signals) signal - An array of the brightness temperatures of an O3-666 line. ONLY USED WITH data_type='numpy' noise - An array of the noise of the radiometer ONLY USED WITH data_type='numpy' freq - An array of the central frequency of each datapoint in "signal" and "noise" ONLY USED WITH data_type='numpy' OUTPUTS: """ eform = '>i16f7500f100f28f28f' dform = '>i16f4092f100f28f28f' aform = '>i16f16384f128f28f28f' xform = '>i16f1019f128f28f28f' if data_type == 'radiometer': # Load data from a .raw file from the 210MHz radiometer # Load in the raw data if filename[0] == 'a': data_format = aform elif filename[0] == 'd': data_format = dform elif filename[0] == 'e': data_format = eform elif filename[0] == 'x': data_format = xform else: raise ImportError( 'Could not understand the data format based on the file name.' ) Calibrated = calibration(format=data_format) Calibrated.load(filename) Calibrated.calibrate() # Pre-allocate signal and noise arrays self.signal = np.array(Calibrated._signal[use_data:]) self.noise = np.array(Calibrated._noise[use_data:]) # Generate the frequency array based on the data format if filename[0] == 'a': # First we split the data, since two signals are embedded in the ._signal array # NOTE: Currently this only keeps the first half of the data. It should keep all of it. self.signal = Calibrated._signal[use_data:] self.noise = Calibrated._noise[use_data:] Half_Length = int(len(Calibrated._noise[0]) / 2) # Data is set as one long array for i in range(len(self.signal)): self.signal[i] = np.copy( Calibrated._signal[i][0:Half_Length]) self.noise[i] = np.copy( Calibrated._noise[i][0:Half_Length]) self.signal = np.array(self.signal) self.noise = np.array(self.noise) # Width of data is 1.5GHz # Central frequency is data number 14 in _raw array, requested frequency Min_Freq = Calibrated._raw[0][14] - 1.50 / 2 Max_Freq = Calibrated._raw[0][14] + 1.50 / 2 self.freq = np.linspace(Min_Freq, Max_Freq, Half_Length) elif filename[0] == 'd': # Width of data is 40MHz # Central frequency is data number 14 in _raw array, requested frequency Min_Freq = Calibrated._raw[0][14] - .040 / 2 Max_Freq = Calibrated._raw[0][14] + .040 / 2 self.freq = np.linspace(Min_Freq, Max_Freq, len(Calibrated._noise[0])) elif filename[0] == 'e': # Width of data is 210MHz # Central frequency is data number 14 in _raw array, requested frequency Min_Freq = Calibrated._raw[0][14] - .210 / 2 Max_Freq = Calibrated._raw[0][14] + .210 / 2 self.freq = np.linspace(Min_Freq, Max_Freq, len(Calibrated._noise[0])) elif filename[0] == 'x': # Width of data is 4.4GHz # Central frequency is data number 14 in _raw array, requested frequency Min_Freq = Calibrated._raw[0][14] - 4.4 / 2 Max_Freq = Calibrated._raw[0][14] + 4.4 / 2 self.freq = np.linspace(Min_Freq, Max_Freq, len(Calibrated._noise[0])) else: raise ImportError( 'Could not generate frequency array. Unknown file format.') Calibrated = None elif data_type == 'numpy': # Load in arbitrary data that is already in a numpy array self.signal = np.array(signal) self.noise = np.array(noise) self.freq = np.array(freq) # Define fit_ values here in case subtract_sinewave is never called self.fit_freq = np.copy(self.freq) self.fit_signal = np.copy(self.signal) self.fit_noise = np.copy(self.noise) def process_data(self, lims=[21, 23], shift_freq=False, fit_sine=False, plot_example=True, initial_guess=[3, 1 / 1000, 0], central_freq=1.42175037e+2): """ AUTHOR: Hayden Smotherman DESCRIPTION: This function clips data from the front and back of the dataset based on front_lim and back_lim. If "fit_sine=True", then it also subtracts out sine waves from the data. INPUTS: lims - Optional frequency limits (in GHz) that determine the range of data to use. shift_freq - Boolean that determines whether or not to shift the frequency such that the max of the averaged data aligns with the peak of the O3 signal. fit_sine - Boolean that determines whether or not to fit a sine wave to the noise then subtract it out of the signal. plot_example - Boolean that determines whether or not to plot an example fit. Only used if fit_sine=True initial_guess - This is the initial guess for the sine parameters used by lsqfit. Only used if fit_sine=True central_freq - This is the frequency center of the measurements. Only used if shift_freq=True OUTPUTS: NONE NOTES: - This function works best for data in dform and eform. Datasets with longer frequency baselines are not fit as well and may give bad results. - By default, "process_data" clips the first 25 data points regardless of what is set in "lims". This is done because the first ~20 data points are often severe outliers. """ # Calculate the index corresponding to the minimum frequency limit if shift_freq: if len(self.signal.shape) > 1: Average_Signal = np.mean(self.signal, axis=0) else: Average_Signal = np.copy(self.signal) Freq_Peak = self.freq[Average_Signal == np.max(Average_Signal)] Freq_Difference = Freq_Peak - central_freq self.freq -= Freq_Difference front_lim = np.where( self.freq == np.min(self.freq[self.freq > lims[0]])) front_lim = int(front_lim[0]) if (front_lim < 25): # Set front_lim minimum to 25, since the first ~25 data points are very often bad front_lim = 25 # Calculate the index corresponding to the maximum frequency limit back_lim = np.where( self.freq == np.max(self.freq[self.freq < lims[1]])) back_lim = int(back_lim[0]) Signal_Length = self.signal.shape[1] # Length of the signal array Signal_Number = self.signal.shape[ 0] # Number of signals in the data file X_Data = np.linspace( 0, Signal_Length - 1, Signal_Length) # Array of indicies for the fitting X_Data = X_Data[front_lim:back_lim] # Preallocate the memory for the fitted values of signal and noise self.fit_signal = np.zeros([ self.signal.shape[0], np.size(self.signal[0][front_lim:back_lim]) ]) self.fit_noise = np.zeros( [self.noise.shape[0], np.size(self.noise[0][front_lim:back_lim])]) self.fit_freq = np.copy(self.freq[front_lim:back_lim]) if fit_sine: for i in range(Signal_Number): # Iterate over all signals and subtract out the best-fit sine curve # Load in noise locally for ease of use Noise_0 = self.noise[i] # Limit the data based on front_lim and back_lim and subtract the mean Noise_0 = Noise_0[front_lim:back_lim] Noise_Median = np.median(Noise_0) Base_Noise = Noise_0 - Noise_Median # Subtract out the noise mean for fitting Sine_Params = initial_guess # Amplitude, freq, phase # Sine function optimize_func = lambda x: x[0] * np.sin(x[1] * X_Data + x[2] ) - Base_Noise # Perform least square fit to the sine function est_amp, est_freq, est_phase = leastsq(optimize_func, Sine_Params)[0] # Save the fit Fit = est_amp * np.sin(est_freq * X_Data + est_phase) # Load in signal locally Signal_0 = self.signal[i] Signal_0 = Signal_0[front_lim:back_lim] # Subtract the fit from the signal and the noise and save it in new variables #print(np.size(Signal_0), np.size(Fit)) self.fit_signal[i] = Signal_0 - Fit self.fit_noise[i] = Noise_0 - Fit if plot_example: # Plot the last "noise" value along with the fit sine curve plt.figure(figsize=[12, 8]) plt.plot(self.fit_freq, Noise_0) plt.plot(self.fit_freq, Fit + Noise_Median) plt.legend(['Unfitted Noise', 'Best Fit Sine Curve'], fontsize=20) plt.xlabel('Frequency [GHz]', fontsize=20) plt.ylabel('Brightness Temperature [K]', fontsize=20) plt.title('Raw Noise and Best Fit Curve', fontsize=24) # Plot the fitted noise over the raw noise plt.figure(figsize=[12, 8]) plt.plot(self.fit_freq, self.noise[-1][front_lim:back_lim]) plt.plot(self.fit_freq, self.fit_noise[-1]) plt.legend(['Unfitted Noise', 'Fitted Noise'], fontsize=20) plt.xlabel('Frequency [GHz]', fontsize=20) plt.ylabel('Brightness Temperature [K]', fontsize=20) plt.title('Unfitted Noise and Fitted Noise', fontsize=24) # Plot the fitted signal over the raw signal plt.figure(figsize=[12, 8]) plt.plot(self.fit_freq, self.signal[-1][front_lim:back_lim]) plt.plot(self.fit_freq, self.fit_signal[-1]) plt.legend(['Unfitted Signal', 'Fitted Signal'], fontsize=20) plt.xlabel('Frequency [GHz]', fontsize=20) plt.ylabel('Brightness Temperature [K]', fontsize=20) plt.title('Unfitted Signal and Fitted Signal', fontsize=24) else: for i in range(Signal_Number): self.fit_signal[i] = self.signal[i][front_lim:back_lim] self.fit_noise[i] = self.noise[i][front_lim:back_lim] def mean_retrieval(self, use_data=0, filtered=False, shift_freq=False, sigma=None, central_freq=1.42175037e+2): """ AUTHOR: Hayden Smotherman DESCRIPTION: This function uses the Typhon OEM retrieval package to recover atmospheric O3 levels given the brightness temperature signal of the O3-666 line. If self.signal is a 2D matrix, then this function will run the retrieval on the mean of this data. INPUTS: filtered - Boolean that determines whether or not to use the scipy.signal.filtfilt function on the mean signal value in order to potentially improve the signal shift_freq - Boolean that determines whether or not to shift the frequency such that the max of the averaged data aligns with the peak of the O3 signal. central_freq - This is the frequency center of the measurements. Only used if shift_freq=True OUTPUTS: NONE """ if len(self.signal.shape) > 1: self.average_signal = np.mean(self.fit_signal, axis=0) self.average_noise = np.mean(self.fit_noise, axis=0) else: self.average_signal = np.copy(self.fit_signal) self.average_noise = np.copy(self.fit_noise) if filtered: n = 4 # the larger n is, the smoother the curve will be b = [1.0 / n] * n a = 1 self.average_signal = sg.filtfilt(b, a, self.average_signal) self.average_noise = sg.filtfilt(b, a, self.average_noise) if shift_freq: Freq_Peak = self.fit_freq[self.average_signal == np.max( self.average_signal)] Freq_Difference = Freq_Peak - central_freq self.fit_freq -= Freq_Difference self._initialize_arts_workspace() if sigma is None: self.sigma = np.sqrt( np.sum( np.abs(self.average_noise - np.mean(self.average_noise))) / len(self.average_noise)) else: self.sigma = sigma self._initialize_covmat() self._run_retrieval() def plot_oem(self, basename='', plot_results=True, plot_statistics=True): """ AUTHOR: Hayden Smotherman DESCRIPTION: This function uses the Typhon OEM retrieval package to recover atmospheric O3 levels given the brightness temperature signal of the O3-666 line. If self.signal is a 2D matrix, then this function will run the retrieval on the mean of this data. INPUTS: plot_results - Boolean that determines whether or not to plot the results of the retrieval. plot_statistics - Boolean that determines whether or not to plot relevant statistics from the retrieval. OUTPUTS: NONE """ if plot_results: # Plot the retrieved signal and the retrieval values # First plot the averaged signal and the retrieved signal plt.figure(1, figsize=[12, 8]) plt.clf() plt.plot(self.fit_freq, self.average_signal) plt.plot(self.arts.f_grid.value / 1e9, self.arts.yf.value, 'r') plt.xlabel('Frequency [GHz]', fontsize=20) plt.ylabel('Brightness Temperature [K]', fontsize=20) plt.legend(['Data', 'Retrieval'], fontsize=20) plt.title('Average Signal and Retrieved Signal', fontsize=24) plt.savefig(basename + 'fig1.png') # Now plot the actual retrieved Ozone VMR Altitude = self.arts.z_field.value.flatten() plt.figure(2, figsize=[8, 12]) plt.clf() plt.plot(10**self.arts.xa.value[:-1] * 1e6, Altitude) plt.plot(10**self.arts.x.value[:-1] * 1e6, Altitude) plt.legend(['Prior', 'OEM Retrieval'], fontsize=20) plt.ylabel('Altitude [m]', fontsize=20) plt.xlabel('O3 [ppmv]', fontsize=20) plt.title('Altitude vs. O3 VMR', fontsize=24) plt.savefig(basename + 'fig2.png') plt.figure(1, figsize=[12, 8]) plt.clf() plt.plot(self.fit_freq, self.average_signal - self.arts.yf.value, 'r') plt.xlabel('Frequency [GHz]', fontsize=20) plt.ylabel('Residual [K]', fontsize=20) plt.title('Residual Signal', fontsize=24) plt.savefig(basename + 'fig5.png') if plot_statistics: # Plot some relevant statistics of the OEM retrieval Altitude = self.arts.z_field.value.flatten() averaging_kernel = self.arts.dxdy.value @ self.arts.jacobian.value measurement_response = averaging_kernel @ np.ones( averaging_kernel.shape[1]) plt.figure(3, figsize=[12, 8]) plt.clf() [plt.plot(kernel[:-1], Altitude) for kernel in averaging_kernel.T] plt.title('Averaging kernel for the OEM retrieval', fontsize=24) plt.ylabel('Altitude [m]', fontsize=20) plt.savefig(basename + 'fig3.png') plt.figure(4, figsize=[12, 8]) plt.clf() plt.plot(measurement_response[:-1], Altitude) plt.title('Measurement response for the OEM retrieval', fontsize=24) plt.ylabel('Alititude [m]', fontsize=20) plt.savefig(basename + 'fig4.png') def _initialize_arts_workspace(self): """ AUTHOR: Richard Larsson, Hayden Smotherman DESCRIPTION: This function is meant to be run interally to this class. It initializes the arts workspace. INPUTS: NONE OUTPUTS: NONE """ # -*- coding: utf-8 -*- """ Created on Thu Jul 5 15:53:02 2018 @author: larsson """ xmls = typhon.environ.get('ARTS_DATA_PATH') self.arts = Workspace(0) @arts_agenda def water_psat_agenda(ws): ws.water_p_eq_fieldMK05() @arts_agenda def propmat_clearsky_agenda_zeeman(ws): ws.propmat_clearskyInit() ws.propmat_clearskyAddOnTheFly() ws.Ignore(ws.rtp_mag) ws.Ignore(ws.rtp_los) @arts_agenda def ppath_agenda_step_by_step(ws): ws.Ignore(ws.rte_pos2) ws.ppathStepByStep() @arts_agenda def iy_main_agenda_emission(ws): ws.Ignore(ws.iy_id) ws.ppathCalc() ws.iyEmissionStandard() @arts_agenda def iy_space_agenda_cosmic_background(ws): ws.Ignore(ws.rtp_pos) ws.Ignore(ws.rtp_los) ws.MatrixCBR(ws.iy, ws.stokes_dim, ws.f_grid) @arts_agenda def iy_surface_agenda(ws): ws.SurfaceDummy() ws.iySurfaceRtpropAgenda() @arts_agenda def ppath_step_agenda_geometric(ws): ws.Ignore(ws.t_field) ws.Ignore(ws.vmr_field) ws.Ignore(ws.f_grid) ws.Ignore(ws.ppath_lraytrace) ws.ppath_stepGeometric() @arts_agenda def abs_xsec_agenda_lines(ws): ws.abs_xsec_per_speciesInit() ws.abs_xsec_per_speciesAddLines2() ws.abs_xsec_per_speciesAddConts() @arts_agenda def surface_rtprop_agenda(ws): ws.InterpSurfaceFieldToPosition(out=ws.surface_skin_t, field=ws.t_surface) ws.surfaceBlackbody() @arts_agenda def geo_pos_agenda(ws): ws.Ignore(ws.ppath) ws.VectorSet(ws.geo_pos, np.array([])) @arts_agenda def sensor_response_agenda(ws): ws.AntennaOff() ws.sensorOff() ws.Ignore(ws.f_backend) # Set some agendas self.arts.Copy(self.arts.surface_rtprop_agenda, surface_rtprop_agenda) self.arts.Copy(self.arts.abs_xsec_agenda, abs_xsec_agenda_lines) self.arts.Copy(self.arts.ppath_step_agenda, ppath_step_agenda_geometric) self.arts.Copy(self.arts.propmat_clearsky_agenda, propmat_clearsky_agenda_zeeman) self.arts.Copy(self.arts.iy_main_agenda, iy_main_agenda_emission) self.arts.Copy(self.arts.iy_space_agenda, iy_space_agenda_cosmic_background) self.arts.Copy(self.arts.ppath_agenda, ppath_agenda_step_by_step) self.arts.Copy(self.arts.iy_surface_agenda, iy_surface_agenda) self.arts.Copy(self.arts.geo_pos_agenda, geo_pos_agenda) self.arts.Copy(self.arts.water_p_eq_agenda, water_psat_agenda) self.arts.Copy(self.arts.sensor_response_agenda, sensor_response_agenda) # Set some quantities that are unused because you do not need them (for now) self.arts.Touch(self.arts.surface_props_data) self.arts.Touch(self.arts.surface_props_names) self.arts.Touch(self.arts.mag_u_field) self.arts.Touch(self.arts.mag_v_field) self.arts.Touch(self.arts.mag_w_field) self.arts.Touch(self.arts.wind_u_field) self.arts.Touch(self.arts.wind_v_field) self.arts.Touch(self.arts.wind_w_field) self.arts.Touch(self.arts.transmitter_pos) self.arts.Touch(self.arts.iy_aux_vars) self.arts.Touch(self.arts.mblock_dlos_grid) self.arts.Touch(self.arts.scat_species) # Ozone line and continua self.arts.abs_cont_descriptionInit() self.arts.abs_cont_descriptionAppend(tagname="H2O-PWR98", model="Rosenkranz") self.arts.abs_cont_descriptionAppend(tagname="O2-PWR98", model="Rosenkranz") self.arts.abs_cont_descriptionAppend(tagname="O2-PWR98", model="Rosenkranz") self.arts.abs_cont_descriptionAppend(tagname="N2-CIArotCKDMT252", model="CKDMT252") self.arts.abs_cont_descriptionAppend(tagname="N2-CIAfunCKDMT252", model="CKDMT252") self.arts.abs_speciesSet(species=[ 'O2-PWR98', 'H2O-PWR98', 'N2-CIAfunCKDMT252, N2-CIArotCKDMT252' ]) self.arts.abs_linesReadFromSplitArtscat( basename=os.path.join(xmls, 'spectroscopy/Perrin/'), fmin=np.min(self.fit_freq) * 1e9 - 1e8, fmax=np.max(self.fit_freq) * 1e9 + 1e8) self.arts.abs_lines_per_speciesCreateFromLines() # Set builtin Earth-viable isotopologue values and partition functions self.arts.isotopologue_ratiosInitFromBuiltin() self.arts.partition_functionsInitFromBuiltin() self.arts.nlteOff() # LTE self.arts.stokes_dim = 1 # No polarization self.arts.xsec_speedup_switch = 0 # No speedup (experimental feature) self.arts.rte_alonglos_v = 0. # No movement of satellite or rotation of planet self.arts.lm_p_lim = 0. # Just do line mixing if available (it is not) self.arts.abs_f_interp_order = 1 # Interpolation in frequency if you add a sensor self.arts.ppath_lmax = 10000. # Maximum path length (Original) self.arts.ppath_lraytrace = 10000. # Maximum path trace (Original) self.arts.refellipsoidEarth(model="Sphere") # Europa average radius self.arts.iy_unit = "PlanckBT" # Output results in Planck Brightess Temperature # Set the size of the problem (change to your own numbers) NP = 201 # Number of pressure levels NF = len(self.fit_freq) self.arts.lon_true = np.array([0]) self.arts.lat_true = np.array([0]) self.arts.AtmosphereSet1D() self.arts.p_grid = np.logspace(5.04, -1.2, NP) self.arts.z_surface = np.zeros((1, 1)) self.arts.t_surface = np.full((1, 1), 295.) self.arts.f_grid = np.linspace( np.min(self.fit_freq) * 1e9, np.max(self.fit_freq) * 1e9, NF) self.arts.sensorOff() # No sensor simulations # Read the atmosphere... folder should contain: # "H2O.xml" # "t.xml" # "z.xml" # The files can be in binary format self.arts.AtmRawRead( basename=xmls + 'planets/Earth/Fascod/subarctic-summer/subarctic-summer') self.arts.AtmFieldsCalc() # Set observation geometry... You can make more positions and los self.arts.sensor_pos = np.array([[10000] ]) # [[ALT, LAT, LON]] (Original) self.arts.sensor_los = np.array([[63]]) # [[ZENITH, AZIMUTH]] # Temperature and Ozone VMR Jacobian self.arts.jacobianInit() self.arts.jacobianAddTemperature(g1=self.arts.p_grid.value, g2=np.array([]), g3=np.array([])) self.arts.jacobianAddAbsSpecies(g1=self.arts.p_grid.value, g2=np.array([]), g3=np.array([]), species='H2O-PWR98') self.arts.jacobianClose() self.arts.cloudboxOff() # No Clouds # Check that the input looks OK self.arts.atmgeom_checkedCalc() self.arts.atmfields_checkedCalc() self.arts.cloudbox_checkedCalc() self.arts.sensor_checkedCalc() self.arts.propmat_clearsky_agenda_checkedCalc() self.arts.abs_xsec_agenda_checkedCalc() # Perform the calculations! self.arts.yCalc() def _initialize_covmat(self): """ AUTHOR: Simon Pfreundschuh, Hayden Smotherman DESCRIPTION: This function is meant to be run interally to this class. It initializes the covariance matricies. INPUTS: NONE OUTPUTS: NONE """ z_grid = self.arts.z_field.value.flatten() n_p = self.arts.p_grid.value.size self.arts.retrievalDefInit() # Kernel panic if 'grid_1' and 'sigma_1' are not the same size self.arts.covmat1D( self.arts.covmat_block, grid_1=z_grid, sigma_1=1e-7 * np.ones(n_p), # Relative uncertainty cls_1=1.0e3 * np.ones(n_p), # Correlation length [m] fname="exp") self.arts.retrievalAddAbsSpecies(species="H2O-PWR98", unit="vmr", atmosphere_dim=1, g1=self.arts.p_grid, g2=np.array([]), g3=np.array([])) self.arts.jacobianSetFuncTransformation(transformation_func="none") self.arts.covmatDiagonal(out=self.arts.covmat_block, out_inverse=self.arts.covmat_inv_block, vars=100.0 * np.ones(1)) self.arts.retrievalAddPolyfit(poly_order=0) # self.arts.covmatDiagonal(out = self.arts.covmat_block, # out_inverse = self.arts.covmat_inv_block, # vars = 100.0 * np.ones(8*2)) # self.arts.retrievalAddSinefit(period_lengths = np.array([10e3, 20e3, 1e6, 2e6, 5e6, 10e6, 20e6, 1e9])) self.arts.retrievalDefClose() # More uncertainty measurements self.arts.covmatDiagonal(self.arts.covmat_block, self.arts.covmat_inv_block, vars=self.sigma**2 * np.ones(self.arts.y.value.shape)) self.arts.covmat_seSet(self.arts.covmat_block) self.arts.jacobianAdjustAndTransform # Kernel panic if 'arts.Ignore(arts.inversion_iteration_counter)' is not included @arts_agenda def inversion_iterate_agenda(arts): arts.Ignore(arts.inversion_iteration_counter) arts.x2artsAtmAndSurf() arts.Copy(arts.f_backend, arts.f_grid) arts.x2artsSensor() arts.atmfields_checkedCalc( negative_vmr_ok=1 ) # negative_vmr_ok added to avoid error with negative vmr values arts.atmgeom_checkedCalc() arts.yCalc() arts.jacobianAdjustAndTransform() arts.VectorAddVector(arts.yf, arts.y, arts.y_baseline) self.arts.Copy(self.arts.inversion_iterate_agenda, inversion_iterate_agenda) def _run_retrieval(self): """ AUTHOR: Simon Pfreundschuh, Hayden Smotherman DESCRIPTION: This function is meant to be run interally to this class. It runs the actual retrieval. INPUTS: NONE OUTPUTS: NONE """ # Run the OEM retrieval self.arts.Touch(self.arts.particle_bulkprop_field) self.arts.Touch(self.arts.particle_bulkprop_names) self.arts.VectorSetConstant(self.arts.sensor_time, 1, 0.) self.arts.xaStandard() self.arts.x = np.zeros(0) self.arts.y.value[:] = self.average_signal self.arts.jacobian = np.zeros((0, 0)) self.arts.OEM(method="li", max_iter=20, display_progress=1, lm_ga_settings=np.array( [100.0, 5.0, 2.0, 10.0, 1.0, 1.0])) self.arts.avkCalc()