def process(config, obs):
"""
Returns a numpy array of obs for use in mosx_predictors. The first dimension is date; all other dimensions are
serialized.
:param config:
:param obs: dict: dictionary of processed obs data
:return:
"""
if config['verbose']:
print('obs.process: processing array for obs data...')
# Surface observations
sfc = obs['SFC']
num_days = len(sfc.keys())
variables = sorted(sfc[sfc.keys()[0]].keys())
sfc_array = get_array(sfc)
sfc_array_r = np.reshape(sfc_array, (num_days, -1))
# Sounding observations
if config['Obs']['use_soundings']:
sndg_array = get_array(obs['SNDG'])
# num_days should be the same first dimension
sndg_array_r = np.reshape(sndg_array, (num_days, -1))
return np.hstack((sfc_array_r, sndg_array_r))
else:
return sfc_array_r
def process(config, verif_list):
"""
Returns a numpy array of verification data for use in mosx_predictors. The first dimension is date, the second is
variable.
:param config:
:param verif: dict: dictionary of processed verification data; may be None
:return: ndarray: array of processed verification targets
"""
verif_arrays = []
if verif_list is not None:
for i in range(len(verif_list)):
verif = verif_list[i]
print('verification.process: processing array for verification data')
num_days = len(verif.keys())
variables = ['Tmax', 'Tmin', 'Wind', 'Rain']
day_verif_array = np.full((num_days, len(variables)), np.nan, dtype=np.float64)
for d in range(len(verif.keys())):
date = list(verif.keys())[d]
for v in range(len(variables)):
var = variables[v]
day_verif_array[d, v] = verif[date][var]
if config['Model']['predict_timeseries']:
hour_verif = OrderedDict(verif)
for date in hour_verif.keys():
for variable in variables:
hour_verif[date].pop(variable, None)
hour_verif_array = get_array(hour_verif)
hour_verif_array = np.reshape(hour_verif_array, (num_days, -1))
verif_array = np.concatenate((day_verif_array, hour_verif_array), axis=1)
else:
verif_array = day_verif_array
verif_arrays.append(verif_array)
return verif_arrays
else:
return None
def process(config, bufr, advection_diagnostic=True):
"""
Imports the data contained in a bufr dictionary and returns a time-by-x numpy array for use in mosx_predictors. The
first dimension is date; all other dimensions are first extracted using get_array and then one-dimensionalized.
:param config:
:param bufr: dict: dictionary of processed BUFR data
:param advection_diagnostic: bool: if True, add temperature advection diagnostic to the data
:return: ndarray: array of formatted BUFR predictor values
"""
if config['verbose']:
print('bufr.process: processing array for BUFR data...')
# PROF part of the BUFR data
bufr_prof = get_array(bufr['PROF'])
bufr_dims = list(range(len(bufr_prof.shape)))
bufr_dims[0] = 1
bufr_dims[1] = 0
bufr_prof = bufr_prof.transpose(bufr_dims)
bufr_shape = bufr_prof.shape
bufr_reshape = [bufr_shape[0]] + [np.cumprod(bufr_shape[1:])[-1]]
bufr_prof = bufr_prof.reshape(tuple(bufr_reshape))
# SFC part of the BUFR data
bufr_sfc = get_array(bufr['SFC'])
bufr_dims = list(range(len(bufr_sfc.shape)))
bufr_dims[0] = 1
bufr_dims[1] = 0
bufr_sfc = bufr_sfc.transpose(bufr_dims)
bufr_shape = bufr_sfc.shape
bufr_reshape = [bufr_shape[0]] + [np.cumprod(bufr_shape[1:])[-1]]
bufr_sfc = bufr_sfc.reshape(tuple(bufr_reshape))
# DAY part of the BUFR data
bufr_day = get_array(bufr['DAY'])
bufr_dims = list(range(len(bufr_day.shape)))
bufr_dims[0] = 1
bufr_dims[1] = 0
bufr_day = bufr_day.transpose(bufr_dims)
bufr_shape = bufr_day.shape
bufr_reshape = [bufr_shape[0]] + [np.cumprod(bufr_shape[1:])[-1]]
bufr_day = bufr_day.reshape(tuple(bufr_reshape))
bufr_out = np.concatenate((bufr_prof, bufr_sfc, bufr_day), axis=1)
# Fix missing values
bufr_out[bufr_out < -1000.] = np.nan
if advection_diagnostic:
advection_array = temp_advection(bufr)
bufr_out = np.concatenate((bufr_out, advection_array), axis=1)
return bufr_out
def process(config, obs_list):
"""
Returns a numpy array of obs for use in mosx_predictors. The first dimension is date; all other dimensions are
serialized.
:param config:
:param obs_list: list of dictionaries of processed obs data for the different stations
:return:
"""
# Surface observations
if config['verbose']:
print('obs.process: processing array for obs data...')
for i in range(len(obs_list)):
obs = obs_list[i]
try:
sfc = obs['SFC']
except KeyError:
sfc = obs[b'SFC']
num_days = len(sfc.keys())
variables = sorted(sfc[list(sfc.keys())[0]].keys())
sfc_array = get_array(sfc)
sfc_array_r = np.reshape(sfc_array, (num_days, -1))
# Sounding observations
if config['Obs']['use_soundings']:
try:
sndg_array = get_array(obs['SNDG'])
except KeyError:
sndg_array = get_array(obs[b'SNDG'])
# num_days should be the same first dimension
sndg_array_r = np.reshape(sndg_array, (num_days, -1))
obs_one_array = np.hstack((sfc_array_r, sndg_array_r))
if i == 0: #first station
obs_array = obs_one_array
else:
obs_array = np.hstack((obs_array, obs_one_array))
else:
if i == 0: #first station
obs_array = sfc_array_r
else:
obs_array = np.hstack((obs_array, sfc_array_r))
return obs_array