def break_points_timestamp(self, epoch):
"""Transform breaking point value into a date of the from YYYYMMDD """
converter = TimeBaseDays(epoch)
return numpy.array([
'{0.year:4d}-{0.month:02d}-{0.day:02d}'.format(
converter.number_to_datetime(int(number)))
for number in self.break_time
]) #, dtype='datetime64[D]')
def early_look_grid_batch(
storage_climatology, storage_large_scale, storage_local, outputfiles,
time_indices, covariates_descriptor, insitu_biases, breakpoints_file,
global_biases, global_biases_group_list, compute_uncertainties, method,
compute_sample, sample_size, compute_prior_sample):
"""Produce 'early look' NetCDF output files without loading or gridding uncertainty information
For inspection of analysis output prior to the final gridding step.
"""
from eustace.analysis.advanced_standard.fileio.output_projector import Projector
print 'VERSION: {0}'.format(get_revision_id_for_module(eustace))
# Build analysis system
analysissystem = AnalysisSystem_EUSTACE(
storage_climatology, storage_large_scale, storage_local,
covariates_descriptor, insitu_biases, breakpoints_file, global_biases,
global_biases_group_list, compute_uncertainties, method)
grid_resolution = [
180. / definitions.GLOBAL_FIELD_SHAPE[1],
360. / definitions.GLOBAL_FIELD_SHAPE[2]
]
latitudes = numpy.linspace(-90. + grid_resolution[0] / 2.,
90. - grid_resolution[0] / 2,
num=definitions.GLOBAL_FIELD_SHAPE[1])
longitudes = numpy.linspace(-180. + grid_resolution[1] / 2.,
180. - grid_resolution[1] / 2,
num=definitions.GLOBAL_FIELD_SHAPE[2])
timebase = TimeBaseDays(eustace.timeutils.epoch.EPOCH)
processdates = [
timebase.number_to_datetime(daynumber) for daynumber in time_indices
]
cell_sampling = [1, 1]
blocking = 10
# thinned set of sample indices for inclusion in output product
climatology_projector = None
large_scale_projector = None
local_projector = None
for (inner_index, time_index, processdate) in zip(range(len(time_indices)),
time_indices,
processdates):
print time_index
# Configure output grid
outputstructure = OutputRectilinearGridStructure(time_index,
processdate,
latitudes=latitudes,
longitudes=longitudes)
# climatology component
print 'Evaluating: climatology'
if climatology_projector is None:
climatology_projector = Projector(latitudes, longitudes,
grid_resolution, time_index,
cell_sampling, blocking)
climatology_projector.set_component(analysissystem.components[0])
climatology_projector.update_time_index(time_index, keep_design=False)
climatology_projector.evaluate_design_matrix()
climatology_expected_value = climatology_projector.project_expected_value(
).reshape((-1, 1))
# large scale component
print 'Evaluating: large-scale'
if large_scale_projector is None:
large_scale_projector = Projector(latitudes, longitudes,
grid_resolution, time_index,
cell_sampling, blocking)
large_scale_projector.set_component(analysissystem.components[1])
large_scale_projector.update_time_index(time_index, keep_design=False)
large_scale_projector.evaluate_design_matrix()
large_scale_expected_value = large_scale_projector.project_expected_value(
).reshape((-1, 1))
# local component - time handling updates state to new time but does not recompute the design matrix
print 'Evaluating: local'
if local_projector is None:
local_projector = Projector(latitudes, longitudes, grid_resolution,
time_index, cell_sampling, blocking)
local_projector.set_component(analysissystem.components[2])
local_projector.evaluate_design_matrix()
else:
local_projector.update_time_index(time_index, keep_design=True)
local_projector.set_component(analysissystem.components[2],
keep_design=True)
print analysissystem.components
local_expected_value = local_projector.project_expected_value(
).reshape((-1, 1))
# Save results
outputfile = outputfiles[inner_index]
print outputfile
# main merged product output files
filebuilder = FileBuilderGlobalField(
outputfile, eustace.timeutils.epoch.days_since_epoch(processdate),
'EUSTACE Analysis', get_revision_id_for_module(eustace),
definitions.TAS.name, '', 'Provisional output', __name__, '')
field_definition_tas = definitions.OutputVariable.from_template(
definitions.TEMPLATE_TEMPERATURE,
'tas',
quantity='average',
cell_methods='time: mean')
field_definition_tas_climatology = definitions.OutputVariable.from_template(
definitions.TEMPLATE_TEMPERATURE,
'tas_climatology',
quantity='average',
cell_methods='time: mean')
field_definition_tas_large_scale = definitions.OutputVariable.from_template(
definitions.TEMPLATE_PERTURBATION,
'tas_large_scale',
quantity='average',
cell_methods='time: mean')
field_definition_tas_daily_local = definitions.OutputVariable.from_template(
definitions.TEMPLATE_PERTURBATION,
'tas_daily_local',
quantity='average',
cell_methods='time: mean')
result_expected_value = climatology_expected_value + large_scale_expected_value + local_expected_value
filebuilder.add_global_field(
field_definition_tas,
result_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(
field_definition_tas_climatology,
climatology_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(
field_definition_tas_large_scale,
large_scale_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(
field_definition_tas_daily_local,
local_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.save_and_close()
print "Memory usage (MB):", psutil.Process(
os.getpid()).memory_info().rss / (1024 * 1024)
def output_grid_batch(storage_climatology, storage_large_scale, storage_local,
outputfiles, climatologyfiles, largescalefiles,
localfiles, time_indices, covariates_descriptor,
insitu_biases, breakpoints_file, global_biases,
global_biases_group_list, compute_uncertainties, method,
compute_sample, sample_size, compute_prior_sample):
from eustace.analysis.advanced_standard.fileio.output_projector import Projector
variance_ratio_upper_bound = 1.0
print 'VERSION: {0}'.format(get_revision_id_for_module(eustace))
# Build analysis system
analysissystem = AnalysisSystem_EUSTACE(
storage_climatology, storage_large_scale, storage_local,
covariates_descriptor, insitu_biases, breakpoints_file, global_biases,
global_biases_group_list, compute_uncertainties, method)
grid_resolution = [
180. / definitions.GLOBAL_FIELD_SHAPE[1],
360. / definitions.GLOBAL_FIELD_SHAPE[2]
]
latitudes = numpy.linspace(-90. + grid_resolution[0] / 2.,
90. - grid_resolution[0] / 2,
num=definitions.GLOBAL_FIELD_SHAPE[1])
longitudes = numpy.linspace(-180. + grid_resolution[1] / 2.,
180. - grid_resolution[1] / 2,
num=definitions.GLOBAL_FIELD_SHAPE[2])
timebase = TimeBaseDays(eustace.timeutils.epoch.EPOCH)
#processdates = [datetime_numeric.build( timebase.number_to_datetime(daynumber) ) for daynumber in time_indices]
processdates = [
timebase.number_to_datetime(daynumber) for daynumber in time_indices
]
cell_sampling = [1, 1]
blocking = 10
# thinned set of sample indices for inclusion in output product
sample_indices = range(definitions.GLOBAL_SAMPLE_SHAPE[3])
climatology_projector = None
large_scale_projector = None
local_projector = None
for (inner_index, time_index, processdate) in zip(range(len(time_indices)),
time_indices,
processdates):
print time_index
# Configure output grid
outputstructure = OutputRectilinearGridStructure(time_index,
processdate,
latitudes=latitudes,
longitudes=longitudes)
# climatology component
print 'Evaluating: climatology'
if climatology_projector is None:
climatology_projector = Projector(latitudes, longitudes,
grid_resolution, time_index,
cell_sampling, blocking)
climatology_projector.set_component(analysissystem.components[0])
climatology_projector.update_time_index(time_index, keep_design=False)
climatology_projector.evaluate_design_matrix()
climatology_expected_value = climatology_projector.project_expected_value(
).reshape((-1, 1))
climatology_uncertainties = climatology_projector.project_sample_deviation(
)
climatology_samples = climatology_projector.project_sample_values(
sample_indices=sample_indices) + climatology_expected_value
climatology_unconstraint = numpy.minimum(
climatology_uncertainties**2 /
climatology_projector.project_sample_deviation(prior=True)**2,
variance_ratio_upper_bound)
# large scale component
print 'Evaluating: large-scale'
if large_scale_projector is None:
large_scale_projector = Projector(latitudes, longitudes,
grid_resolution, time_index,
cell_sampling, blocking)
large_scale_projector.set_component(analysissystem.components[1])
large_scale_projector.update_time_index(time_index, keep_design=False)
large_scale_projector.evaluate_design_matrix()
large_scale_expected_value = large_scale_projector.project_expected_value(
).reshape((-1, 1))
large_scale_uncertainties = large_scale_projector.project_sample_deviation(
)
large_scale_samples = large_scale_projector.project_sample_values(
sample_indices=sample_indices) + large_scale_expected_value
large_scale_unconstraint = numpy.minimum(
large_scale_uncertainties**2 /
large_scale_projector.project_sample_deviation(prior=True)**2,
variance_ratio_upper_bound)
# local component - time handling updates state to new time but does not recompute the design matrix
print 'Evaluating: local'
if local_projector is None:
local_projector = Projector(latitudes, longitudes, grid_resolution,
time_index, cell_sampling, blocking)
local_projector.set_component(analysissystem.components[2])
local_projector.evaluate_design_matrix()
else:
local_projector.update_time_index(time_index, keep_design=True)
local_projector.set_component(analysissystem.components[2],
keep_design=True)
print analysissystem.components
local_expected_value = local_projector.project_expected_value(
).reshape((-1, 1))
local_uncertainties = local_projector.project_sample_deviation()
local_samples = local_projector.project_sample_values(
sample_indices=sample_indices) + local_expected_value
local_unconstraint = numpy.minimum(
local_uncertainties**2 /
local_projector.project_sample_deviation(prior=True)**2,
variance_ratio_upper_bound)
# Save results
outputfile = outputfiles[inner_index]
print outputfile
# main merged product output files
filebuilder = FileBuilderGlobalField(
outputfile, eustace.timeutils.epoch.days_since_epoch(processdate),
'EUSTACE Analysis', get_revision_id_for_module(eustace),
definitions.TAS.name, '', 'Provisional output', __name__, '')
result_expected_value = climatology_expected_value + large_scale_expected_value + local_expected_value
result_expected_uncertainties = numpy.sqrt(
climatology_uncertainties**2 + large_scale_uncertainties**2 +
local_uncertainties**2)
climatology_fraction = local_unconstraint # defined as ratio of posterior to prior variance in local component
filebuilder.add_global_field(
definitions.TAS,
result_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(
definitions.TASUNCERTAINTY,
result_expected_uncertainties.reshape(
definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(
definitions.TAS_OBSERVATION_INFLUENCE,
1.0 - climatology_fraction.reshape(definitions.GLOBAL_FIELD_SHAPE))
for index in range(definitions.GLOBAL_SAMPLE_SHAPE[3]):
variable = copy.deepcopy(definitions.TASENSEMBLE)
variable.name = variable.name + '_' + str(index)
selected_sample = (climatology_samples[:, index] +
large_scale_samples[:, index] +
local_samples[:, index]).ravel()
filebuilder.add_global_field(
variable,
selected_sample.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.save_and_close()
# climatology only output
climatologyfile = climatologyfiles[inner_index]
filebuilder = FileBuilderGlobalField(
climatologyfile,
eustace.timeutils.epoch.days_since_epoch(processdate),
'EUSTACE Analysis', get_revision_id_for_module(eustace),
definitions.TAS.name, '',
'Provisional component output - climatology', __name__, '')
filebuilder.add_global_field(
definitions.TAS,
climatology_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(
definitions.TASUNCERTAINTY,
climatology_uncertainties.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(
definitions.TAS_OBSERVATION_INFLUENCE, 1.0 -
climatology_unconstraint.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.save_and_close()
# large scale only output
largescalefile = largescalefiles[inner_index]
filebuilder = FileBuilderGlobalField(
largescalefile,
eustace.timeutils.epoch.days_since_epoch(processdate),
'EUSTACE Analysis', get_revision_id_for_module(eustace),
definitions.TAS.name, '',
'Provisional component output - large scale', __name__, '')
filebuilder.add_global_field(
definitions.TASPERTURBATION,
large_scale_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(
definitions.TASUNCERTAINTY,
large_scale_uncertainties.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(
definitions.TAS_OBSERVATION_INFLUENCE, 1.0 -
large_scale_unconstraint.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.save_and_close()
# local only output
localfile = localfiles[inner_index]
filebuilder = FileBuilderGlobalField(
localfile, eustace.timeutils.epoch.days_since_epoch(processdate),
'EUSTACE Analysis', get_revision_id_for_module(eustace),
definitions.TAS.name, '', 'Provisional component output - local',
__name__, '')
filebuilder.add_global_field(
definitions.TASPERTURBATION,
local_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(
definitions.TASUNCERTAINTY,
local_uncertainties.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(
definitions.TAS_OBSERVATION_INFLUENCE,
1.0 - local_unconstraint.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.save_and_close()
print "Memory usage (MB):", psutil.Process(
os.getpid()).memory_info().rss / (1024 * 1024)
def test_run_idl(self):
outputfile = tempfile.NamedTemporaryFile(
prefix=FILENAMES_OUTPUT[0][1] + '_', suffix='.nc')
# Should be updated to final directory for parameters
model = ModelLand([
'/gws/nopw/j04/eustace/users/ejn2/Output/sat_lsat/May2017/MYG_model1_coefs.txt',
'/gws/nopw/j04/eustace/users/ejn2/Output/sat_lsat/May2017/MYG_model2_coefs.txt',
'/gws/nopw/j04/eustace/users/ejn2/Output/sat_lsat/May2017/MYG_model3_coefs.txt'
], .2, 3.)
model.run_idl(
[[tempfile.gettempdir() + '/',
os.path.basename(outputfile.name)]], FILENAMES_INPUT)
#check netcdf output produced something
dataset = netCDF4.Dataset(outputfile.name)
lat = dataset.variables['lat']
self.assertEqual('latitude', lat.standard_name)
lat = dataset.variables['lon']
self.assertEqual('longitude', lat.standard_name)
tasmin = dataset.variables['tasmin']
self.assertEqual('air_temperature', tasmin.standard_name)
self.assertEqual('Minimum daily surface air temperature (K)',
tasmin.long_name)
tasmax = dataset.variables['tasmax']
self.assertEqual('air_temperature', tasmax.standard_name)
self.assertEqual('Maximum daily surface air temperature (K)',
tasmax.long_name)
unc_rand_tasmin = dataset.variables['unc_rand_tasmin']
self.assertEqual(
'Random uncertainty on minimum daily surface air temperature (K)',
unc_rand_tasmin.long_name)
unc_rand_tasmax = dataset.variables['unc_rand_tasmax']
self.assertEqual(
'Random uncertainty on maximum daily surface air temperature (K)',
unc_rand_tasmax.long_name)
unc_corr_atm_tasmin = dataset.variables['unc_corr_atm_tasmin']
self.assertEqual(
'Locally correlated uncertainty (atmospheric scale lengths) on minimum daily surface air temperature (K)',
unc_corr_atm_tasmin.long_name)
unc_corr_sfc_tasmin = dataset.variables['unc_corr_sfc_tasmin']
self.assertEqual(
'Locally correlated uncertainty (surface scale lengths) on minimum daily surface air temperature (K)',
unc_corr_sfc_tasmin.long_name)
unc_corr_atm_tasmax = dataset.variables['unc_corr_atm_tasmax']
self.assertEqual(
'Locally correlated uncertainty (atmospheric scale lengths) on maximum daily surface air temperature (K)',
unc_corr_atm_tasmax.long_name)
unc_corr_sfc_tasmax = dataset.variables['unc_corr_sfc_tasmax']
self.assertEqual(
'Locally correlated uncertainty (surface scale lengths) on maximum daily surface air temperature (K)',
unc_corr_sfc_tasmax.long_name)
unc_sys_tasmin = dataset.variables['unc_sys_tasmin']
self.assertEqual(
'Systematic uncertainty on minimum daily surface air temperature (K)',
unc_sys_tasmin.long_name)
unc_sys_tasmax = dataset.variables['unc_sys_tasmax']
self.assertEqual(
'Systematic uncertainty on maximum daily surface air temperature (K)',
unc_sys_tasmax.long_name)
tasmin_model_number = dataset.variables['tasmin_model_number']
self.assertEqual(
'Model number used for estimating Tmin from satellite data',
tasmin_model_number.long_name)
tasmax_model_number = dataset.variables['tasmax_model_number']
self.assertEqual(
'Model number used for estimating Tmax from satellite data',
tasmax_model_number.long_name)
file_time = dataset.variables['time']
self.assertEqual('time', file_time.standard_name)
self.assertEqual('Time (days)', file_time.long_name)
t = TimeBaseDays(EPOCH)
file_date = t.number_to_datetime(int(file_time[0]))
self.assertEqual(datetime.strptime('20110104', '%Y%m%d'), file_date)