def break_points_timestamp(self, epoch):
"""Transform breaking point value into a date of the from YYYYMMDD """
converter = TimeBaseDays(epoch)
return numpy.array([
'{0.year:4d}-{0.month:02d}-{0.day:02d}'.format(
converter.number_to_datetime(int(number)))
for number in self.break_time
]) #, dtype='datetime64[D]')
def test_time_datetime(self):
index = 3
obs = 'TMAX'
test_source = SimulatedObservationSource(self.times,
self.look_up,
self.masks,
self.observations,
daynumber=self.times[index])
test_connector = ObservationStructureSourceConnector(
test_source, obs,
TimeBaseDays(datetime.datetime(1850, 1, 1)).number_to_datetime(
self.times[index]))
self.assertEqual(
TimeBaseDays(datetime.datetime(1850, 1, 1)).number_to_datetime(
self.times[index]), test_connector.time_datetime())
def test_check_all_candidates(self):
index = 6
obs = 'TMEAN'
test_source = SimulatedObservationSource(self.times,
self.look_up,
self.masks,
self.observations,
daynumber=self.times[index])
test_connector = ObservationStructureSourceConnector(
test_source, obs,
TimeBaseDays(datetime.datetime(1850, 1, 1)).number_to_datetime(
self.times[index]))
pattern = '%a%C%B.b1.%Y%m%d.000000.cdf'
self.assertFalse(test_connector.check_all_candidates(pattern))
pattern = 'some_file'
self.assertTrue(test_connector.check_all_candidates(pattern))
test_connector.observable_filenames = [
'inZitu_land_Tmin_20140202.bin', 'insitu_land_Tmax_20140202.bin'
]
pattern = 'insitu_land_%A_%Y%m%d.bin'
self.assertFalse(test_connector.check_all_candidates(pattern))
test_connector.observable_filenames = [
'insitu_land_Tmin_20140202.bin', 'insitu_land_Tmax_20140202.bin'
]
pattern = 'insitu_land_%A_%Y%m%d.bin'
self.assertTrue(test_connector.check_all_candidates(pattern))
def test_time_index(self):
index = 8
obs = 'TMEAN'
test_source = SimulatedObservationSource(self.times,
self.look_up,
self.masks,
self.observations,
daynumber=self.times[index])
test_connector = ObservationStructureSourceConnector(
test_source, obs,
TimeBaseDays(datetime.datetime(1850, 1, 1)).number_to_datetime(
self.times[index]))
numpy.testing.assert_array_equal(self.times[index],
test_connector.time_index())
def test_init(self):
index = 7
obs = 'TMIN'
test_source = SimulatedObservationSource(self.times,
self.look_up,
self.masks,
self.observations,
daynumber=self.times[index])
test_connector = ObservationStructureSourceConnector(
test_source, obs,
TimeBaseDays(datetime.datetime(1850, 1, 1)).number_to_datetime(
self.times[index]))
numpy.testing.assert_array_equal(self.look_up,
test_connector.location_lookup)
numpy.testing.assert_array_equal(
numpy.nonzero(~self.masks[obs][index, :]),
test_connector.valid_indices)
self.assertEqual(
TimeBaseDays(datetime.datetime(1850, 1, 1)).number_to_datetime(
self.times[index]), test_connector.corresponding_datetime)
self.assertEqual(['some_file'], test_connector.observable_filenames)
self.assertTrue(isinstance(test_connector.observations, Observations))
def test_location_polar_coordinates(self):
index = 4
obs = 'TMIN'
test_source = SimulatedObservationSource(self.times,
self.look_up,
self.masks,
self.observations,
daynumber=self.times[index])
test_connector = ObservationStructureSourceConnector(
test_source, obs,
TimeBaseDays(datetime.datetime(1850, 1, 1)).number_to_datetime(
self.times[index]))
numpy.testing.assert_array_equal(
self.look_up[:, [1, 4, 6, 7, 8]].T,
test_connector.location_polar_coordinates())
def test_covariate_effect_exception(self):
index = 6
obs = 'TMEAN'
test_source = SimulatedObservationSource(self.times,
self.look_up,
self.masks,
self.observations,
daynumber=self.times[index])
test_connector = ObservationStructureSourceConnector(
test_source, obs,
TimeBaseDays(datetime.datetime(1850, 1, 1)).number_to_datetime(
self.times[index]))
self.assertRaises(ValueError, test_connector.covariate_effect,
'gababob')
self.assertEqual(None, test_connector.covariate_effect('insitu_land'))
def test_observation_precision(self):
index = 6
obs = 'TMEAN'
test_source = SimulatedObservationSource(self.times,
self.look_up,
self.masks,
self.observations,
daynumber=self.times[index])
test_connector = ObservationStructureSourceConnector(
test_source, obs,
TimeBaseDays(datetime.datetime(1850, 1, 1)).number_to_datetime(
self.times[index]))
test_array = numpy.zeros(9) + 1.
test_matrix = scipy.sparse.diags(1.0 / (test_array[[1, 2, 4, 7]]**2),
format='csc')
numpy.testing.assert_array_equal(
test_matrix.toarray(),
test_connector.observation_precision().toarray())
def early_look_grid_batch(
storage_climatology, storage_large_scale, storage_local, outputfiles,
time_indices, covariates_descriptor, insitu_biases, breakpoints_file,
global_biases, global_biases_group_list, compute_uncertainties, method,
compute_sample, sample_size, compute_prior_sample):
"""Produce 'early look' NetCDF output files without loading or gridding uncertainty information
For inspection of analysis output prior to the final gridding step.
"""
from eustace.analysis.advanced_standard.fileio.output_projector import Projector
print 'VERSION: {0}'.format(get_revision_id_for_module(eustace))
# Build analysis system
analysissystem = AnalysisSystem_EUSTACE(
storage_climatology, storage_large_scale, storage_local,
covariates_descriptor, insitu_biases, breakpoints_file, global_biases,
global_biases_group_list, compute_uncertainties, method)
grid_resolution = [
180. / definitions.GLOBAL_FIELD_SHAPE[1],
360. / definitions.GLOBAL_FIELD_SHAPE[2]
]
latitudes = numpy.linspace(-90. + grid_resolution[0] / 2.,
90. - grid_resolution[0] / 2,
num=definitions.GLOBAL_FIELD_SHAPE[1])
longitudes = numpy.linspace(-180. + grid_resolution[1] / 2.,
180. - grid_resolution[1] / 2,
num=definitions.GLOBAL_FIELD_SHAPE[2])
timebase = TimeBaseDays(eustace.timeutils.epoch.EPOCH)
processdates = [
timebase.number_to_datetime(daynumber) for daynumber in time_indices
]
cell_sampling = [1, 1]
blocking = 10
# thinned set of sample indices for inclusion in output product
climatology_projector = None
large_scale_projector = None
local_projector = None
for (inner_index, time_index, processdate) in zip(range(len(time_indices)),
time_indices,
processdates):
print time_index
# Configure output grid
outputstructure = OutputRectilinearGridStructure(time_index,
processdate,
latitudes=latitudes,
longitudes=longitudes)
# climatology component
print 'Evaluating: climatology'
if climatology_projector is None:
climatology_projector = Projector(latitudes, longitudes,
grid_resolution, time_index,
cell_sampling, blocking)
climatology_projector.set_component(analysissystem.components[0])
climatology_projector.update_time_index(time_index, keep_design=False)
climatology_projector.evaluate_design_matrix()
climatology_expected_value = climatology_projector.project_expected_value(
).reshape((-1, 1))
# large scale component
print 'Evaluating: large-scale'
if large_scale_projector is None:
large_scale_projector = Projector(latitudes, longitudes,
grid_resolution, time_index,
cell_sampling, blocking)
large_scale_projector.set_component(analysissystem.components[1])
large_scale_projector.update_time_index(time_index, keep_design=False)
large_scale_projector.evaluate_design_matrix()
large_scale_expected_value = large_scale_projector.project_expected_value(
).reshape((-1, 1))
# local component - time handling updates state to new time but does not recompute the design matrix
print 'Evaluating: local'
if local_projector is None:
local_projector = Projector(latitudes, longitudes, grid_resolution,
time_index, cell_sampling, blocking)
local_projector.set_component(analysissystem.components[2])
local_projector.evaluate_design_matrix()
else:
local_projector.update_time_index(time_index, keep_design=True)
local_projector.set_component(analysissystem.components[2],
keep_design=True)
print analysissystem.components
local_expected_value = local_projector.project_expected_value(
).reshape((-1, 1))
# Save results
outputfile = outputfiles[inner_index]
print outputfile
# main merged product output files
filebuilder = FileBuilderGlobalField(
outputfile, eustace.timeutils.epoch.days_since_epoch(processdate),
'EUSTACE Analysis', get_revision_id_for_module(eustace),
definitions.TAS.name, '', 'Provisional output', __name__, '')
field_definition_tas = definitions.OutputVariable.from_template(
definitions.TEMPLATE_TEMPERATURE,
'tas',
quantity='average',
cell_methods='time: mean')
field_definition_tas_climatology = definitions.OutputVariable.from_template(
definitions.TEMPLATE_TEMPERATURE,
'tas_climatology',
quantity='average',
cell_methods='time: mean')
field_definition_tas_large_scale = definitions.OutputVariable.from_template(
definitions.TEMPLATE_PERTURBATION,
'tas_large_scale',
quantity='average',
cell_methods='time: mean')
field_definition_tas_daily_local = definitions.OutputVariable.from_template(
definitions.TEMPLATE_PERTURBATION,
'tas_daily_local',
quantity='average',
cell_methods='time: mean')
result_expected_value = climatology_expected_value + large_scale_expected_value + local_expected_value
filebuilder.add_global_field(
field_definition_tas,
result_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(
field_definition_tas_climatology,
climatology_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(
field_definition_tas_large_scale,
large_scale_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(
field_definition_tas_daily_local,
local_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.save_and_close()
print "Memory usage (MB):", psutil.Process(
os.getpid()).memory_info().rss / (1024 * 1024)
def output_grid_batch(storage_climatology, storage_large_scale, storage_local,
outputfiles, climatologyfiles, largescalefiles,
localfiles, time_indices, covariates_descriptor,
insitu_biases, breakpoints_file, global_biases,
global_biases_group_list, compute_uncertainties, method,
compute_sample, sample_size, compute_prior_sample):
from eustace.analysis.advanced_standard.fileio.output_projector import Projector
variance_ratio_upper_bound = 1.0
print 'VERSION: {0}'.format(get_revision_id_for_module(eustace))
# Build analysis system
analysissystem = AnalysisSystem_EUSTACE(
storage_climatology, storage_large_scale, storage_local,
covariates_descriptor, insitu_biases, breakpoints_file, global_biases,
global_biases_group_list, compute_uncertainties, method)
grid_resolution = [
180. / definitions.GLOBAL_FIELD_SHAPE[1],
360. / definitions.GLOBAL_FIELD_SHAPE[2]
]
latitudes = numpy.linspace(-90. + grid_resolution[0] / 2.,
90. - grid_resolution[0] / 2,
num=definitions.GLOBAL_FIELD_SHAPE[1])
longitudes = numpy.linspace(-180. + grid_resolution[1] / 2.,
180. - grid_resolution[1] / 2,
num=definitions.GLOBAL_FIELD_SHAPE[2])
timebase = TimeBaseDays(eustace.timeutils.epoch.EPOCH)
#processdates = [datetime_numeric.build( timebase.number_to_datetime(daynumber) ) for daynumber in time_indices]
processdates = [
timebase.number_to_datetime(daynumber) for daynumber in time_indices
]
cell_sampling = [1, 1]
blocking = 10
# thinned set of sample indices for inclusion in output product
sample_indices = range(definitions.GLOBAL_SAMPLE_SHAPE[3])
climatology_projector = None
large_scale_projector = None
local_projector = None
for (inner_index, time_index, processdate) in zip(range(len(time_indices)),
time_indices,
processdates):
print time_index
# Configure output grid
outputstructure = OutputRectilinearGridStructure(time_index,
processdate,
latitudes=latitudes,
longitudes=longitudes)
# climatology component
print 'Evaluating: climatology'
if climatology_projector is None:
climatology_projector = Projector(latitudes, longitudes,
grid_resolution, time_index,
cell_sampling, blocking)
climatology_projector.set_component(analysissystem.components[0])
climatology_projector.update_time_index(time_index, keep_design=False)
climatology_projector.evaluate_design_matrix()
climatology_expected_value = climatology_projector.project_expected_value(
).reshape((-1, 1))
climatology_uncertainties = climatology_projector.project_sample_deviation(
)
climatology_samples = climatology_projector.project_sample_values(
sample_indices=sample_indices) + climatology_expected_value
climatology_unconstraint = numpy.minimum(
climatology_uncertainties**2 /
climatology_projector.project_sample_deviation(prior=True)**2,
variance_ratio_upper_bound)
# large scale component
print 'Evaluating: large-scale'
if large_scale_projector is None:
large_scale_projector = Projector(latitudes, longitudes,
grid_resolution, time_index,
cell_sampling, blocking)
large_scale_projector.set_component(analysissystem.components[1])
large_scale_projector.update_time_index(time_index, keep_design=False)
large_scale_projector.evaluate_design_matrix()
large_scale_expected_value = large_scale_projector.project_expected_value(
).reshape((-1, 1))
large_scale_uncertainties = large_scale_projector.project_sample_deviation(
)
large_scale_samples = large_scale_projector.project_sample_values(
sample_indices=sample_indices) + large_scale_expected_value
large_scale_unconstraint = numpy.minimum(
large_scale_uncertainties**2 /
large_scale_projector.project_sample_deviation(prior=True)**2,
variance_ratio_upper_bound)
# local component - time handling updates state to new time but does not recompute the design matrix
print 'Evaluating: local'
if local_projector is None:
local_projector = Projector(latitudes, longitudes, grid_resolution,
time_index, cell_sampling, blocking)
local_projector.set_component(analysissystem.components[2])
local_projector.evaluate_design_matrix()
else:
local_projector.update_time_index(time_index, keep_design=True)
local_projector.set_component(analysissystem.components[2],
keep_design=True)
print analysissystem.components
local_expected_value = local_projector.project_expected_value(
).reshape((-1, 1))
local_uncertainties = local_projector.project_sample_deviation()
local_samples = local_projector.project_sample_values(
sample_indices=sample_indices) + local_expected_value
local_unconstraint = numpy.minimum(
local_uncertainties**2 /
local_projector.project_sample_deviation(prior=True)**2,
variance_ratio_upper_bound)
# Save results
outputfile = outputfiles[inner_index]
print outputfile
# main merged product output files
filebuilder = FileBuilderGlobalField(
outputfile, eustace.timeutils.epoch.days_since_epoch(processdate),
'EUSTACE Analysis', get_revision_id_for_module(eustace),
definitions.TAS.name, '', 'Provisional output', __name__, '')
result_expected_value = climatology_expected_value + large_scale_expected_value + local_expected_value
result_expected_uncertainties = numpy.sqrt(
climatology_uncertainties**2 + large_scale_uncertainties**2 +
local_uncertainties**2)
climatology_fraction = local_unconstraint # defined as ratio of posterior to prior variance in local component
filebuilder.add_global_field(
definitions.TAS,
result_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(
definitions.TASUNCERTAINTY,
result_expected_uncertainties.reshape(
definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(
definitions.TAS_OBSERVATION_INFLUENCE,
1.0 - climatology_fraction.reshape(definitions.GLOBAL_FIELD_SHAPE))
for index in range(definitions.GLOBAL_SAMPLE_SHAPE[3]):
variable = copy.deepcopy(definitions.TASENSEMBLE)
variable.name = variable.name + '_' + str(index)
selected_sample = (climatology_samples[:, index] +
large_scale_samples[:, index] +
local_samples[:, index]).ravel()
filebuilder.add_global_field(
variable,
selected_sample.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.save_and_close()
# climatology only output
climatologyfile = climatologyfiles[inner_index]
filebuilder = FileBuilderGlobalField(
climatologyfile,
eustace.timeutils.epoch.days_since_epoch(processdate),
'EUSTACE Analysis', get_revision_id_for_module(eustace),
definitions.TAS.name, '',
'Provisional component output - climatology', __name__, '')
filebuilder.add_global_field(
definitions.TAS,
climatology_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(
definitions.TASUNCERTAINTY,
climatology_uncertainties.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(
definitions.TAS_OBSERVATION_INFLUENCE, 1.0 -
climatology_unconstraint.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.save_and_close()
# large scale only output
largescalefile = largescalefiles[inner_index]
filebuilder = FileBuilderGlobalField(
largescalefile,
eustace.timeutils.epoch.days_since_epoch(processdate),
'EUSTACE Analysis', get_revision_id_for_module(eustace),
definitions.TAS.name, '',
'Provisional component output - large scale', __name__, '')
filebuilder.add_global_field(
definitions.TASPERTURBATION,
large_scale_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(
definitions.TASUNCERTAINTY,
large_scale_uncertainties.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(
definitions.TAS_OBSERVATION_INFLUENCE, 1.0 -
large_scale_unconstraint.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.save_and_close()
# local only output
localfile = localfiles[inner_index]
filebuilder = FileBuilderGlobalField(
localfile, eustace.timeutils.epoch.days_since_epoch(processdate),
'EUSTACE Analysis', get_revision_id_for_module(eustace),
definitions.TAS.name, '', 'Provisional component output - local',
__name__, '')
filebuilder.add_global_field(
definitions.TASPERTURBATION,
local_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(
definitions.TASUNCERTAINTY,
local_uncertainties.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(
definitions.TAS_OBSERVATION_INFLUENCE,
1.0 - local_unconstraint.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.save_and_close()
print "Memory usage (MB):", psutil.Process(
os.getpid()).memory_info().rss / (1024 * 1024)
def timebase(self):
return TimeBaseDays(self.start)
def test_bias_global_satellite(self):
index = 9
obs = 'TMIN'
# valid observation indices = [0, 3, 4, 5, 8]
test_source = TestInsituGlobalSatellite.SimulatedObservationSource(
self.times,
self.look_up,
self.masks,
self.observations,
daynumber=self.times[index])
test_connector = ObservationStructureSourceConnector(
test_source, obs,
TimeBaseDays(datetime.datetime(1850, 1, 1)).number_to_datetime(
self.times[index]))
bias = BiasElement('surfaceairmodel_land_global', 1)
bias_design = bias.element_design(test_connector)
self.assertEqual(bias_design.number_of_biases, 1)
self.assertEqual(bias_design.number_of_observations,
(~self.masks[obs][index, :]).sum())
numpy.testing.assert_array_equal(
bias_design.effect,
numpy.array([[0, 0], [1, 0], [2, 0], [3, 0], [4, 0]]))
numpy.testing.assert_array_equal(
bias_design.design_matrix().todense(),
numpy.array([[1.], [1.], [1.], [1.], [1.]]))
index = 6
obs = 'TMEAN'
# valid observation indices = [1, 2, 4, 7]
test_source = TestInsituGlobalSatellite.SimulatedObservationSource(
self.times,
self.look_up,
self.masks,
self.observations,
daynumber=self.times[index])
test_connector = ObservationStructureSourceConnector(
test_source, obs,
TimeBaseDays(datetime.datetime(1850, 1, 1)).number_to_datetime(
self.times[index]))
test_connector.observable_filenames = [
'surfaceairmodel_ocean_Tmean_20140202.bin'
]
bias = BiasElement('surfaceairmodel_ocean_global', 1)
bias_design = bias.element_design(test_connector)
self.assertEqual(bias_design.number_of_biases, 1)
self.assertEqual(bias_design.number_of_observations,
(~self.masks[obs][index, :]).sum())
numpy.testing.assert_array_equal(
bias_design.effect, numpy.array([[0, 0], [1, 0], [2, 0], [3, 0]]))
numpy.testing.assert_array_equal(bias_design.design_matrix().todense(),
numpy.array([[1.], [1.], [1.], [1.]]))
index = 2
obs = 'TMAX'
# valid observation indices = [0, 1, 2, 5, 7, 8]
test_source = TestInsituGlobalSatellite.SimulatedObservationSource(
self.times,
self.look_up,
self.masks,
self.observations,
daynumber=self.times[index])
test_connector = ObservationStructureSourceConnector(
test_source, obs,
TimeBaseDays(datetime.datetime(1850, 1, 1)).number_to_datetime(
self.times[index]))
test_connector.observable_filenames = [
'surfaceairmodel_ice_Tmin_20140202.bin',
'surfaceairmodel_ice_Tmean_20140202.bin',
'surfaceairmodel_ice_Tmax_20140202.bin'
]
bias = BiasElement('surfaceairmodel_ice_global', 1)
bias_design = bias.element_design(test_connector)
self.assertEqual(bias_design.number_of_biases, 1)
self.assertEqual(bias_design.number_of_observations,
(~self.masks[obs][index, :]).sum())
numpy.testing.assert_array_equal(
bias_design.effect,
numpy.array([[0, 0], [1, 0], [2, 0], [3, 0], [4, 0], [5, 0]]))
numpy.testing.assert_array_equal(
bias_design.design_matrix().todense(),
numpy.array([[1.], [1.], [1.], [1.], [1.], [1.]]))
def timebase(self):
return TimeBaseDays(EPOCH)
def test_global_satellite_effect(self):
index = 9
obs = 'TMIN'
# valid observation indices = [0, 3, 4, 5, 8]
test_source = SimulatedObservationSource(self.times,
self.look_up,
self.masks,
self.observations,
daynumber=self.times[index])
test_connector = ObservationStructureSourceConnector(
test_source, obs,
TimeBaseDays(datetime.datetime(1850, 1, 1)).number_to_datetime(
self.times[index]))
test_connector.observable_filenames = [
'surfaceairmodel_land_Tmin_20140202.bin',
'surfaceairmodel_land_Tmax_20140202.bin'
]
numpy.testing.assert_array_equal(
test_connector.covariate_effect('surfaceairmodel_land_global'),
numpy.array([[0, 0], [1, 0], [2, 0], [3, 0], [4, 0]]))
index = 6
obs = 'TMEAN'
# valid observation indices = [1, 2, 4, 7]
test_source = SimulatedObservationSource(self.times,
self.look_up,
self.masks,
self.observations,
daynumber=self.times[index])
test_connector = ObservationStructureSourceConnector(
test_source, obs,
TimeBaseDays(datetime.datetime(1850, 1, 1)).number_to_datetime(
self.times[index]))
test_connector.observable_filenames = [
'surfaceairmodel_ocean_Tmean_20140202.bin'
]
numpy.testing.assert_array_equal(
test_connector.covariate_effect('surfaceairmodel_ocean_global'),
numpy.array([[0, 0], [1, 0], [2, 0], [3, 0]]))
index = 2
obs = 'TMAX'
# valid observation indices = [0, 1, 2, 5, 7, 8]
test_source = SimulatedObservationSource(self.times,
self.look_up,
self.masks,
self.observations,
daynumber=self.times[index])
test_connector = ObservationStructureSourceConnector(
test_source, obs,
TimeBaseDays(datetime.datetime(1850, 1, 1)).number_to_datetime(
self.times[index]))
test_connector.observable_filenames = [
'surfaceairmodel_ice_Tmin_20140202.bin',
'surfaceairmodel_ice_Tmean_20140202.bin',
'surfaceairmodel_ice_Tmax_20140202.bin'
]
numpy.testing.assert_array_equal(
test_connector.covariate_effect('surfaceairmodel_ice_global'),
numpy.array([[0, 0], [1, 0], [2, 0], [3, 0], [4, 0], [5, 0]]))
def test_insitu_land_covariate_effect(self):
reader = ObservationBreakPointSourceInsituLand(
self.breakpoints_file.name)
observed_breakpoints = reader.observations('merged_break')
# Testing covariate effect for all the available observables
# break points = [47116, 51499, 53325, 54056, 38715, 39811, 47116, 23740, 24836, 27027]
# stations = [4, 4, 4, 4, 6, 6, 7, 8 , 8 , 8]
index = 9
obs = 'TMIN'
# valid observation indices = [0, 3, 4, 5, 8]
# t = 54056 -> [[2, 3]]
test_source = SimulatedObservationSource(self.times,
self.look_up,
self.masks,
self.observations,
daynumber=self.times[index])
test_connector = ObservationStructureSourceConnector(
test_source, obs,
TimeBaseDays(datetime.datetime(1850, 1, 1)).number_to_datetime(
self.times[index]))
test_connector.observable_filenames = [
'insitu_land_Tmin_20140202.bin', 'insitu_land_Tmax_20140202.bin'
]
numpy.testing.assert_array_equal(
test_connector.covariate_effect(
'insitu_land', **{'breakpoints': observed_breakpoints}),
numpy.array([[2, 3]]))
index = 6
obs = 'TMEAN'
# valid observation indices = [1, 2, 4, 7]
# t = 47117 -> [[2,1]]
test_source = SimulatedObservationSource(self.times,
self.look_up,
self.masks,
self.observations,
daynumber=self.times[index])
test_connector = ObservationStructureSourceConnector(
test_source, obs,
TimeBaseDays(datetime.datetime(1850, 1, 1)).number_to_datetime(
self.times[index]))
test_connector.observable_filenames = [
'insitu_land_Tmin_20140202.bin', 'insitu_land_Tmax_20140202.bin'
]
numpy.testing.assert_array_equal(
test_connector.covariate_effect(
'insitu_land', **{'breakpoints': observed_breakpoints}),
numpy.array([[2, 1]]))
index = 2
obs = 'TMAX'
# valid observation indices = [0, 1, 2, 5, 7, 8]
# t = 27027 -> [[4, 6], [5, 9]]
test_source = SimulatedObservationSource(self.times,
self.look_up,
self.masks,
self.observations,
daynumber=self.times[index])
test_connector = ObservationStructureSourceConnector(
test_source, obs,
TimeBaseDays(datetime.datetime(1850, 1, 1)).number_to_datetime(
self.times[index]))
test_connector.observable_filenames = [
'insitu_land_Tmin_20140202.bin', 'insitu_land_Tmax_20140202.bin'
]
numpy.testing.assert_array_equal(
test_connector.covariate_effect(
'insitu_land', **{'breakpoints': observed_breakpoints}),
numpy.array([[4, 6], [5, 9]]))
def test_run_idl(self):
outputfile = tempfile.NamedTemporaryFile(
prefix=FILENAMES_OUTPUT[0][1] + '_', suffix='.nc')
# Should be updated to final directory for parameters
model = ModelLand([
'/gws/nopw/j04/eustace/users/ejn2/Output/sat_lsat/May2017/MYG_model1_coefs.txt',
'/gws/nopw/j04/eustace/users/ejn2/Output/sat_lsat/May2017/MYG_model2_coefs.txt',
'/gws/nopw/j04/eustace/users/ejn2/Output/sat_lsat/May2017/MYG_model3_coefs.txt'
], .2, 3.)
model.run_idl(
[[tempfile.gettempdir() + '/',
os.path.basename(outputfile.name)]], FILENAMES_INPUT)
#check netcdf output produced something
dataset = netCDF4.Dataset(outputfile.name)
lat = dataset.variables['lat']
self.assertEqual('latitude', lat.standard_name)
lat = dataset.variables['lon']
self.assertEqual('longitude', lat.standard_name)
tasmin = dataset.variables['tasmin']
self.assertEqual('air_temperature', tasmin.standard_name)
self.assertEqual('Minimum daily surface air temperature (K)',
tasmin.long_name)
tasmax = dataset.variables['tasmax']
self.assertEqual('air_temperature', tasmax.standard_name)
self.assertEqual('Maximum daily surface air temperature (K)',
tasmax.long_name)
unc_rand_tasmin = dataset.variables['unc_rand_tasmin']
self.assertEqual(
'Random uncertainty on minimum daily surface air temperature (K)',
unc_rand_tasmin.long_name)
unc_rand_tasmax = dataset.variables['unc_rand_tasmax']
self.assertEqual(
'Random uncertainty on maximum daily surface air temperature (K)',
unc_rand_tasmax.long_name)
unc_corr_atm_tasmin = dataset.variables['unc_corr_atm_tasmin']
self.assertEqual(
'Locally correlated uncertainty (atmospheric scale lengths) on minimum daily surface air temperature (K)',
unc_corr_atm_tasmin.long_name)
unc_corr_sfc_tasmin = dataset.variables['unc_corr_sfc_tasmin']
self.assertEqual(
'Locally correlated uncertainty (surface scale lengths) on minimum daily surface air temperature (K)',
unc_corr_sfc_tasmin.long_name)
unc_corr_atm_tasmax = dataset.variables['unc_corr_atm_tasmax']
self.assertEqual(
'Locally correlated uncertainty (atmospheric scale lengths) on maximum daily surface air temperature (K)',
unc_corr_atm_tasmax.long_name)
unc_corr_sfc_tasmax = dataset.variables['unc_corr_sfc_tasmax']
self.assertEqual(
'Locally correlated uncertainty (surface scale lengths) on maximum daily surface air temperature (K)',
unc_corr_sfc_tasmax.long_name)
unc_sys_tasmin = dataset.variables['unc_sys_tasmin']
self.assertEqual(
'Systematic uncertainty on minimum daily surface air temperature (K)',
unc_sys_tasmin.long_name)
unc_sys_tasmax = dataset.variables['unc_sys_tasmax']
self.assertEqual(
'Systematic uncertainty on maximum daily surface air temperature (K)',
unc_sys_tasmax.long_name)
tasmin_model_number = dataset.variables['tasmin_model_number']
self.assertEqual(
'Model number used for estimating Tmin from satellite data',
tasmin_model_number.long_name)
tasmax_model_number = dataset.variables['tasmax_model_number']
self.assertEqual(
'Model number used for estimating Tmax from satellite data',
tasmax_model_number.long_name)
file_time = dataset.variables['time']
self.assertEqual('time', file_time.standard_name)
self.assertEqual('Time (days)', file_time.long_name)
t = TimeBaseDays(EPOCH)
file_date = t.number_to_datetime(int(file_time[0]))
self.assertEqual(datetime.strptime('20110104', '%Y%m%d'), file_date)
def days_since_epoch(t):
"""Compute days to datetime t from EUSTACE epoch 01/01/1850 00:00 UTC, as 32-bit floating point."""
return TimeBaseDays(EPOCH).datetime_to_number(t)
def epoch_plus_days(d):
"""Add d days to EUSTACE epoch 01/01/1850 00:00 UTC and return as datetime object."""
return TimeBaseDays(EPOCH).number_to_datetime(d)
def test_bias_insitu_land(self):
bias = InsituLandBiasElement(self.breakpoints_file.name)
index = 9
obs = 'TMIN'
# valid observation indices = [0, 3, 4, 5, 8]
# t = 54056 -> [[2, 3]]
# resulting design matrix
#[0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
#[0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
#[0. 0. 0. 1. 0. 0. 0. 0. 0. 0.]
#[0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
#[0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
test_source = TestInsituLandBias.SimulatedObservationSource(
self.times,
self.look_up,
self.masks,
self.observations,
daynumber=self.times[index])
test_connector = ObservationStructureSourceConnector(
test_source, obs,
TimeBaseDays(datetime.datetime(1850, 1, 1)).number_to_datetime(
self.times[index]))
bias_design = bias.element_design(test_connector)
self.assertEqual(bias_design.groupname, 'insitu_land')
self.assertEqual(bias_design.number_of_biases, len(self.break_times))
self.assertEqual(bias_design.number_of_observations,
(~self.masks[obs][index, :]).sum())
numpy.testing.assert_array_equal(bias_design.effect,
numpy.array([[2, 3]]))
numpy.testing.assert_array_equal(
bias_design.design_matrix().todense(),
numpy.array([[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 1., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]]))
index = 4
obs = 'TMEAN'
# valid observation indices = [0, 3, 4, 5, 7]
# t = 39811 -> [[2, 0], [4, 6]]
# resulting design matrix
#[0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
#[0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
#[1. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
#[0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
#[0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]
test_source = TestInsituLandBias.SimulatedObservationSource(
self.times,
self.look_up,
self.masks,
self.observations,
daynumber=self.times[index])
test_connector = ObservationStructureSourceConnector(
test_source, obs,
TimeBaseDays(datetime.datetime(1850, 1, 1)).number_to_datetime(
self.times[index]))
bias_design = bias.element_design(test_connector)
self.assertEqual(bias_design.groupname, 'insitu_land')
self.assertEqual(bias_design.number_of_biases, len(self.break_times))
self.assertEqual(bias_design.number_of_observations,
(~self.masks[obs][index, :]).sum())
numpy.testing.assert_array_equal(bias_design.effect,
numpy.array([[2, 0], [4, 6]]))
numpy.testing.assert_array_equal(
bias_design.design_matrix().todense(),
numpy.array([[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[1., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 1., 0., 0., 0.]]))
index = 9
obs = 'TMAX'
# valid observation indices = [0, 1, 3, 4, 5, 6, 7]
# t = 54056 -> [[3, 3]]
# resulting design matrix
#[0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
#[0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
#[0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
#[0. 0. 0. 1. 0. 0. 0. 0. 0. 0.]
#[0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
#[0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
#[0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
test_source = TestInsituLandBias.SimulatedObservationSource(
self.times,
self.look_up,
self.masks,
self.observations,
daynumber=self.times[index])
test_connector = ObservationStructureSourceConnector(
test_source, obs,
TimeBaseDays(datetime.datetime(1850, 1, 1)).number_to_datetime(
self.times[index]))
bias_design = bias.element_design(test_connector)
self.assertEqual(bias_design.groupname, 'insitu_land')
self.assertEqual(bias_design.number_of_biases, len(self.break_times))
self.assertEqual(bias_design.number_of_observations,
(~self.masks[obs][index, :]).sum())
numpy.testing.assert_array_equal(bias_design.effect,
numpy.array([[3, 3]]))
numpy.testing.assert_array_equal(
bias_design.design_matrix().todense(),
numpy.array([[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 1., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]]))
index = 0
obs = 'TMAX'
# valid observation indices = [ 1, 2, 3, 4, 5, 6, 7, 8]
# t = 23740 -> [[3, 0], [5, 4], [6, 6], [7, 7]]
# resulting design matrix
#[0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
#[0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
#[0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
#[1. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
#[0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
#[0. 0. 0. 0. 1. 0. 0. 0. 0. 0.]
#[0. 0. 0. 0. 0. 0. 1. 0. 0. 0.]
#[0. 0. 0. 0. 0. 0. 0. 1. 0. 0.]
test_source = TestInsituLandBias.SimulatedObservationSource(
self.times,
self.look_up,
self.masks,
self.observations,
daynumber=self.times[index])
test_connector = ObservationStructureSourceConnector(
test_source, obs,
TimeBaseDays(datetime.datetime(1850, 1, 1)).number_to_datetime(
self.times[index]))
bias_design = bias.element_design(test_connector)
self.assertEqual(bias_design.groupname, 'insitu_land')
self.assertEqual(bias_design.number_of_biases, len(self.break_times))
self.assertEqual(bias_design.number_of_observations,
(~self.masks[obs][index, :]).sum())
numpy.testing.assert_array_equal(
bias_design.effect, numpy.array([[3, 0], [5, 4], [6, 6], [7, 7]]))
numpy.testing.assert_array_equal(
bias_design.design_matrix().todense(),
numpy.array([[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[1., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 1., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 1., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 1., 0., 0.]]))
