def __init__(self,
bias_terms=False,
global_biases_group_list=[],
local_hyperparameter_file=None):
setup = ShortScaleSetup(local_hyperparameter_file)
bias_elements, bias_hyperparameters = [], []
if bias_terms:
for groupname in global_biases_group_list:
if (groupname == 'surfaceairmodel_land_global') or (
groupname == 'surfaceairmodel_ocean_global'):
bias_elements.append(BiasElement(groupname, 1))
bias_hyperparameters.append(
CovariateHyperparameters(
numpy.log(setup.bias_settings.bias_amplitude)))
elif (groupname == 'surfaceairmodel_ice_global'):
bias_elements.append(BiasElement(groupname, 2))
bias_hyperparameters.append(
CovariateHyperparameters(
numpy.log(setup.bias_settings.bias_amplitude)))
local_hyperparameters = ExpandedLocalHyperparameters(log_sigma=None,
log_rho=None)
local_hyperparameters.values_from_npy_savefile(
local_hyperparameter_file)
super(NonStationaryLocalDefinition, self).__init__(
CombinationElement([
NonStationaryLocal(
setup.local_settings.n_triangulation_divisions)
] + bias_elements),
CombinationHyperparameters([local_hyperparameters] +
bias_hyperparameters))
def test_init(self):
p = CombinationHyperparameters(
[CovariateHyperparameters(23.6),
CovariateHyperparameters(22.9)])
self.assertEqual(2, len(p.elementparameters))
self.assertEqual(23.6, p.elementparameters[0].value)
self.assertEqual(22.9, p.elementparameters[1].value)
def test_get_element_ranges(self):
p = CombinationHyperparameters([
CovariateHyperparameters(23.6),
LocalHyperparameters(log_sigma=0.1, log_rho=1.2),
CovariateHyperparameters(24.7)
])
self.assertEqual([[0], [1, 2], [3]], p.get_element_ranges())
def test_prior_precision_derivative(self):
dQ = CovariatePrior(CovariateHyperparameters(-0.5 * numpy.log(1.1)),
7).prior_precision_derivative(0)
self.assertEqual(SPARSEFORMAT, dQ.getformat())
self.assertEqual((7, 7), dQ.shape)
self.assertEqual(7, dQ.nnz)
self.assertAlmostEqual(-2.2, dQ[1, 1])
with self.assertRaises(ValueError):
CovariatePrior(CovariateHyperparameters(-0.5 * numpy.log(1.1)),
7).prior_precision_derivative(1)
def test_prior_number_of_state_parameters(self):
h = CombinationHyperparameters(
[CovariateHyperparameters(0.1),
CovariateHyperparameters(0.2)])
priorlist = [
CovariatePrior(h.elementparameters[0],
number_of_state_parameters=4),
CovariatePrior(h.elementparameters[1],
number_of_state_parameters=5)
]
prior = CombinationPrior(h, priorlist)
self.assertEqual(9, prior.prior_number_of_state_parameters())
def test_set_array(self):
p = CombinationHyperparameters([
CovariateHyperparameters(23.6),
LocalHyperparameters(log_sigma=0.1, log_rho=1.2),
CovariateHyperparameters(24.7)
])
p.set_array(numpy.array([1.4, 1.5, 1.6, 1.7]))
self.assertEqual(3, len(p.elementparameters))
self.assertEqual(1.4, p.elementparameters[0].value)
self.assertEqual(1.5, p.elementparameters[1].log_sigma)
self.assertEqual(1.6, p.elementparameters[1].log_rho)
self.assertEqual(1.7, p.elementparameters[2].value)
numpy.testing.assert_equal([1.4, 1.5, 1.6, 1.7], p.get_array())
def test_element_prior(self):
prior = GrandMeanElement().element_prior(
CovariateHyperparameters(3.3333))
self.assertTrue(isinstance(prior, CovariatePrior))
self.assertEqual(1, prior.number_of_state_parameters)
self.assertEqual(3.3333, prior.hyperparameters.value)
def test_get_array(self):
p = CombinationHyperparameters([
CovariateHyperparameters(23.6),
LocalHyperparameters(log_sigma=0.1, log_rho=1.2)
])
numpy.testing.assert_equal([23.6, 0.1, 1.2], p.get_array())
def test_mini_world_geography_based_mock_data(self):
"""Testing on a simple mock data file, with mock covariate values"""
# GENERATING OBSERVATIONS
# Simulated locations: they will exactly sits on the grid points of the covariate datafile
locations = numpy.array([[0.0, 0.0], [0.0, 0.5], [0.05, 0.0]])
# Simulated measurements: simple linear relation of type: y = 2*x
measurement = numpy.array([2., 2., 2.])
# Simulated errors
uncorrelatederror = 0.1 * numpy.ones(measurement.shape)
# Simulated inputs
simulated_input_loader = SimulatedInputLoader(locations, measurement,
uncorrelatederror)
# Simulate evaluation of this time index
simulated_time_indices = [0]
# GENERATING THE MODEL
# Local component
geography_covariate_element = GeographyBasedElement(
self.covariate_file.name, 'lat', 'lon', 'covariate', 1.0)
geography_covariate_element.load()
geography_based_component = SpatialComponent(
ComponentStorage_InMemory(
geography_covariate_element,
CovariateHyperparameters(-0.5 * numpy.log(10.))),
SpatialComponentSolutionStorage_InMemory())
# GENERATING THE ANALYSIS
# Analysis system using the specified components, for the Tmean observable
analysis_system = AnalysisSystem([geography_based_component],
ObservationSource.TMEAN,
log=StringIO())
# Update with data
analysis_system.update([simulated_input_loader],
simulated_time_indices)
# Check state vector directly
statevector = analysis_system.components[
0].solutionstorage.partial_state_read(0).ravel()
# These are the nodes where observations were put (see SimulatedObservationSource above)
# - check they correspond to within 3 times the stated noise level
self.assertAlmostEqual(2., statevector[0], delta=0.3)
# Also check entire state vector within outer bounds set by obs
self.assertTrue(all(statevector < 2.0))
# And check output corresponds too
# (evaluate result on output structure same as input)
simulated_output_structure = SimulatedObservationStructure(
0, locations, None, None)
result = analysis_system.evaluate_expected_value(
'MAP', simulated_output_structure, flag='POINTWISE')
numpy.testing.assert_almost_equal(
statevector[0] * numpy.ones(len(measurement)), result)
def test_element_prior(self):
prior = BiasElement('Bob', 42).element_prior(CovariateHyperparameters(9.9))
self.assertTrue(isinstance(prior, CovariatePrior))
self.assertEqual(9.9, prior.hyperparameters.value)
# The number of state parameters should be the total number of biases (irrespective of number observed)
self.assertEqual(42, prior.number_of_state_parameters)
def test_prior_precision(self):
Q = CovariatePrior(CovariateHyperparameters(-0.5 * numpy.log(1.1)),
7).prior_precision()
self.assertEqual(SPARSEFORMAT, Q.getformat())
self.assertEqual((7, 7), Q.shape)
self.assertEqual(7, Q.nnz)
numpy.testing.assert_almost_equal(
Q.todense(), numpy.diag([1.1, 1.1, 1.1, 1.1, 1.1, 1.1, 1.1]))
def test_prior_precision(self):
h = CombinationHyperparameters([
CovariateHyperparameters(-0.5 * numpy.log(23.6)),
CovariateHyperparameters(-0.5 * numpy.log(88.9))
])
priorlist = [
CovariatePrior(h.elementparameters[0],
number_of_state_parameters=2),
CovariatePrior(h.elementparameters[1],
number_of_state_parameters=3)
]
prior = CombinationPrior(h, priorlist)
Q = prior.prior_precision()
self.assertEqual(SPARSEFORMAT, Q.getformat())
numpy.testing.assert_almost_equal(
Q.todense(),
[[23.6, 0.0, 0.0, 0.0, 0.0], [0.0, 23.6, 0.0, 0.0, 0.0],
[0.0, 0.0, 88.9, 0.0, 0.0], [0.0, 0.0, 0.0, 88.9, 0.0],
[0.0, 0.0, 0.0, 0.0, 88.9]])
def test_prior_precision_derivative(self):
h = CombinationHyperparameters([
CovariateHyperparameters(-0.5 * numpy.log(23.6)),
CovariateHyperparameters(-0.5 * numpy.log(88.9))
])
priorlist = [
CovariatePrior(h.elementparameters[0],
number_of_state_parameters=2),
CovariatePrior(h.elementparameters[1],
number_of_state_parameters=3)
]
prior = CombinationPrior(h, priorlist)
dQ = prior.prior_precision_derivative(1)
self.assertEqual(SPARSEFORMAT, dQ.getformat())
self.assertEqual((5, 5), dQ.shape)
self.assertEqual(3, dQ.nnz)
self.assertAlmostEqual(-2.0 * 88.9, dQ[2, 2])
self.assertAlmostEqual(-2.0 * 88.9, dQ[3, 3])
self.assertAlmostEqual(-2.0 * 88.9, dQ[4, 4])
def test_element_prior(self):
prior = InsituLandBiasElement(
self.breakpoints_file.name).element_prior(
CovariateHyperparameters(9.9))
self.assertTrue(isinstance(prior, CovariatePrior))
self.assertEqual(9.9, prior.hyperparameters.value)
# The number of state parameters should be the total number of biases (irrespective of number observed)
self.assertEqual(10, prior.number_of_state_parameters)
def test_element_prior(self):
hyperparameters = CombinationHyperparameters([
CovariateHyperparameters(23.6),
LocalHyperparameters(log_sigma=0.1, log_rho=1.2)
])
prior = CombinationElement([GrandMeanElement(),
LocalElement(0)
]).element_prior(hyperparameters)
self.assertTrue(isinstance(prior, CombinationPrior))
self.assertEqual(2, len(prior.priorlist))
self.assertTrue(isinstance(prior.priorlist[0], CovariatePrior))
self.assertTrue(isinstance(prior.priorlist[1], LocalPrior))
def atest_process_observations_compute_uncertainties(self):
# Our test system is:
#
# ( [ 2.0 0.0 ] + [ -1.5 0.0 ] [ 5.0 0.0 ] [ -1.5 2.2 ] ) x = [ -1.5 0.0 ] [ 10.0 3.0 ] [ 7.0 - 2.0 ]
# ( [ 0.0 2.0 ] [ 2.2 3.3 ] [ 0.0 5.0 ] [ 0.0 3.3 ] ) [ 2.2 3.3 ] [ 3.0 10.0 ] [ 9.0 - 3.0 ]
#
# [ 3.25 -16.5 ] x = [ -37.5 ]
# [-16.5 80.65 ] [ 154.0 ]
#
# => x = [ -0.60697861 ]
# [ 1.78530506 ]
#
c = DelayedSpatialComponent(
ComponentStorage_InMemory(
TestDelayedSpatialComponentSolution.TestElement(),
CovariateHyperparameters(-0.5 * numpy.log(2.0))),
DelayedSpatialComponentSolutionStorage_Files(),
compute_uncertainties=True)
s = c.component_solution()
self.assertIsInstance(s, DelayedSpatialComponentSolution)
self.assertTrue(s.compute_uncertainties)
test_offset = numpy.array([2.0, 3.0])
s.process_observations(
TestDelayedSpatialComponentSolution.TestObservations(t=21),
test_offset[0:1])
s.update_time_step()
s.process_observations(
TestDelayedSpatialComponentSolution.TestObservations(t=532),
test_offset[1:2])
s.update_time_step()
s.update()
# In this case we are considering the last iteration of model solving, hence marginal variances should have been stored
expected_marginal_std = numpy.sqrt(
numpy.diag(
numpy.linalg.inv(numpy.array([[13.25, -16.5], [-16.5,
80.65]]))))
numpy.testing.assert_array_almost_equal(
s.solutionstorage.state_marginal_std, expected_marginal_std)
# Now we compute the projection of marginal variances onto the given observations
for time in [532]:
# Observation at time t=t* should be design matrix for that time multiplied by expected state
expected_projection = TestDelayedSpatialComponentSolution.TestDesign(
t=time).design_function(expected_marginal_std)
numpy.testing.assert_almost_equal(
s.solution_observation_expected_uncertainties(
TestDelayedSpatialComponentSolution.TestObservations(
t=time)), expected_projection)
def __init__(self, bias_terms = False, global_biases_group_list = []):
setup = LocalSetup()
if bias_terms:
bias_elements = [BiasElement(groupname, 1) for groupname in global_biases_group_list]
bias_hyperparameters = [CovariateHyperparameters(numpy.log(setup.bias_amplitude)) for index in range(len(global_biases_group_list))]
else:
bias_elements, bias_hyperparameters = [], []
super(LocalDefinition, self).__init__(
CombinationElement([LocalElement(setup.n_triangulation_divisions)] + bias_elements),
CombinationHyperparameters([LocalHyperparameters(numpy.log(setup.amplitude),
numpy.log(numpy.radians(setup.space_length_scale)))] + bias_hyperparameters))
def test_process_observations_compute_sample(self):
# Our test system is:
#
# ( [ 2.0 0.0 ] + [ -1.5 0.0 ] [ 5.0 0.0 ] [ -1.5 2.2 ] ) x = [ -1.5 0.0 ] [ 10.0 3.0 ] [ 7.0 - 2.0 ]
# ( [ 0.0 2.0 ] [ 2.2 3.3 ] [ 0.0 5.0 ] [ 0.0 3.3 ] ) [ 2.2 3.3 ] [ 3.0 10.0 ] [ 9.0 - 3.0 ]
#
# [ 13.25 -16.5 ] x = [ -37.5 ]
# [-16.5 80.65 ] [ 154.0 ]
#
# => x = [ -0.60697861 ]
# [ 1.78530506 ]
#
number_of_samples = 200
c = SpaceTimeComponent(ComponentStorage_InMemory(
TestSpaceTimeComponentSolution.TestElement(),
CovariateHyperparameters(-0.5 * numpy.log(2.0))),
SpaceTimeComponentSolutionStorage_InMemory(),
compute_sample=True,
sample_size=number_of_samples)
s = c.component_solution()
self.assertIsInstance(s, SpaceTimeComponentSolution)
self.assertTrue(s.compute_sample)
test_offset = numpy.array([2.0, 3.0])
s.process_observations(
TestSpaceTimeComponentSolution.TestObservations(t=21),
test_offset[0:1])
s.update_time_step()
s.process_observations(
TestSpaceTimeComponentSolution.TestObservations(t=532),
test_offset[1:2])
s.update_time_step()
numpy.random.seed(0)
s.update()
# In this case we are considering the last iteration of model solving, hence sample should have been stored
computed_sample = s.solutionstorage.state_sample
self.assertEqual((2, number_of_samples), computed_sample.shape)
numpy.random.seed(0)
variate = scipy.random.normal(0.0, 1.0, (2, number_of_samples))
expected_posterior_precision = numpy.array([[13.25, -16.5],
[-16.5, 80.65]])
# check result
numpy.testing.assert_almost_equal(
numpy.dot(variate.T, variate),
numpy.dot(computed_sample.T,
numpy.dot(expected_posterior_precision,
computed_sample)))
def test_element_prior(self):
element = LatitudeHarmonicsElement()
prior = element.element_prior(
CombinationHyperparameters(
[CovariateHyperparameters(c) for c in [1.0, 1.1, 1.2, 1.3]]))
self.assertIsInstance(prior, LatitudeHarmonicsPrior)
# As an example check precision - should be diagonals with exp(-2 x hyperparameter)
precision = prior.prior_precision()
self.assertEqual(SPARSEFORMAT, precision.getformat())
self.assertEqual(4, precision.nnz)
self.assertAlmostEqual(numpy.exp(-2.0), precision[0, 0])
self.assertAlmostEqual(numpy.exp(-2.2), precision[1, 1])
self.assertAlmostEqual(numpy.exp(-2.4), precision[2, 2])
self.assertAlmostEqual(numpy.exp(-2.6), precision[3, 3])
def __init__(self, bias_terms = False, global_biases_group_list = [], local_hyperparameter_file = None):
setup = NonStationaryLocalSetup()
if bias_terms:
bias_elements = [BiasElement(groupname, 1) for groupname in global_biases_group_list]
bias_hyperparameters = [CovariateHyperparameters(numpy.log(setup.bias_amplitude)) for index in range(len(global_biases_group_list))]
else:
bias_elements, bias_hyperparameters = [], []
local_hyperparameters = ExpandedLocalHyperparameters(log_sigma = None, log_rho = None)
local_hyperparameters.values_from_npy_savefile(local_hyperparameter_file)
super(NonStationaryLocalDefinition, self).__init__(
CombinationElement([NonStationaryLocal(setup.n_triangulation_divisions)] + bias_elements),
CombinationHyperparameters([local_hyperparameters]+bias_hyperparameters))
def __init__(self, bias_terms=False, breakpoints_file=None):
setup = MidScaleSetup()
if bias_terms:
bias_element = [
InsituLandBiasElement(breakpoints_file,
apply_policy=True,
cut_value=3)
]
bias_hyperparameters = [
CovariateHyperparameters(
numpy.log(setup.bias_settings.bias_amplitude))
]
else:
bias_element, bias_hyperparameters = [], []
super(LargeScaleDefinition, self).__init__(
CombinationElement([
SpaceTimeKroneckerElement(
setup.midscale_settings.n_triangulation_divisions, setup.
midscale_settings.alpha, setup.midscale_settings.starttime,
setup.midscale_settings.endtime, setup.midscale_settings.
n_nodes, setup.midscale_settings.overlap_factor,
setup.midscale_settings.H),
AnnualKroneckerElement(
setup.slow_settings.n_triangulation_divisions,
setup.slow_settings.alpha, setup.slow_settings.starttime,
setup.slow_settings.endtime, setup.slow_settings.n_nodes,
setup.slow_settings.overlap_factor, setup.slow_settings.H),
] + bias_element),
CombinationHyperparameters([
SpaceTimeSPDEHyperparameters(
numpy.log(setup.midscale_settings.amplitude),
numpy.log(
numpy.radians(
setup.midscale_settings.space_length_scale)),
numpy.log(setup.midscale_settings.time_length_scale)),
SpaceTimeSPDEHyperparameters(
numpy.log(setup.slow_settings.amplitude),
numpy.log(
numpy.radians(setup.slow_settings.space_length_scale)),
numpy.log(setup.slow_settings.time_length_scale)),
] + bias_hyperparameters))
def test_element_prior(self):
geography = GeographyBasedElement(self.covariate_file.name, 'lat',
'lon', 'covariate', 1.0)
value = 3.333
prior = geography.element_prior(CovariateHyperparameters(value))
self.assertIsInstance(prior, GeographyBasedPrior)
# As an example check precision - should be diagonals with exp(-2 x hyperparameter)
precision = prior.prior_precision()
self.assertEqual(SPARSEFORMAT, precision.getformat())
self.assertEqual(1, precision.nnz)
self.assertAlmostEqual(numpy.exp(-2.0 * value), precision[0, 0])
precision_derivative = prior.prior_precision_derivative(0)
self.assertEqual(SPARSEFORMAT, precision_derivative.getformat())
self.assertEqual(1, precision_derivative.nnz)
self.assertAlmostEqual(-2.0 * precision[0, 0], precision_derivative[0,
0])
def __init__(self, covariates_descriptor):
if covariates_descriptor is not None:
loader = LoadCovariateElement(covariates_descriptor)
loader.check_keys()
covariate_elements, covariate_hyperparameters = loader.load_covariates_and_hyperparameters()
print('The following fields have been added as covariates of the climatology model')
print(loader.data.keys())
else:
covariate_elements, covariate_hyperparameters = [], []
setup = ClimatologySetup()
super(ClimatologyDefinition, self).__init__(
CombinationElement( [SeasonalElement(setup.n_triangulation_divisions,
setup.n_harmonics,
include_local_mean=True), GrandMeanElement()] + covariate_elements),
CombinationHyperparameters( [SeasonalHyperparameters(setup.n_spatial_components,
numpy.log(setup.amplitude),
numpy.log(numpy.radians(setup.space_length_scale))),
CovariateHyperparameters(numpy.log(setup.grandmean_amplitude))] + covariate_hyperparameters))
def __init__(self, bias_terms=False, global_biases_group_list=[]):
setup = ShortScaleSetup()
if bias_terms:
bias_elements = [
BiasElement(groupname, 1)
for groupname in global_biases_group_list
]
bias_hyperparameters = [
CovariateHyperparameters(
numpy.log(setup.bias_settings.bias_amplitude))
for index in range(len(global_biases_group_list))
]
else:
bias_elements, bias_hyperparameters = [], []
super(PureBiasComponentDefinition,
self).__init__(CombinationElement(bias_elements),
CombinationHyperparameters(bias_hyperparameters))
def test_mini_world_altitude(self):
"""Testing using altitude as a covariate"""
# GENERATING OBSERVATIONS
# Simulated locations: they will exactly sits on the grid points of the covariate datafile
DEM = Dataset(self.altitude_datafile)
latitude = DEM.variables['lat'][:]
longitude = DEM.variables['lon'][:]
altitude = DEM.variables['dem'][:]
indices = numpy.stack(
(numpy.array([1, 3, 267, 80, 10, 215, 17, 120]),
numpy.array([2, 256, 9, 110, 290, 154, 34, 151])),
axis=1)
selected_location = []
altitude_observations = []
for couple in indices:
selected_location.append([
latitude[couple[0], couple[1]], longitude[couple[0], couple[1]]
])
altitude_observations.append(altitude[couple[0], couple[1]])
DEM.close()
locations = numpy.array(selected_location)
# Simulated measurements: simple linear relation of type: y = PI*x
measurement = numpy.pi * numpy.array(altitude_observations)
# Simulated errors
uncorrelatederror = 0.1 * numpy.ones(measurement.shape)
# Simulated inputs
simulated_input_loader = SimulatedInputLoader(locations, measurement,
uncorrelatederror)
# Simulate evaluation of this time index
simulated_time_indices = [0]
# GENERATING THE MODEL
# Local component
geography_covariate_element = GeographyBasedElement(
self.altitude_datafile, 'lat', 'lon', 'dem', 1.0)
geography_covariate_element.load()
geography_based_component = SpatialComponent(
ComponentStorage_InMemory(
geography_covariate_element,
CovariateHyperparameters(-0.5 * numpy.log(10.))),
SpatialComponentSolutionStorage_InMemory())
# GENERATING THE ANALYSIS
# Analysis system using the specified components, for the Tmean observable
analysis_system = AnalysisSystem([geography_based_component],
ObservationSource.TMEAN,
log=StringIO())
# Update with data
analysis_system.update([simulated_input_loader],
simulated_time_indices)
# Check state vector directly
statevector = analysis_system.components[
0].solutionstorage.partial_state_read(0).ravel()
# These are the nodes where observations were put (see SimulatedObservationSource above)
# - check they correspond to within 3 times the stated noise level
self.assertAlmostEqual(numpy.pi, statevector[0], delta=0.3)
# Also check entire state vector within outer bounds set by obs
self.assertTrue(all(statevector < numpy.pi))
# And check output corresponds too
# (evaluate result on output structure same as input)
simulated_output_structure = SimulatedObservationStructure(
0, locations, None, None)
result = analysis_system.evaluate_expected_value(
'MAP', simulated_output_structure, flag='POINTWISE')
numpy.testing.assert_almost_equal(
statevector[0] * numpy.array(altitude_observations), result)
def test_process_observations(self):
# Our test system is:
#
# ( [ 2.0 0.0 ] + [ -1.5 0.0 ] [ 10.0 3.0 ] [ -1.5 2.2 ] ) x = [ -1.5 0.0 ] [ 10.0 3.0 ] [ 7.0 - 2.0 ]
# ( [ 0.0 2.0 ] [ 2.2 3.3 ] [ 3.0 10.0 ] [ 0.0 3.3 ] ) [ 2.2 3.3 ] [ 3.0 10.0 ] [ 9.0 - 3.0 ]
#
# [ 24.5 -47.85 ] x = [ -102.0 ]
# [ -47.85 202.86 ] [ 397.1 ]
#
# => x = [ -0.63067268 ]
# [ 1.80874649 ]
#
# Input data
test_offset = numpy.array([2.0, 3.0])
test_obs = TestSpatialComponentSolution.TestObservations()
# Make component and check it's of the correct class
c = SpatialComponent(
ComponentStorage_InMemory(
TestSpatialComponentSolution.TestElement(),
CovariateHyperparameters(-0.5 * numpy.log(2.0))),
SpatialComponentSolutionStorage_InMemory())
s = c.component_solution()
self.assertIsInstance(s, SpatialComponentSolution)
self.assertFalse(s.compute_uncertainties)
self.assertFalse(s.compute_sample)
# Do the processing
s.process_observations(test_obs, test_offset)
s.update_time_step()
# Should be the value of x (see matrices in comment above)
numpy.testing.assert_almost_equal(
s.solutionstorage.partial_state_read(21),
numpy.array([[-0.63067268], [1.80874649]]))
# Should be the value of x multiplied by the design matrix
numpy.testing.assert_almost_equal(
s.solution_observation_expected_value(
TestSpatialComponentSolution.TestObservations()),
numpy.array([[-1.5 * -0.63067268 + 2.2 * 1.80874649],
[3.3 * 1.80874649]]))
# This should be zero as we have no information for other times
numpy.testing.assert_almost_equal(
s.solution_observation_expected_value(
TestSpatialComponentSolution.TestSomeOtherTime()),
numpy.zeros((7, )))
# In this case we did not allowed the computation of uncertainties or samples, hence the state of marginal variances should be None, while the expected uncertainties zero
self.assertEqual(s.solutionstorage.partial_state_marginal_std_read(21),
None)
self.assertEqual(s.solutionstorage.state_marginal_std_at_time, {})
numpy.testing.assert_almost_equal(
s.solution_observation_expected_uncertainties(
TestSpatialComponentSolution.TestObservations()), 0.)
self.assertEqual(s.solutionstorage.partial_state_sample_read(21), None)
self.assertEqual(s.solutionstorage.state_sample_at_time, {})
numpy.testing.assert_almost_equal(
s.solution_observation_projected_sample(
TestSpatialComponentSolution.TestObservations()), 0.)
def test_process_observations_compute_sample(self):
# Our test system is:
#
# ( [ 2.0 0.0 ] + [ -1.5 0.0 ] [ 10.0 3.0 ] [ -1.5 2.2 ] ) x = [ -1.5 0.0 ] [ 10.0 3.0 ] [ 7.0 - 2.0 ]
# ( [ 0.0 2.0 ] [ 2.2 3.3 ] [ 3.0 10.0 ] [ 0.0 3.3 ] ) [ 2.2 3.3 ] [ 3.0 10.0 ] [ 9.0 - 3.0 ]
#
# [ 24.5 -47.85 ] x = [ -102.0 ]
# [ -47.85 202.86 ] [ 397.1 ]
#
# => x = [ -0.63067268 ]
# [ 1.80874649 ]
#
# Input data
test_offset = numpy.array([2.0, 3.0])
test_obs = TestSpatialComponentSolution.TestObservations()
sample_size = 300
# Make component and check it's of the correct class
c = SpatialComponent(ComponentStorage_InMemory(
TestSpatialComponentSolution.TestElement(),
CovariateHyperparameters(-0.5 * numpy.log(2.0))),
SpatialComponentSolutionStorage_InMemory(),
compute_sample=True,
sample_size=sample_size)
s = c.component_solution()
self.assertIsInstance(s, SpatialComponentSolution)
self.assertFalse(s.compute_uncertainties)
self.assertTrue(s.compute_sample)
# Do the processing
s.process_observations(test_obs, test_offset)
numpy.random.seed(0)
s.update_time_step()
computed_sample = s.solutionstorage.partial_state_sample_read(21)
self.assertEqual((2, sample_size), computed_sample.shape)
numpy.random.seed(0)
variate = scipy.random.normal(0.0, 1.0, (2, sample_size))
expected_posterior_precision = numpy.array([[24.5, -47.85],
[-47.85, 202.86]])
self.assertEqual(len(s.solutionstorage.state_sample_at_time), 1)
self.assertListEqual(s.solutionstorage.state_sample_at_time.keys(),
[21])
# check result
numpy.testing.assert_almost_equal(
numpy.dot(variate.T, variate),
numpy.dot(computed_sample.T,
numpy.dot(expected_posterior_precision,
computed_sample)))
# In this case we did not allowed the computation of uncertainties
self.assertEqual(s.solutionstorage.partial_state_marginal_std_read(21),
None)
self.assertEqual(s.solutionstorage.state_marginal_std_at_time, {})
numpy.testing.assert_almost_equal(
s.solution_observation_expected_uncertainties(
TestSpatialComponentSolution.TestObservations()), 0.)
def test_process_observations_compute_uncertainties(self):
# Our test system is:
#
# ( [ 2.0 0.0 ] + [ -1.5 0.0 ] [ 10.0 3.0 ] [ -1.5 2.2 ] ) x = [ -1.5 0.0 ] [ 10.0 3.0 ] [ 7.0 - 2.0 ]
# ( [ 0.0 2.0 ] [ 2.2 3.3 ] [ 3.0 10.0 ] [ 0.0 3.3 ] ) [ 2.2 3.3 ] [ 3.0 10.0 ] [ 9.0 - 3.0 ]
#
# [ 24.5 -47.85 ] x = [ -102.0 ]
# [ -47.85 202.86 ] [ 397.1 ]
#
# => x = [ -0.63067268 ]
# [ 1.80874649 ]
#
# Input data
test_offset = numpy.array([2.0, 3.0])
test_obs = TestSpatialComponentSolution.TestObservations()
# Make component and check it's of the correct class
c = SpatialComponent(ComponentStorage_InMemory(
TestSpatialComponentSolution.TestElement(),
CovariateHyperparameters(-0.5 * numpy.log(2.0))),
SpatialComponentSolutionStorage_InMemory(),
compute_uncertainties=True,
sample_size=666)
s = c.component_solution()
self.assertIsInstance(s, SpatialComponentSolution)
self.assertTrue(s.compute_uncertainties)
self.assertFalse(s.compute_sample)
# Do the processing
s.process_observations(test_obs, test_offset)
s.update_time_step()
# In this case we are considering the last iteration of model solving, hence marginal variances should have been stored
expected_marginal_std = numpy.sqrt(
numpy.diag(
numpy.linalg.inv(
numpy.array([[24.5, -47.85], [-47.85, 202.86]]))))
numpy.testing.assert_array_almost_equal(
s.solutionstorage.partial_state_marginal_std_read(21),
expected_marginal_std)
self.assertEqual(len(s.solutionstorage.state_marginal_std_at_time), 1)
self.assertListEqual(
s.solutionstorage.state_marginal_std_at_time.keys(), [21])
# We also test the computation of prior marginal variances, and their projection onto observations
expected_prior_std = numpy.sqrt(
numpy.diag(numpy.linalg.inv(numpy.array([[2., 0.], [0., 2.]]))))
numpy.testing.assert_array_almost_equal(s.solution_prior_std(100),
expected_prior_std,
decimal=1)
numpy.testing.assert_array_almost_equal(s.solution_prior_std(10000),
expected_prior_std,
decimal=2)
design_matrix = numpy.array([[-1.5, 2.2], [0.0, 3.3]])
expected_prior_std_projection = numpy.dot(design_matrix,
expected_prior_std)
numpy.testing.assert_array_almost_equal(
s.solution_observation_prior_uncertainties(None),
expected_prior_std_projection,
decimal=1)
# In this case we did not allowed the computation of uncertainties samples
self.assertEqual(s.solutionstorage.partial_state_sample_read(21), None)
self.assertEqual(s.solutionstorage.state_sample_at_time, {})
numpy.testing.assert_almost_equal(
s.solution_observation_projected_sample(
TestSpatialComponentSolution.TestObservations()), 0.)
self.assertEqual(
666,
s.solution_observation_projected_sample(
TestSpatialComponentSolution.TestObservations()).shape[1])
def main():
print 'Advanced standard example using a few days of EUSTACE data'
parser = argparse.ArgumentParser(description='Advanced standard example using a few days of EUSTACE data')
parser.add_argument('outpath', help='directory where the output should be redirected')
parser.add_argument('--json_descriptor', default = None, help='a json descriptor containing the covariates to include in the climatology model')
parser.add_argument('--land_biases', action='store_true', help='include insitu land homogenization bias terms')
parser.add_argument('--global_biases', action='store_true', help='include global satellite bias terms')
parser.add_argument('--n_iterations', type=int, default=5, help='number of solving iterations')
args = parser.parse_args()
# Input data path
basepath = os.path.join('/work/scratch/eustace/rawbinary3')
# Days to process
time_indices = range(int(days_since_epoch(datetime(2006, 2, 1))), int(days_since_epoch(datetime(2006, 2, 2))))
# Sources to use
sources = [ 'surfaceairmodel_land', 'surfaceairmodel_ocean', 'surfaceairmodel_ice', 'insitu_land', 'insitu_ocean' ]
#SETUP
# setup for the seasonal core: climatology covariates setup read from file
seasonal_setup = {'n_triangulation_divisions':5,
'n_harmonics':4,
'n_spatial_components':6,
'amplitude':2.,
'space_length_scale':5., # length scale in units of degrees
}
grandmean_amplitude = 15.0
# setup for the large scale component
spacetime_setup = {'n_triangulation_divisions':2,
'alpha':2,
'starttime':0,
'endtime':10.,
'n_nodes':2,
'overlap_factor':2.5,
'H':1,
'amplitude':1.,
'space_lenght_scale':15.0, # length scale in units of degrees
'time_length_scale':15.0 # length scal in units of days
}
bias_amplitude = .9
# setup for the local component
local_setup = {'n_triangulation_divisions':6,
'amplitude':2.,
'space_length_scale':2. # length scale in units of degrees
}
globalbias_amplitude = 15.0
# CLIMATOLOGY COMPONENT: combining the seasonal core along with latitude harmonics, altitude and coastal effects
if args.json_descriptor is not None:
loader = LoadCovariateElement(args.json_descriptor)
loader.check_keys()
covariate_elements, covariate_hyperparameters = loader.load_covariates_and_hyperparameters()
print('The following fields have been added as covariates of the climatology model')
print(loader.data.keys())
else:
covariate_elements, covariate_hyperparameters = [], []
climatology_element = CombinationElement( [SeasonalElement(n_triangulation_divisions=seasonal_setup['n_triangulation_divisions'],
n_harmonics=seasonal_setup['n_harmonics'],
include_local_mean=True),
GrandMeanElement()]+covariate_elements)
climatology_hyperparameters = CombinationHyperparameters( [SeasonalHyperparameters(n_spatial_components=seasonal_setup['n_spatial_components'],
common_log_sigma=numpy.log(seasonal_setup['amplitude']),
common_log_rho=numpy.log(numpy.radians(seasonal_setup['space_length_scale']))),
CovariateHyperparameters(numpy.log(grandmean_amplitude))] + covariate_hyperparameters )
climatology_component = SpaceTimeComponent(ComponentStorage_InMemory(climatology_element, climatology_hyperparameters), SpaceTimeComponentSolutionStorage_InMemory(),
compute_uncertainties=True, method='APPROXIMATED',
compute_sample=True, sample_size=definitions.GLOBAL_SAMPLE_SHAPE[3])
# LARGE SCALE (kronecker product) COMPONENT: combining large scale trends with bias terms accounting for homogeneization effects
if args.land_biases:
bias_element, bias_hyperparameters = [InsituLandBiasElement(BREAKPOINTS_FILE)], [CovariateHyperparameters(numpy.log(bias_amplitude))]
print('Adding bias terms for insitu land homogenization')
else:
bias_element, bias_hyperparameters = [], []
large_scale_element = CombinationElement( [SpaceTimeKroneckerElement(n_triangulation_divisions=spacetime_setup['n_triangulation_divisions'],
alpha=spacetime_setup['alpha'],
starttime=spacetime_setup['starttime'],
endtime=spacetime_setup['endtime'],
n_nodes=spacetime_setup['n_nodes'],
overlap_factor=spacetime_setup['overlap_factor'],
H=spacetime_setup['H'])] + bias_element)
large_scale_hyperparameters = CombinationHyperparameters( [SpaceTimeSPDEHyperparameters(space_log_sigma=numpy.log(spacetime_setup['amplitude']),
space_log_rho=numpy.log(numpy.radians(spacetime_setup['space_lenght_scale'])),
time_log_rho=numpy.log(spacetime_setup['time_length_scale']))] + bias_hyperparameters)
large_scale_component = SpaceTimeComponent(ComponentStorage_InMemory(large_scale_element, large_scale_hyperparameters), SpaceTimeComponentSolutionStorage_InMemory(),
compute_uncertainties=True, method='APPROXIMATED',
compute_sample=True, sample_size=definitions.GLOBAL_SAMPLE_SHAPE[3])
# LOCAL COMPONENT: combining local scale variations with global satellite bias terms
if args.global_biases:
bias_elements = [BiasElement(groupname, 1) for groupname in GLOBAL_BIASES_GROUP_LIST]
bias_hyperparameters = [CovariateHyperparameters(numpy.log(globalbias_amplitude)) for index in range(len(GLOBAL_BIASES_GROUP_LIST))]
print('Adding global bias terms for all the surfaces')
else:
bias_elements, bias_hyperparameters = [], []
local_scale_element = CombinationElement([LocalElement(n_triangulation_divisions=local_setup['n_triangulation_divisions'])] + bias_elements)
local_scale_hyperparameters = CombinationHyperparameters([LocalHyperparameters(log_sigma=numpy.log(local_setup['amplitude']),
log_rho=numpy.log(numpy.radians(local_setup['space_length_scale'])))] + bias_hyperparameters)
local_component = SpatialComponent(ComponentStorage_InMemory(local_scale_element, local_scale_hyperparameters), SpatialComponentSolutionStorage_InMemory(),
compute_uncertainties=True, method='APPROXIMATED',
compute_sample=True, sample_size=definitions.GLOBAL_SAMPLE_SHAPE[3])
# Analysis system using the specified components, for the Tmean observable
print 'Analysing inputs'
analysis_system = AnalysisSystem(
[ climatology_component, large_scale_component, local_component ],
ObservationSource.TMEAN)
# Object to load raw binary inputs at time indices
inputloaders = [ AnalysisSystemInputLoaderRawBinary_Sources(basepath, source, time_indices) for source in sources ]
for iteration in range(args.n_iterations):
message = 'Iteration {}'.format(iteration)
print(message)
# Update with data
analysis_system.update(inputloaders, time_indices)
print 'Computing outputs'
# Produce an output for each time index
for time_index in time_indices:
# Get date for output
outputdate = inputloaders[0].datetime_at_time_index(time_index)
print 'Evaluating output grid: ', outputdate
#Configure output grid
outputstructure = OutputRectilinearGridStructure(
time_index, outputdate,
latitudes=numpy.linspace(-90.+definitions.GLOBAL_FIELD_RESOLUTION/2., 90.-definitions.GLOBAL_FIELD_RESOLUTION/2., num=definitions.GLOBAL_FIELD_SHAPE[1]),
longitudes=numpy.linspace(-180.+definitions.GLOBAL_FIELD_RESOLUTION/2., 180.-definitions.GLOBAL_FIELD_RESOLUTION/2., num=definitions.GLOBAL_FIELD_SHAPE[2]))
# Evaluate expected value at these locations
for field in ['MAP', 'post_STD']:
print 'Evaluating: ',field
result_expected_value = analysis_system.evaluate_expected_value('MAP', outputstructure, 'GRID_CELL_AREA_AVERAGE', [1,1], 1000)
result_expected_uncertainties = analysis_system.evaluate_expected_value('post_STD', outputstructure, 'GRID_CELL_AREA_AVERAGE', [1,1], 1000)
print 'Evaluating: climatology fraction'
climatology_fraction = analysis_system.evaluate_climatology_fraction(outputstructure, [1,1], 1000)
print 'Evaluating: the sample'
sample = analysis_system.evaluate_projected_sample(outputstructure)
# Make output filename
pathname = 'eustace_example_output_{0:04d}{1:02d}{2:02d}.nc'.format(outputdate.year, outputdate.month, outputdate.day)
pathname = os.path.join(args.outpath, pathname)
print 'Saving: ', pathname
# Save results
filebuilder = FileBuilderGlobalField(
pathname,
time_index,
'Infilling Example',
'UNVERSIONED',
definitions.TAS.name,
'',
'Example data only',
'eustace.analysis.advanced_standard.examples.example_eustace_few_days',
'')
filebuilder.add_global_field(definitions.TAS, result_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(definitions.TASUNCERTAINTY, result_expected_uncertainties.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(definitions.TAS_CLIMATOLOGY_FRACTION, climatology_fraction.reshape(definitions.GLOBAL_FIELD_SHAPE))
for index in range(definitions.GLOBAL_SAMPLE_SHAPE[3]):
variable = copy.deepcopy(definitions.TASENSEMBLE)
variable.name = variable.name + '_' + str(index)
selected_sample = sample[:,index].ravel()+result_expected_value
filebuilder.add_global_field(variable, selected_sample.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.save_and_close()
print 'Complete'
def test_mini_world_altitude_with_latitude(self):
"""Testing using altitude as a covariate"""
# GENERATING OBSERVATIONS
# Simulated locations: they will exactly sits on the grid points of the covariate datafile
DEM = Dataset(self.altitude_datafile)
latitude = DEM.variables['lat'][:]
longitude = DEM.variables['lon'][:]
altitude = DEM.variables['dem'][:]
indices = numpy.stack(
(numpy.array([1, 3, 5, 7, 8, 9, 10, 11
]), numpy.array([0, 0, 0, 0, 0, 0, 0, 0])),
axis=1)
selected_location = []
altitude_observations = []
for couple in indices:
selected_location.append([
latitude[couple[0], couple[1]], longitude[couple[0], couple[1]]
])
altitude_observations.append(altitude[couple[0], couple[1]])
DEM.close()
locations = numpy.array(selected_location)
# Simulated model is y = z + a*cos(2x) + c*cos(4*x) + b*sin(2x) + d*sin(4*x), with z = altitude, x = latitude, a=b=c=d=0
slope = 1e-3
measurement = slope * numpy.array(altitude_observations)
# Simulated errors
uncorrelatederror = 0.1 * numpy.ones(measurement.shape)
# Simulated inputs
simulated_input_loader = SimulatedInputLoader(locations, measurement,
uncorrelatederror)
# Simulate evaluation of this time index
simulated_time_indices = [0]
# GENERATING THE MODEL
# Local component
geography_covariate_element = GeographyBasedElement(
self.altitude_datafile, 'lat', 'lon', 'dem', 1.0)
geography_covariate_element.load()
combined_element = CombinationElement(
[geography_covariate_element,
LatitudeHarmonicsElement()])
combined_hyperparamters = CombinationHyperparameters([
CovariateHyperparameters(-0.5 * numpy.log(10.)),
CombinationHyperparameters([
CovariateHyperparameters(-0.5 * numpy.log(p))
for p in [10.0, 10.0, 10.0, 10.0]
])
])
combined_component = SpatialComponent(
ComponentStorage_InMemory(combined_element,
combined_hyperparamters),
SpatialComponentSolutionStorage_InMemory())
# GENERATING THE ANALYSIS
# Analysis system using the specified components, for the Tmean observable
analysis_system = AnalysisSystem([combined_component],
ObservationSource.TMEAN,
log=StringIO())
# Update with data
analysis_system.update([simulated_input_loader],
simulated_time_indices)
# Check state vector directly
statevector = analysis_system.components[
0].solutionstorage.partial_state_read(0).ravel()
# These are the nodes where observations were put (see SimulatedObservationSource above)
# - check they correspond to within 3 times the stated noise level
self.assertAlmostEqual(slope, statevector[0], delta=0.3)
self.assertAlmostEqual(0., statevector[1], delta=0.3)
self.assertAlmostEqual(0., statevector[2], delta=0.3)
self.assertAlmostEqual(0., statevector[3], delta=0.3)
self.assertAlmostEqual(0., statevector[4], delta=0.3)
# And check output corresponds too
# (evaluate result on output structure same as input)
simulated_output_structure = SimulatedObservationStructure(
0, locations, None, None)
result = analysis_system.evaluate_expected_value(
'MAP', simulated_output_structure, flag='POINTWISE')
expected = statevector[0]*numpy.array(altitude_observations)\
+ statevector[1]*LatitudeFunction(numpy.cos, 2.0).compute(locations[:,0]).ravel()\
+ statevector[2]*LatitudeFunction(numpy.sin, 2.0).compute(locations[:,0]).ravel()\
+ statevector[3]*LatitudeFunction(numpy.cos, 4.0).compute(locations[:,0]).ravel()\
+ statevector[4]*LatitudeFunction(numpy.sin, 2.0).compute(locations[:,0]).ravel()
numpy.testing.assert_almost_equal(expected, result)
# test output gridding, pointwise limit
outputstructure = OutputRectilinearGridStructure(
2,
epoch_plus_days(2),
latitudes=numpy.linspace(-60., 60., num=5),
longitudes=numpy.linspace(-90., 90, num=10))
pointwise_result = analysis_system.evaluate_expected_value(
'MAP', outputstructure, 'POINTWISE')
pointwise_limit_result = analysis_system.evaluate_expected_value(
'MAP', outputstructure, 'GRID_CELL_AREA_AVERAGE', [1, 1], 10)
numpy.testing.assert_array_almost_equal(pointwise_result,
pointwise_limit_result)