def test_mini_world_local(self):
# Local component
local_component = SpatialComponent(
ComponentStorage_InMemory(
LocalElement(n_triangulation_divisions=1),
LocalHyperparameters(log_sigma=0.0, log_rho=numpy.log(1.0))),
SpatialComponentSolutionStorage_InMemory(),
compute_uncertainties=True,
method='APPROXIMATED')
# Analysis system using the specified components, for the Tmean observable
analysis_system = AnalysisSystem([local_component],
ObservationSource.TMEAN,
log=StringIO())
# Simulated inputs
simulated_input_loader = SimulatedInputLoader()
# Simulate evaluation of this time index
simulated_time_indices = [0]
# Update with data
analysis_system.update([simulated_input_loader],
simulated_time_indices)
# Check state vector directly
statevector = analysis_system.components[
0].solutionstorage.partial_state_read(0).ravel()
# These are the nodes where observations were put (see SimulatedObservationSource above)
# - check they correspond to within 3 times the stated noise level
self.assertAlmostEqual(20.0, statevector[12], delta=0.3)
self.assertAlmostEqual(-15.0, statevector[17], delta=0.3)
self.assertAlmostEqual(5.0, statevector[41], delta=0.3)
# Also check entire state vector within outer bounds set by obs
self.assertTrue(all(statevector < 20.0))
self.assertTrue(all(statevector > -15.0))
# And check output corresponds too
# (evaluate result on output structure same as input)
simulated_output_structure = SimulatedObservationStructure(0)
result = analysis_system.evaluate_expected_value(
'MAP', simulated_output_structure, flag='POINTWISE')
numpy.testing.assert_almost_equal(statevector[[12, 17, 41]], result)
# test output gridding, pointwise limit
outputstructure = OutputRectilinearGridStructure(
2,
epoch_plus_days(2),
latitudes=numpy.linspace(-89.875, 89.875, num=10),
longitudes=numpy.linspace(-179.875, 179.875, num=20))
pointwise_result = analysis_system.evaluate_expected_value(
'MAP', outputstructure, 'POINTWISE')
pointwise_limit_result = analysis_system.evaluate_expected_value(
'MAP', outputstructure, 'GRID_CELL_AREA_AVERAGE', [1, 1], 3)
numpy.testing.assert_array_almost_equal(pointwise_result,
pointwise_limit_result)
def test_epoch_plus_days(self):
result = epoch_plus_days(60573.647766204)
self.assertTrue(isinstance(result, datetime))
self.assertEqual(2015, result.year)
self.assertEqual(11, result.month)
self.assertEqual(5, result.day)
self.assertEqual(15, result.hour)
self.assertEqual(32, result.minute)
self.assertEqual(47, result.second)
def filename_mobile_locations(self, observable, daynumber):
"""Generate individual filename for locations."""
patterns = self.filename_patterns_mobile_locations(observable)
return os.path.join(*[
datetime_numeric.build_from_pattern(pattern,
epoch_plus_days(daynumber))
for pattern in patterns
])
def filename_observations(self, observable, daynumber):
"""Generate individual filename for data on specified day for the specified observable."""
patterns = self.filename_patterns_observations(observable)
return os.path.join(*[
datetime_numeric.build_from_pattern(pattern,
epoch_plus_days(daynumber))
for pattern in patterns
])
def filename_from_patterns(self, patterns, key):
if type(key) is int:
t = epoch.epoch_plus_days(key)
paths = [
datetime_numeric.build_from_pattern(pattern, t)
for pattern in patterns
]
name = os.path.join(*paths)
return name
def specify(manager, options):
"""Return the list of operations and expected output given the input catalogue."""
# Require many local variables here
# pylint: disable=too-many-locals
# parse input option list
parser = argparse.ArgumentParser()
parser.add_argument('--start', required=True)
parser.add_argument('--end', required=True)
args = parser.parse_args(options)
# process a whole number of days
startday = int(days_since_epoch(datetime_numeric.parse(args.start)))
endday = int(days_since_epoch(datetime_numeric.parse(args.end)))
# find items in catalogue grouped by day
lstinput = manager.references_groupbyday(INPUTNAME, subsetindex=0)
auxinput = manager.references_groupbyday(INPUTNAME, subsetindex=1)
# the main output dataset
dataset = manager.newdataset()
# list of subsets (one for each command pattern)
subsets = [dataset.newsubset([spec.outputpattern]) for spec in SUBSETSPECS]
# iterate over subsets
for subsetindex, subset in enumerate(subsets):
# process each day
for dayindex in range(startday, endday + 1):
# convert to datetime object for formatting etc
day = epoch_plus_days(dayindex)
# loop over LST files for this day
inputs = []
for lstreference in lstinput[dayindex]:
# find AUX file matching the LST file
lsttime = manager.match(lstreference).time
print 'day {0} time {1}'.format(dayindex, lsttime)
auxreference = next(
reference for reference in auxinput[dayindex]
if manager.match(reference).time == lsttime)
# append pair
inputs.extend([lstreference, auxreference])
# build output filename
outputs = [subset.newfiletime(day)]
# Append this operation to make it
dataset.newoperation(inputs, outputs, SUBSETSPECS[subsetindex])
def update_time_index(self, time_index, keep_design=False):
"""Update time time information for the projector
Set keep_design to True to allow reuse of a projection for a different time_index,
e.g. for a EUSTACE analysis local component for a different time step.
"""
self.time_index = time_index
self.corresponding_datetime = epoch_plus_days(time_index)
if not keep_design:
self.design_matrix = None
print "New time index:", self.time_index
def __init__(self, centre_latitudes, centre_longitudes, grid_resolution,
time_index, cell_sampling, blocking):
self.outputstructure = None # TODO Allow for an output struture with no spatial location - pure covariate.
self.grid_resolution = grid_resolution
self.centre_latitudes = centre_latitudes
self.centre_longitudes = centre_longitudes
self.time_index = time_index
self.corresponding_datetime = epoch_plus_days(time_index)
self.cell_sampling = cell_sampling
self.blocking = blocking
self.component = None
self.state_solution = None
self.state_samples = None
self.prior_samples = None
self.design_matrix = None
def build_pathname(self, basepath, daynumber):
""" Build whole pathname from basepath and daynumber. """
# Get pattern
patterns = self.build_pathname_patterns()
# Convert daynumber to datetime object
fieldtime = epoch_plus_days(daynumber)
# initialise pathname to basepath
pathname = basepath
# Fill date fields and append
for pattern in patterns:
pathname = os.path.join(
pathname,
datetime_numeric.build_from_pattern(pattern, fieldtime))
return pathname
def datetime_at_time_index(self, time_index):
"""Express a time index as days since epoch as a datetime"""
print time_index
return epoch_plus_days(time_index)
def datetime_at_time_index(self, time_index):
"""Use EUSTACE epoch."""
return epoch_plus_days(time_index)
def time_datetime(self):
return epoch_plus_days(self.t)
def test_mini_world_altitude_with_latitude(self):
"""Testing using altitude as a covariate"""
# GENERATING OBSERVATIONS
# Simulated locations: they will exactly sits on the grid points of the covariate datafile
DEM = Dataset(self.altitude_datafile)
latitude = DEM.variables['lat'][:]
longitude = DEM.variables['lon'][:]
altitude = DEM.variables['dem'][:]
indices = numpy.stack(
(numpy.array([1, 3, 5, 7, 8, 9, 10, 11
]), numpy.array([0, 0, 0, 0, 0, 0, 0, 0])),
axis=1)
selected_location = []
altitude_observations = []
for couple in indices:
selected_location.append([
latitude[couple[0], couple[1]], longitude[couple[0], couple[1]]
])
altitude_observations.append(altitude[couple[0], couple[1]])
DEM.close()
locations = numpy.array(selected_location)
# Simulated model is y = z + a*cos(2x) + c*cos(4*x) + b*sin(2x) + d*sin(4*x), with z = altitude, x = latitude, a=b=c=d=0
slope = 1e-3
measurement = slope * numpy.array(altitude_observations)
# Simulated errors
uncorrelatederror = 0.1 * numpy.ones(measurement.shape)
# Simulated inputs
simulated_input_loader = SimulatedInputLoader(locations, measurement,
uncorrelatederror)
# Simulate evaluation of this time index
simulated_time_indices = [0]
# GENERATING THE MODEL
# Local component
geography_covariate_element = GeographyBasedElement(
self.altitude_datafile, 'lat', 'lon', 'dem', 1.0)
geography_covariate_element.load()
combined_element = CombinationElement(
[geography_covariate_element,
LatitudeHarmonicsElement()])
combined_hyperparamters = CombinationHyperparameters([
CovariateHyperparameters(-0.5 * numpy.log(10.)),
CombinationHyperparameters([
CovariateHyperparameters(-0.5 * numpy.log(p))
for p in [10.0, 10.0, 10.0, 10.0]
])
])
combined_component = SpatialComponent(
ComponentStorage_InMemory(combined_element,
combined_hyperparamters),
SpatialComponentSolutionStorage_InMemory())
# GENERATING THE ANALYSIS
# Analysis system using the specified components, for the Tmean observable
analysis_system = AnalysisSystem([combined_component],
ObservationSource.TMEAN,
log=StringIO())
# Update with data
analysis_system.update([simulated_input_loader],
simulated_time_indices)
# Check state vector directly
statevector = analysis_system.components[
0].solutionstorage.partial_state_read(0).ravel()
# These are the nodes where observations were put (see SimulatedObservationSource above)
# - check they correspond to within 3 times the stated noise level
self.assertAlmostEqual(slope, statevector[0], delta=0.3)
self.assertAlmostEqual(0., statevector[1], delta=0.3)
self.assertAlmostEqual(0., statevector[2], delta=0.3)
self.assertAlmostEqual(0., statevector[3], delta=0.3)
self.assertAlmostEqual(0., statevector[4], delta=0.3)
# And check output corresponds too
# (evaluate result on output structure same as input)
simulated_output_structure = SimulatedObservationStructure(
0, locations, None, None)
result = analysis_system.evaluate_expected_value(
'MAP', simulated_output_structure, flag='POINTWISE')
expected = statevector[0]*numpy.array(altitude_observations)\
+ statevector[1]*LatitudeFunction(numpy.cos, 2.0).compute(locations[:,0]).ravel()\
+ statevector[2]*LatitudeFunction(numpy.sin, 2.0).compute(locations[:,0]).ravel()\
+ statevector[3]*LatitudeFunction(numpy.cos, 4.0).compute(locations[:,0]).ravel()\
+ statevector[4]*LatitudeFunction(numpy.sin, 2.0).compute(locations[:,0]).ravel()
numpy.testing.assert_almost_equal(expected, result)
# test output gridding, pointwise limit
outputstructure = OutputRectilinearGridStructure(
2,
epoch_plus_days(2),
latitudes=numpy.linspace(-60., 60., num=5),
longitudes=numpy.linspace(-90., 90, num=10))
pointwise_result = analysis_system.evaluate_expected_value(
'MAP', outputstructure, 'POINTWISE')
pointwise_limit_result = analysis_system.evaluate_expected_value(
'MAP', outputstructure, 'GRID_CELL_AREA_AVERAGE', [1, 1], 10)
numpy.testing.assert_array_almost_equal(pointwise_result,
pointwise_limit_result)
def datetime_at_time_index(self, time_index):
return epoch_plus_days(time_index)
def test_mini_world_noiseless(self):
number_of_simulated_time_steps = 1
# Build system
element = SeasonalElement(n_triangulation_divisions=3,
n_harmonics=5,
include_local_mean=True)
hyperparameters = SeasonalHyperparameters(n_spatial_components=6,
common_log_sigma=0.0,
common_log_rho=0.0)
component = SpaceTimeComponent(
ComponentStorage_InMemory(element, hyperparameters),
SpaceTimeComponentSolutionStorage_InMemory())
analysis_system = AnalysisSystem([component],
ObservationSource.TMEAN,
log=StringIO())
# use fixed locations from icosahedron
fixed_locations = cartesian_to_polar2d(
MeshIcosahedronSubdivision.build(3).points)
# random measurement at each location
numpy.random.seed(8976)
field_basis = numpy.random.randn(fixed_locations.shape[0])
#print(field_basis.shape)
#time_basis = numpy.array(harmonics_list)
# some time function that varies over a year
#decimal_years = numpy.array([datetime_to_decimal_year(epoch_plus_days(step)) for step in range(number_of_simulated_time_steps)])
time_basis = numpy.cos(
numpy.linspace(0.1, 1.75 * numpy.pi,
number_of_simulated_time_steps))
# kronecker product of the two
#print(numpy.expand_dims(time_basis, 1))
measurement = numpy.kron(field_basis, numpy.expand_dims(
time_basis, 1)) #numpy.expand_dims(time_basis, 1))
#print(measurement.shape)
# Simulated inputs
simulated_input_loader = SimulatedInputLoader(fixed_locations,
measurement, 0.0001)
# Simulate evaluation of this time index
simulated_time_indices = range(number_of_simulated_time_steps)
# Iterate
for iteration in range(5):
analysis_system.update([simulated_input_loader],
simulated_time_indices)
# Get all results
result = numpy.zeros(measurement.shape)
for t in range(number_of_simulated_time_steps):
result[t, :] = analysis_system.evaluate_expected_value(
'MAP',
SimulatedObservationStructure(t, fixed_locations, None, None),
flag='POINTWISE')
# Should be very close to original because specified noise is low
numpy.testing.assert_almost_equal(result, measurement)
max_disparity = (numpy.abs(result - measurement)).ravel().max()
self.assertTrue(max_disparity < 1E-5)
# test output gridding, pointwise limit
outputstructure = OutputRectilinearGridStructure(
2,
epoch_plus_days(2),
latitudes=numpy.linspace(-60., 60., num=5),
longitudes=numpy.linspace(-90., 90, num=10))
pointwise_result = analysis_system.evaluate_expected_value(
'MAP', outputstructure, 'POINTWISE')
pointwise_limit_result = analysis_system.evaluate_expected_value(
'MAP', outputstructure, 'GRID_CELL_AREA_AVERAGE', [1, 1], 10)
numpy.testing.assert_array_almost_equal(pointwise_result,
pointwise_limit_result)
