def test_mini_world_geography_based_mock_data(self): """Testing on a simple mock data file, with mock covariate values""" # GENERATING OBSERVATIONS # Simulated locations: they will exactly sits on the grid points of the covariate datafile locations = numpy.array([[0.0, 0.0], [0.0, 0.5], [0.05, 0.0]]) # Simulated measurements: simple linear relation of type: y = 2*x measurement = numpy.array([2., 2., 2.]) # Simulated errors uncorrelatederror = 0.1 * numpy.ones(measurement.shape) # Simulated inputs simulated_input_loader = SimulatedInputLoader(locations, measurement, uncorrelatederror) # Simulate evaluation of this time index simulated_time_indices = [0] # GENERATING THE MODEL # Local component geography_covariate_element = GeographyBasedElement( self.covariate_file.name, 'lat', 'lon', 'covariate', 1.0) geography_covariate_element.load() geography_based_component = SpatialComponent( ComponentStorage_InMemory( geography_covariate_element, CovariateHyperparameters(-0.5 * numpy.log(10.))), SpatialComponentSolutionStorage_InMemory()) # GENERATING THE ANALYSIS # Analysis system using the specified components, for the Tmean observable analysis_system = AnalysisSystem([geography_based_component], ObservationSource.TMEAN, log=StringIO()) # Update with data analysis_system.update([simulated_input_loader], simulated_time_indices) # Check state vector directly statevector = analysis_system.components[ 0].solutionstorage.partial_state_read(0).ravel() # These are the nodes where observations were put (see SimulatedObservationSource above) # - check they correspond to within 3 times the stated noise level self.assertAlmostEqual(2., statevector[0], delta=0.3) # Also check entire state vector within outer bounds set by obs self.assertTrue(all(statevector < 2.0)) # And check output corresponds too # (evaluate result on output structure same as input) simulated_output_structure = SimulatedObservationStructure( 0, locations, None, None) result = analysis_system.evaluate_expected_value( 'MAP', simulated_output_structure, flag='POINTWISE') numpy.testing.assert_almost_equal( statevector[0] * numpy.ones(len(measurement)), result)
def test_mini_world_local(self): # Local component local_component = SpatialComponent( ComponentStorage_InMemory( LocalElement(n_triangulation_divisions=1), LocalHyperparameters(log_sigma=0.0, log_rho=numpy.log(1.0))), SpatialComponentSolutionStorage_InMemory(), compute_uncertainties=True, method='APPROXIMATED') # Analysis system using the specified components, for the Tmean observable analysis_system = AnalysisSystem([local_component], ObservationSource.TMEAN, log=StringIO()) # Simulated inputs simulated_input_loader = SimulatedInputLoader() # Simulate evaluation of this time index simulated_time_indices = [0] # Update with data analysis_system.update([simulated_input_loader], simulated_time_indices) # Check state vector directly statevector = analysis_system.components[ 0].solutionstorage.partial_state_read(0).ravel() # These are the nodes where observations were put (see SimulatedObservationSource above) # - check they correspond to within 3 times the stated noise level self.assertAlmostEqual(20.0, statevector[12], delta=0.3) self.assertAlmostEqual(-15.0, statevector[17], delta=0.3) self.assertAlmostEqual(5.0, statevector[41], delta=0.3) # Also check entire state vector within outer bounds set by obs self.assertTrue(all(statevector < 20.0)) self.assertTrue(all(statevector > -15.0)) # And check output corresponds too # (evaluate result on output structure same as input) simulated_output_structure = SimulatedObservationStructure(0) result = analysis_system.evaluate_expected_value( 'MAP', simulated_output_structure, flag='POINTWISE') numpy.testing.assert_almost_equal(statevector[[12, 17, 41]], result) # test output gridding, pointwise limit outputstructure = OutputRectilinearGridStructure( 2, epoch_plus_days(2), latitudes=numpy.linspace(-89.875, 89.875, num=10), longitudes=numpy.linspace(-179.875, 179.875, num=20)) pointwise_result = analysis_system.evaluate_expected_value( 'MAP', outputstructure, 'POINTWISE') pointwise_limit_result = analysis_system.evaluate_expected_value( 'MAP', outputstructure, 'GRID_CELL_AREA_AVERAGE', [1, 1], 3) numpy.testing.assert_array_almost_equal(pointwise_result, pointwise_limit_result)
def atest_process_observations_compute_uncertainties(self): # Our test system is: # # ( [ 2.0 0.0 ] + [ -1.5 0.0 ] [ 5.0 0.0 ] [ -1.5 2.2 ] ) x = [ -1.5 0.0 ] [ 10.0 3.0 ] [ 7.0 - 2.0 ] # ( [ 0.0 2.0 ] [ 2.2 3.3 ] [ 0.0 5.0 ] [ 0.0 3.3 ] ) [ 2.2 3.3 ] [ 3.0 10.0 ] [ 9.0 - 3.0 ] # # [ 3.25 -16.5 ] x = [ -37.5 ] # [-16.5 80.65 ] [ 154.0 ] # # => x = [ -0.60697861 ] # [ 1.78530506 ] # c = DelayedSpatialComponent( ComponentStorage_InMemory( TestDelayedSpatialComponentSolution.TestElement(), CovariateHyperparameters(-0.5 * numpy.log(2.0))), DelayedSpatialComponentSolutionStorage_Files(), compute_uncertainties=True) s = c.component_solution() self.assertIsInstance(s, DelayedSpatialComponentSolution) self.assertTrue(s.compute_uncertainties) test_offset = numpy.array([2.0, 3.0]) s.process_observations( TestDelayedSpatialComponentSolution.TestObservations(t=21), test_offset[0:1]) s.update_time_step() s.process_observations( TestDelayedSpatialComponentSolution.TestObservations(t=532), test_offset[1:2]) s.update_time_step() s.update() # In this case we are considering the last iteration of model solving, hence marginal variances should have been stored expected_marginal_std = numpy.sqrt( numpy.diag( numpy.linalg.inv(numpy.array([[13.25, -16.5], [-16.5, 80.65]])))) numpy.testing.assert_array_almost_equal( s.solutionstorage.state_marginal_std, expected_marginal_std) # Now we compute the projection of marginal variances onto the given observations for time in [532]: # Observation at time t=t* should be design matrix for that time multiplied by expected state expected_projection = TestDelayedSpatialComponentSolution.TestDesign( t=time).design_function(expected_marginal_std) numpy.testing.assert_almost_equal( s.solution_observation_expected_uncertainties( TestDelayedSpatialComponentSolution.TestObservations( t=time)), expected_projection)
def test_process_observations_compute_sample(self): # Our test system is: # # ( [ 2.0 0.0 ] + [ -1.5 0.0 ] [ 5.0 0.0 ] [ -1.5 2.2 ] ) x = [ -1.5 0.0 ] [ 10.0 3.0 ] [ 7.0 - 2.0 ] # ( [ 0.0 2.0 ] [ 2.2 3.3 ] [ 0.0 5.0 ] [ 0.0 3.3 ] ) [ 2.2 3.3 ] [ 3.0 10.0 ] [ 9.0 - 3.0 ] # # [ 13.25 -16.5 ] x = [ -37.5 ] # [-16.5 80.65 ] [ 154.0 ] # # => x = [ -0.60697861 ] # [ 1.78530506 ] # number_of_samples = 200 c = SpaceTimeComponent(ComponentStorage_InMemory( TestSpaceTimeComponentSolution.TestElement(), CovariateHyperparameters(-0.5 * numpy.log(2.0))), SpaceTimeComponentSolutionStorage_InMemory(), compute_sample=True, sample_size=number_of_samples) s = c.component_solution() self.assertIsInstance(s, SpaceTimeComponentSolution) self.assertTrue(s.compute_sample) test_offset = numpy.array([2.0, 3.0]) s.process_observations( TestSpaceTimeComponentSolution.TestObservations(t=21), test_offset[0:1]) s.update_time_step() s.process_observations( TestSpaceTimeComponentSolution.TestObservations(t=532), test_offset[1:2]) s.update_time_step() numpy.random.seed(0) s.update() # In this case we are considering the last iteration of model solving, hence sample should have been stored computed_sample = s.solutionstorage.state_sample self.assertEqual((2, number_of_samples), computed_sample.shape) numpy.random.seed(0) variate = scipy.random.normal(0.0, 1.0, (2, number_of_samples)) expected_posterior_precision = numpy.array([[13.25, -16.5], [-16.5, 80.65]]) # check result numpy.testing.assert_almost_equal( numpy.dot(variate.T, variate), numpy.dot(computed_sample.T, numpy.dot(expected_posterior_precision, computed_sample)))
def mini_world_spatial(): # Local component local_component = SpatialComponent(ComponentStorage_InMemory(LocalElement(n_triangulation_divisions=4), LocalHyperparameters(log_sigma=numpy.log(5.0), log_rho=numpy.log( 15. * numpy.pi / 180. ) )), SpatialComponentSolutionStorage_InMemory(), compute_uncertainties=False, method='APPROXIMATED', compute_sample = True, sample_size=100) # Analysis system using the specified components, for the Tmean observable analysis_system = AnalysisSystem([ local_component ], ObservationSource.TMEAN, log=StringIO()) # Simulated inputs simulated_input_loader = SimulatedInputLoader() # Simulate evaluation of this time index simulated_time_indices = [ 0 ] # Update with data analysis_system.update([ simulated_input_loader ], simulated_time_indices) # Check state vector directly statevector = analysis_system.components[0].solutionstorage.partial_state_read(0) return analysis_system, statevector
def test_mini_world_latitude_harmonics(self): """Testing on a simple mock data file using latitude harmonics""" # GENERATING OBSERVATIONS # Simulated locations: they will exactly sits on the grid points of the covariate datafile locations = numpy.array([[0.0, 0.0], [0.25, 0.5], [0.5, 0.0]]) # Simulated model is y = a*cos(2x) + c*cos(4*x) + b*sin(2x) + d*sin(4*x) with x = latitude, so we expect a=c=1, c=d=0 measurement = LatitudeFunction(numpy.cos, 2.0).compute( locations[:, 0]).ravel() + LatitudeFunction( numpy.cos, 4.0).compute(locations[:, 0]).ravel() # Simulated errors uncorrelatederror = 0.1 * numpy.ones(measurement.shape) # Simulated inputs simulated_input_loader = SimulatedInputLoader(locations, measurement, uncorrelatederror) # Simulate evaluation of this time index simulated_time_indices = [0] latitude_harmonics_component = SpatialComponent( ComponentStorage_InMemory( LatitudeHarmonicsElement(), CombinationHyperparameters([ CovariateHyperparameters(-0.5 * numpy.log(p)) for p in [10.0, 10.0, 10.0, 10.0] ])), SpatialComponentSolutionStorage_InMemory()) # Analysis system using the specified components, for the Tmean observable analysis_system = AnalysisSystem([latitude_harmonics_component], ObservationSource.TMEAN, log=StringIO()) # GENERATING THE ANALYSIS # Update with data analysis_system.update([simulated_input_loader], simulated_time_indices) # Check state vector directly statevector = analysis_system.components[ 0].solutionstorage.partial_state_read(0).ravel() # These are the nodes where observations were put (see SimulatedObservationSource above) # - check they correspond to within 3 times the stated noise level self.assertAlmostEqual(1., statevector[0], delta=0.3) self.assertAlmostEqual(1., statevector[2], delta=0.3) self.assertAlmostEqual(0., statevector[1], delta=0.3) self.assertAlmostEqual(0., statevector[3], delta=0.3) # Also check entire state vector within outer bounds set by obs self.assertTrue(all(statevector < 1.0)) # And check output corresponds too # (evaluate result on output structure same as input) simulated_output_structure = SimulatedObservationStructure( 0, locations, None, None) result = analysis_system.evaluate_expected_value( 'MAP', simulated_output_structure, flag='POINTWISE') expected = statevector[0]*LatitudeFunction(numpy.cos, 2.0).compute(locations[:,0]).ravel() + statevector[1]*LatitudeFunction(numpy.sin, 2.0).compute(locations[:,0]).ravel()\ + statevector[2] *LatitudeFunction(numpy.cos, 4.0).compute(locations[:,0]).ravel()+ statevector[3]*LatitudeFunction(numpy.sin, 4.0).compute(locations[:,0]).ravel() numpy.testing.assert_almost_equal(expected, result)
def test_mini_world_large_and_local(self): # Use a number of time steps number_of_simulated_time_steps = 30 # Large-scale spatial variability simulated_large_variation = 10.0 # Local variability simulated_local_variation = 1.0 # Iterations to use number_of_solution_iterations = 5 # Build system # Large-scale factor element_large = SpaceTimeKroneckerElement( n_triangulation_divisions=1, alpha=2, starttime=0, endtime=number_of_simulated_time_steps + 1, n_nodes=number_of_simulated_time_steps + 2, overlap_factor=2.5, H=1) initial_hyperparameters_large = SpaceTimeSPDEHyperparameters( space_log_sigma=0.0, space_log_rho=numpy.log(numpy.radians(5.0)), time_log_rho=numpy.log(1.0 / 365.0)) component_large = SpaceTimeComponent( ComponentStorage_InMemory(element_large, initial_hyperparameters_large), SpaceTimeComponentSolutionStorage_InMemory()) # And a local process component_local = SpatialComponent( ComponentStorage_InMemory( LocalElement(n_triangulation_divisions=3), LocalHyperparameters(log_sigma=0.0, log_rho=numpy.log(numpy.radians(5.0)))), SpatialComponentSolutionStorage_InMemory()) analysis_system = AnalysisSystem([component_large, component_local], ObservationSource.TMEAN, log=StringIO()) # analysis_system = AnalysisSystem([ component_large ], ObservationSource.TMEAN) # analysis_system = AnalysisSystem([ component_local ], ObservationSource.TMEAN) # use fixed locations from icosahedron fixed_locations = cartesian_to_polar2d( MeshIcosahedronSubdivision.build(3).points) # random measurement at each location numpy.random.seed(8976) field_basis = simulated_large_variation * numpy.random.randn( fixed_locations.shape[0]) # some time function that varies over a year time_basis = numpy.cos( numpy.linspace(0.1, 1.75 * numpy.pi, number_of_simulated_time_steps)) # kronecker product of the two large_scale_process = numpy.kron(field_basis, numpy.expand_dims(time_basis, 1)) # Random local changes where mean change at each time is zero # local_process = simulated_local_variation * numpy.random.randn(large_scale_process.shape[0], large_scale_process.shape[1]) # local_process -= numpy.tile(local_process.mean(axis=1), (local_process.shape[1], 1)).T local_process = numpy.zeros(large_scale_process.shape) somefield = simulated_local_variation * numpy.random.randn( 1, large_scale_process.shape[1]) somefield -= somefield.ravel().mean() local_process[10, :] = somefield local_process[11, :] = -somefield # Add the two processes measurement = large_scale_process + local_process # Simulated inputs simulated_input_loader = SimulatedInputLoader(fixed_locations, measurement, 0.001) # Simulate evaluation of this time index simulated_time_indices = range(number_of_simulated_time_steps) # All systems linear so single update should be ok analysis_system.update([simulated_input_loader], simulated_time_indices) # Get all results result = numpy.zeros(measurement.shape) for t in range(number_of_simulated_time_steps): result[t, :] = analysis_system.evaluate_expected_value( 'MAP', SimulatedObservationStructure(t, fixed_locations, None, None), flag='POINTWISE') disparity_large_scale = (numpy.abs(result - large_scale_process)).ravel().max() # print 'large scale disparity: ', disparity_large_scale disparity_overall = (numpy.abs(result - measurement)).ravel().max() # print 'overall disparity: ', disparity_overall numpy.testing.assert_almost_equal(result, measurement, decimal=4) self.assertTrue(disparity_overall < 1E-4)
def test_process_observations_compute_sample(self): # Our test system is: # # ( [ 2.0 0.0 ] + [ -1.5 0.0 ] [ 10.0 3.0 ] [ -1.5 2.2 ] ) x = [ -1.5 0.0 ] [ 10.0 3.0 ] [ 7.0 - 2.0 ] # ( [ 0.0 2.0 ] [ 2.2 3.3 ] [ 3.0 10.0 ] [ 0.0 3.3 ] ) [ 2.2 3.3 ] [ 3.0 10.0 ] [ 9.0 - 3.0 ] # # [ 24.5 -47.85 ] x = [ -102.0 ] # [ -47.85 202.86 ] [ 397.1 ] # # => x = [ -0.63067268 ] # [ 1.80874649 ] # # Input data test_offset = numpy.array([2.0, 3.0]) test_obs = TestSpatialComponentSolution.TestObservations() sample_size = 300 # Make component and check it's of the correct class c = SpatialComponent(ComponentStorage_InMemory( TestSpatialComponentSolution.TestElement(), CovariateHyperparameters(-0.5 * numpy.log(2.0))), SpatialComponentSolutionStorage_InMemory(), compute_sample=True, sample_size=sample_size) s = c.component_solution() self.assertIsInstance(s, SpatialComponentSolution) self.assertFalse(s.compute_uncertainties) self.assertTrue(s.compute_sample) # Do the processing s.process_observations(test_obs, test_offset) numpy.random.seed(0) s.update_time_step() computed_sample = s.solutionstorage.partial_state_sample_read(21) self.assertEqual((2, sample_size), computed_sample.shape) numpy.random.seed(0) variate = scipy.random.normal(0.0, 1.0, (2, sample_size)) expected_posterior_precision = numpy.array([[24.5, -47.85], [-47.85, 202.86]]) self.assertEqual(len(s.solutionstorage.state_sample_at_time), 1) self.assertListEqual(s.solutionstorage.state_sample_at_time.keys(), [21]) # check result numpy.testing.assert_almost_equal( numpy.dot(variate.T, variate), numpy.dot(computed_sample.T, numpy.dot(expected_posterior_precision, computed_sample))) # In this case we did not allowed the computation of uncertainties self.assertEqual(s.solutionstorage.partial_state_marginal_std_read(21), None) self.assertEqual(s.solutionstorage.state_marginal_std_at_time, {}) numpy.testing.assert_almost_equal( s.solution_observation_expected_uncertainties( TestSpatialComponentSolution.TestObservations()), 0.)
def main(): print 'Advanced standard example using a few days of EUSTACE data' parser = argparse.ArgumentParser(description='Advanced standard example using a few days of EUSTACE data') parser.add_argument('outpath', help='directory where the output should be redirected') parser.add_argument('--json_descriptor', default = None, help='a json descriptor containing the covariates to include in the climatology model') parser.add_argument('--land_biases', action='store_true', help='include insitu land homogenization bias terms') parser.add_argument('--global_biases', action='store_true', help='include global satellite bias terms') parser.add_argument('--n_iterations', type=int, default=5, help='number of solving iterations') args = parser.parse_args() # Input data path basepath = os.path.join('/work/scratch/eustace/rawbinary3') # Days to process time_indices = range(int(days_since_epoch(datetime(2006, 2, 1))), int(days_since_epoch(datetime(2006, 2, 2)))) # Sources to use sources = [ 'surfaceairmodel_land', 'surfaceairmodel_ocean', 'surfaceairmodel_ice', 'insitu_land', 'insitu_ocean' ] #SETUP # setup for the seasonal core: climatology covariates setup read from file seasonal_setup = {'n_triangulation_divisions':5, 'n_harmonics':4, 'n_spatial_components':6, 'amplitude':2., 'space_length_scale':5., # length scale in units of degrees } grandmean_amplitude = 15.0 # setup for the large scale component spacetime_setup = {'n_triangulation_divisions':2, 'alpha':2, 'starttime':0, 'endtime':10., 'n_nodes':2, 'overlap_factor':2.5, 'H':1, 'amplitude':1., 'space_lenght_scale':15.0, # length scale in units of degrees 'time_length_scale':15.0 # length scal in units of days } bias_amplitude = .9 # setup for the local component local_setup = {'n_triangulation_divisions':6, 'amplitude':2., 'space_length_scale':2. # length scale in units of degrees } globalbias_amplitude = 15.0 # CLIMATOLOGY COMPONENT: combining the seasonal core along with latitude harmonics, altitude and coastal effects if args.json_descriptor is not None: loader = LoadCovariateElement(args.json_descriptor) loader.check_keys() covariate_elements, covariate_hyperparameters = loader.load_covariates_and_hyperparameters() print('The following fields have been added as covariates of the climatology model') print(loader.data.keys()) else: covariate_elements, covariate_hyperparameters = [], [] climatology_element = CombinationElement( [SeasonalElement(n_triangulation_divisions=seasonal_setup['n_triangulation_divisions'], n_harmonics=seasonal_setup['n_harmonics'], include_local_mean=True), GrandMeanElement()]+covariate_elements) climatology_hyperparameters = CombinationHyperparameters( [SeasonalHyperparameters(n_spatial_components=seasonal_setup['n_spatial_components'], common_log_sigma=numpy.log(seasonal_setup['amplitude']), common_log_rho=numpy.log(numpy.radians(seasonal_setup['space_length_scale']))), CovariateHyperparameters(numpy.log(grandmean_amplitude))] + covariate_hyperparameters ) climatology_component = SpaceTimeComponent(ComponentStorage_InMemory(climatology_element, climatology_hyperparameters), SpaceTimeComponentSolutionStorage_InMemory(), compute_uncertainties=True, method='APPROXIMATED', compute_sample=True, sample_size=definitions.GLOBAL_SAMPLE_SHAPE[3]) # LARGE SCALE (kronecker product) COMPONENT: combining large scale trends with bias terms accounting for homogeneization effects if args.land_biases: bias_element, bias_hyperparameters = [InsituLandBiasElement(BREAKPOINTS_FILE)], [CovariateHyperparameters(numpy.log(bias_amplitude))] print('Adding bias terms for insitu land homogenization') else: bias_element, bias_hyperparameters = [], [] large_scale_element = CombinationElement( [SpaceTimeKroneckerElement(n_triangulation_divisions=spacetime_setup['n_triangulation_divisions'], alpha=spacetime_setup['alpha'], starttime=spacetime_setup['starttime'], endtime=spacetime_setup['endtime'], n_nodes=spacetime_setup['n_nodes'], overlap_factor=spacetime_setup['overlap_factor'], H=spacetime_setup['H'])] + bias_element) large_scale_hyperparameters = CombinationHyperparameters( [SpaceTimeSPDEHyperparameters(space_log_sigma=numpy.log(spacetime_setup['amplitude']), space_log_rho=numpy.log(numpy.radians(spacetime_setup['space_lenght_scale'])), time_log_rho=numpy.log(spacetime_setup['time_length_scale']))] + bias_hyperparameters) large_scale_component = SpaceTimeComponent(ComponentStorage_InMemory(large_scale_element, large_scale_hyperparameters), SpaceTimeComponentSolutionStorage_InMemory(), compute_uncertainties=True, method='APPROXIMATED', compute_sample=True, sample_size=definitions.GLOBAL_SAMPLE_SHAPE[3]) # LOCAL COMPONENT: combining local scale variations with global satellite bias terms if args.global_biases: bias_elements = [BiasElement(groupname, 1) for groupname in GLOBAL_BIASES_GROUP_LIST] bias_hyperparameters = [CovariateHyperparameters(numpy.log(globalbias_amplitude)) for index in range(len(GLOBAL_BIASES_GROUP_LIST))] print('Adding global bias terms for all the surfaces') else: bias_elements, bias_hyperparameters = [], [] local_scale_element = CombinationElement([LocalElement(n_triangulation_divisions=local_setup['n_triangulation_divisions'])] + bias_elements) local_scale_hyperparameters = CombinationHyperparameters([LocalHyperparameters(log_sigma=numpy.log(local_setup['amplitude']), log_rho=numpy.log(numpy.radians(local_setup['space_length_scale'])))] + bias_hyperparameters) local_component = SpatialComponent(ComponentStorage_InMemory(local_scale_element, local_scale_hyperparameters), SpatialComponentSolutionStorage_InMemory(), compute_uncertainties=True, method='APPROXIMATED', compute_sample=True, sample_size=definitions.GLOBAL_SAMPLE_SHAPE[3]) # Analysis system using the specified components, for the Tmean observable print 'Analysing inputs' analysis_system = AnalysisSystem( [ climatology_component, large_scale_component, local_component ], ObservationSource.TMEAN) # Object to load raw binary inputs at time indices inputloaders = [ AnalysisSystemInputLoaderRawBinary_Sources(basepath, source, time_indices) for source in sources ] for iteration in range(args.n_iterations): message = 'Iteration {}'.format(iteration) print(message) # Update with data analysis_system.update(inputloaders, time_indices) print 'Computing outputs' # Produce an output for each time index for time_index in time_indices: # Get date for output outputdate = inputloaders[0].datetime_at_time_index(time_index) print 'Evaluating output grid: ', outputdate #Configure output grid outputstructure = OutputRectilinearGridStructure( time_index, outputdate, latitudes=numpy.linspace(-90.+definitions.GLOBAL_FIELD_RESOLUTION/2., 90.-definitions.GLOBAL_FIELD_RESOLUTION/2., num=definitions.GLOBAL_FIELD_SHAPE[1]), longitudes=numpy.linspace(-180.+definitions.GLOBAL_FIELD_RESOLUTION/2., 180.-definitions.GLOBAL_FIELD_RESOLUTION/2., num=definitions.GLOBAL_FIELD_SHAPE[2])) # Evaluate expected value at these locations for field in ['MAP', 'post_STD']: print 'Evaluating: ',field result_expected_value = analysis_system.evaluate_expected_value('MAP', outputstructure, 'GRID_CELL_AREA_AVERAGE', [1,1], 1000) result_expected_uncertainties = analysis_system.evaluate_expected_value('post_STD', outputstructure, 'GRID_CELL_AREA_AVERAGE', [1,1], 1000) print 'Evaluating: climatology fraction' climatology_fraction = analysis_system.evaluate_climatology_fraction(outputstructure, [1,1], 1000) print 'Evaluating: the sample' sample = analysis_system.evaluate_projected_sample(outputstructure) # Make output filename pathname = 'eustace_example_output_{0:04d}{1:02d}{2:02d}.nc'.format(outputdate.year, outputdate.month, outputdate.day) pathname = os.path.join(args.outpath, pathname) print 'Saving: ', pathname # Save results filebuilder = FileBuilderGlobalField( pathname, time_index, 'Infilling Example', 'UNVERSIONED', definitions.TAS.name, '', 'Example data only', 'eustace.analysis.advanced_standard.examples.example_eustace_few_days', '') filebuilder.add_global_field(definitions.TAS, result_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.add_global_field(definitions.TASUNCERTAINTY, result_expected_uncertainties.reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.add_global_field(definitions.TAS_CLIMATOLOGY_FRACTION, climatology_fraction.reshape(definitions.GLOBAL_FIELD_SHAPE)) for index in range(definitions.GLOBAL_SAMPLE_SHAPE[3]): variable = copy.deepcopy(definitions.TASENSEMBLE) variable.name = variable.name + '_' + str(index) selected_sample = sample[:,index].ravel()+result_expected_value filebuilder.add_global_field(variable, selected_sample.reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.save_and_close() print 'Complete'
def main(): print 'EUSTACE example using HadCRUT4 monthly data' # Input data path input_basepath = os.path.join(WORKSPACE_PATH, 'data/incoming/HadCRUT4.5.0.0') # Input filenames input_filenames = [ 'hadcrut4_median_netcdf.nc', 'hadcrut4_uncorrelated_supplementary.nc', 'hadcrut4_blended_uncorrelated.nc' ] # Months to process time_indices = range(2) # Climatology component climatology_component = SpaceTimeComponent(ComponentStorage_InMemory(SeasonalElement(n_triangulation_divisions=5, n_harmonics=5, include_local_mean=True), SeasonalHyperparameters(n_spatial_components=6, common_log_sigma=1.0, common_log_rho=0.0)), SpaceTimeComponentSolutionStorage_InMemory()) # Number of factors for large scale (factor analysis) component and initial hyperparameters n_factors = 5 factors = [ ] factor_hyperparameters = [ ] for factor_index in range(n_factors): factor_hyperparameters.append( SpaceTimeSPDEHyperparameters( space_log_sigma=0.0, space_log_rho=numpy.log(10.0 * numpy.pi/180 + 25.0 * numpy.pi/180 *(n_factors - factor_index) / n_factors), time_log_rho=numpy.log(1/12.0 + 6/12.0*(n_factors - factor_index) / n_factors)) ) factors.append( SpaceTimeFactorElement(n_triangulation_divisions=5, alpha=2, starttime=0, endtime=36, overlap_factor=2.5, H=1) ) # Large scale (factor analysis) component large_scale_component = SpaceTimeComponent(ComponentStorage_InMemory(CombinationElement(factors), CombinationHyperparameters(factor_hyperparameters)), SpaceTimeComponentSolutionStorage_InMemory()) # Local component local_component = SpatialComponent(ComponentStorage_InMemory(LocalElement(n_triangulation_divisions=4), LocalHyperparameters(log_sigma=0.0, log_rho=numpy.log(10.0 * numpy.pi/180))), SpatialComponentSolutionStorage_InMemory()) print 'Analysing inputs' # Analysis system using the specified components, for the Tmean observable analysis_system = AnalysisSystem( [ climatology_component, large_scale_component, local_component ], ObservationSource.TMEAN) # Make filelist input_filelist = [ os.path.join(input_basepath, filename) for filename in input_filenames ] # Object to load HadCRUT4 inputs at time indices inputloader = AnalysisSystemInputLoaderHadCRUT4(input_filelist) # Update with data analysis_system.update([ inputloader ], time_indices) print 'Computing outputs' # Produce an output for each time index for time_index in time_indices: # Make output filename outputdate = inputloader.datetime_at_time_index(time_index) pathname = 'example_output_{0:04d}{1:02d}.nc'.format(outputdate.year, outputdate.month) print 'Saving: ', pathname # Configure output grid outputstructure = OutputRectilinearGridStructure( time_index, outputdate, latitudes=numpy.linspace(-87.5, 87.5, num=36), longitudes=numpy.linspace(-177.5, 177.5, num=72)) # Evaluate expected value at these locations result_expected_value = analysis_system.evaluate_expected_value(outputstructure) # Save results filebuilder = FileBuilderHadCRUT4ExampleOutput(pathname, outputstructure) filebuilder.add_global_field(TAS_ANOMALY, result_expected_value.reshape(1,36,72)) filebuilder.save_and_close() print 'Complete'
def test_component_storage_in_memory(self): storage = ComponentStorage_InMemory('A', 'B') self.assertEqual('A', storage.element_read()) self.assertEqual('B', storage.hyperparameters_read())
def demo_non_stationary(): full_resolution_level = 5 neighbourhood_level = 2 full_spde = SphereMeshViewGlobal(level=full_resolution_level) active_triangles = full_spde.neighbours_at_level(neighbourhood_level, 0) n_regions = full_spde.n_triangles_at_level(neighbourhood_level) merge_method = 'new' if merge_method == 'old': local_spdes = [] local_hyperparameters = [] for region_index in range(n_regions): local_spdes.append( SphereMeshViewSuperTriangle(full_resolution_level, neighbourhood_level, region_index)) hyperparameters = numpy.array( [numpy.float64(region_index), numpy.float64(region_index)]) hyperparameters = numpy.log( numpy.concatenate([ numpy.random.uniform(1.0, 3.0, 1), numpy.random.uniform(5.0, 30.0, 1) * numpy.pi / 180. ])) #hyperparameters = numpy.log( numpy.concatenate( [numpy.ones(1), numpy.random.uniform(15.0,45.0, 1) *numpy.pi/180.] ) ) #hyperparameters = numpy.array([2.0, 3.0]) #hyperparameters = numpy.log([2.0, numpy.pi/4]) local_hyperparameters.append(hyperparameters) global_hyperparameters, global_sigma_design, global_rho_design = full_spde.merge_local_parameterisations( local_spdes, local_hyperparameters, merge_method='exp_average') log_sigmas = global_sigma_design.dot(global_hyperparameters) log_rhos = global_rho_design.dot(global_hyperparameters) elif merge_method == 'new': sigma_accumulator = None rho_accumulator = None contribution_counter = None for region_index in range(n_regions): local_spde = SphereMeshViewSuperTriangle(full_resolution_level, neighbourhood_level, region_index) local_hyperparameters = hyperparameters = numpy.log( numpy.concatenate([ numpy.random.uniform(1.0, 5.0, 1), numpy.random.uniform(10.0, 45.0, 1) * numpy.pi / 180. ])) accumulators = SphereMeshViewGlobal.accumulate_local_parameterisations( sigma_accumulator, rho_accumulator, contribution_counter, local_spde, local_hyperparameters) sigma_accumulator, rho_accumulator, contribution_counter = accumulators log_sigmas, log_rhos = SphereMeshViewGlobal.finalise_local_parameterisation_sigma_rho( sigma_accumulator, rho_accumulator, contribution_counter) #print global_hyperparameters, global_sigma_design, global_rho_design import matplotlib.pyplot as plt from eustace.analysis.mesh.geometry import cartesian_to_polar2d polar_coords = cartesian_to_polar2d(full_spde.triangulation.points) plt.figure() plt.scatter(polar_coords[:, 1], polar_coords[:, 0], c=255. * log_sigmas / numpy.max(numpy.abs(log_sigmas)), linewidth=0.0, s=8.0) plt.figure() plt.scatter(polar_coords[:, 1], polar_coords[:, 0], c=255. * log_rhos / numpy.max(numpy.abs(log_rhos)), linewidth=0.0, s=8.0) #plt.show() #numpy.testing.assert_almost_equal( log_sigmas, 2.0 * numpy.ones(full_spde.triangulation.points.shape[0]) ) #numpy.testing.assert_almost_equal( log_rhos, 3.0 * numpy.ones(full_spde.triangulation.points.shape[0]) ) from eustace.analysis.advanced_standard.components.storage_inmemory import ComponentStorage_InMemory from eustace.analysis.advanced_standard.components.storage_inmemory import SpatialComponentSolutionStorage_InMemory from eustace.analysis.advanced_standard.components.spatialdelayed import DelayedSpatialComponent from eustace.analysis.advanced_standard.elements.local_view import NonStationaryLocal, ExpandedLocalHyperparameters from eustace.analysis.advanced_standard.elements.local import LocalElement, LocalHyperparameters nonstationary_component = DelayedSpatialComponent( ComponentStorage_InMemory( NonStationaryLocal(full_resolution_level), ExpandedLocalHyperparameters(log_sigma=log_sigmas, log_rho=log_rhos)), SpatialComponentSolutionStorage_InMemory()) #nonstationary_component = DelayedSpatialComponent( #ComponentStorage_InMemory(LocalElement(full_resolution_level), LocalHyperparameters(log_sigma = hyperparameters[0], log_rho = hyperparameters[1])), #SpatialComponentSolutionStorage_InMemory()) #print log_sigmas, log_rhos #plt.figure() #plt.scatter(polar_coords[:,1], polar_coords[:,0], c = 255.* process_sample / numpy.max(numpy.abs(process_sample)), linewidth = 0.0, s = 8.0 ) #plt.figure() #plt.imshow( numpy.asarray( Q.todense() ) ) # setup an output grid out_lats = numpy.linspace(-89.5, 89.5, 180) out_lons = numpy.linspace(-179.5, 179.5, 360) out_lons, out_lats = numpy.meshgrid(out_lons, out_lats) out_coords = numpy.vstack([out_lats.ravel(), out_lons.ravel()]).T design_matrix = nonstationary_component.storage.element.spde.build_A( out_coords) # setup solver for sampling from eustace.analysis.advanced_standard.linalg.extendedcholmodwrapper import ExtendedCholmodWrapper Q = nonstationary_component.storage.element.element_prior( nonstationary_component.storage.hyperparameters).prior_precision() factor = ExtendedCholmodWrapper.cholesky(Q) # draw samples, project onto output grid and plot random_values = numpy.random.normal(0.0, 1.0, (Q.shape[0], 1)) process_sample = factor.solve_backward_substitution(random_values) out_values = design_matrix.dot(process_sample) plt.figure() plt.scatter(out_coords[:, 1], out_coords[:, 0], c=255. * out_values / numpy.max(numpy.abs(out_values)), linewidth=0.0, s=8.0) random_values = numpy.random.normal(0.0, 1.0, (Q.shape[0], 1)) process_sample = factor.solve_backward_substitution(random_values) out_values = design_matrix.dot(process_sample) plt.figure() plt.scatter(out_coords[:, 1], out_coords[:, 0], c=255. * out_values / numpy.max(numpy.abs(out_values)), linewidth=0.0, s=8.0) random_values = numpy.random.normal(0.0, 1.0, (Q.shape[0], 1)) process_sample = factor.solve_backward_substitution(random_values) out_values = design_matrix.dot(process_sample) plt.figure() plt.scatter(out_coords[:, 1], out_coords[:, 0], c=255. * out_values / numpy.max(numpy.abs(out_values)), linewidth=0.0, s=8.0) plt.show()
def test_process_observations_no_uncertainties(self): # Our test system for the first time step (key 21) is: # # ( [ 2.0 0.0 ] + [ -1.5 ] [ 5.0 ] [ -1.5 2.2 ] ) x = [ -1.5 ] [ 5.0 ] [ 7.0 - 2.0 ] # ( [ 0.0 2.0 ] [ 2.2 ] ) [ 2.2 ] # # [ 13.25 -16.5 ] x = [ -37.5 ] # [-16.5 26.2 ] [ 55.0 ] # # => x = [-1.00133511 ] # [ 1.46862483 ] # Our test system for the first time step (key 21) is: # # ( [ 2.0 0.0 ] + [ 0.0 ] [ 5.0 ] [ 0.0 3.3 ] ) x = [ 0.0 ] [ 5.0 ] [ 9.0 - 3.0 ] # ( [ 0.0 2.0 ] [ 3.3 ] ) [ 3.3 ] # # [ 2.0 0.0 ] x = [ 0.0 ] # [ 0.0 56.45 ] [ 99.0 ] # # => x = [ 0. ] # [ 1.75376439 ] for component_storage_class in DelayedSpatialComponentSolutionStorage_Files, SpatialComponentSolutionStorage_InMemory: c = DelayedSpatialComponent( ComponentStorage_InMemory( TestDelayedSpatialComponentSolution.TestElement(), CovariateHyperparameters(-0.5 * numpy.log(2.0))), component_storage_class()) c.solutionstorage.statefiledictionary_read = None c.solutionstorage.statefiledictionary_write = { 21: 'state_test.A.pickle', 532: 'state_test.B.pickle' } c.solutionstorage.measurementfiledictionary_write = c.solutionstorage.measurementfiledictionary_read = { 21: 'measurement_test.A.pickle', 532: 'measurement_test.B.pickle' } s = c.component_solution() self.assertIsInstance(s, DelayedSpatialComponentSolution) self.assertFalse(s.compute_uncertainties) test_offset = numpy.array([2.0, 3.0]) c.solutionstorage.state_time_index = 21 c.solutionstorage.measurement_time_index_write = 21 s.process_observations( TestDelayedSpatialComponentSolution.TestObservations(t=21), test_offset[0:1]) s.update_time_step() c.solutionstorage.state_time_index = 532 c.solutionstorage.measurement_time_index_write = 532 s.process_observations( TestDelayedSpatialComponentSolution.TestObservations(t=532), test_offset[1:2]) s.update_time_step() s.update() c.solutionstorage.statefiledictionary_read = c.solutionstorage.statefiledictionary_write # now enable reading from the previously written files numpy.testing.assert_almost_equal( s.solutionstorage.partial_state_read(21), numpy.array([-1.00133511, 1.46862483])) numpy.testing.assert_almost_equal( s.solutionstorage.partial_state_read(532), numpy.array([0.0, 1.75376439])) # No marginal variances should have been computed at all self.assertEqual( None, s.solutionstorage.partial_state_marginal_std_read(21)) for time, expected_array in zip([21, 532], [ numpy.array([-1.5 * -1.00133511 + 2.2 * 1.46862483]), numpy.array([3.3 * 1.75376439]) ]): # Observation at time t=t* should be design matrix for that time multiplied by expected state numpy.testing.assert_almost_equal( s.solution_observation_expected_value( TestDelayedSpatialComponentSolution.TestObservations( t=time)), expected_array) # In this case we are considering a generical model solving iteration for the model, no marginal variances stored, hence we expect 0. as observation uncertainties numpy.testing.assert_array_equal( s.solution_observation_expected_uncertainties( TestDelayedSpatialComponentSolution.TestObservations( t=time)), 0.)
def test_process_observations_no_uncertainties(self): # Our test system is: # # ( [ 2.0 0.0 ] + [ -1.5 0.0 ] [ 5.0 0.0 ] [ -1.5 2.2 ] ) x = [ -1.5 0.0 ] [ 10.0 3.0 ] [ 7.0 - 2.0 ] # ( [ 0.0 2.0 ] [ 2.2 3.3 ] [ 0.0 5.0 ] [ 0.0 3.3 ] ) [ 2.2 3.3 ] [ 3.0 10.0 ] [ 9.0 - 3.0 ] # # [ 13.25 -16.5 ] x = [ -37.5 ] # [-16.5 80.65 ] [ 154.0 ] # # => x = [ -0.60697861 ] # [ 1.78530506 ] # c = SpaceTimeComponent(ComponentStorage_InMemory( TestSpaceTimeComponentSolution.TestElement(), CovariateHyperparameters(-0.5 * numpy.log(2.0))), SpaceTimeComponentSolutionStorage_InMemory(), sample_size=666) s = c.component_solution() self.assertIsInstance(s, SpaceTimeComponentSolution) self.assertFalse(s.compute_uncertainties) test_offset = numpy.array([2.0, 3.0]) s.process_observations( TestSpaceTimeComponentSolution.TestObservations(t=21), test_offset[0:1]) s.update_time_step() s.process_observations( TestSpaceTimeComponentSolution.TestObservations(t=532), test_offset[1:2]) s.update_time_step() s.update() numpy.testing.assert_almost_equal( s.solutionstorage.state, numpy.array([-0.60697861, 1.78530506])) # No marginal variances should have been computed at all, same for the sample self.assertEqual(None, s.solutionstorage.state_marginal_std) self.assertEqual(None, s.solutionstorage.state_sample) for time, expected_array in zip([21, 532], [ numpy.array([-1.5 * -0.60697861 + 2.2 * 1.78530506]), numpy.array([3.3 * 1.78530506]) ]): # Observation at time t=t* should be design matrix for that time multiplied by expected state numpy.testing.assert_almost_equal( s.solution_observation_expected_value( TestSpaceTimeComponentSolution.TestObservations(t=time)), expected_array) # In this case we are considering a generical model solving iteration for the model, no marginal variances stored, hence we expect 0. as observation uncertainties numpy.testing.assert_array_equal( s.solution_observation_expected_uncertainties( TestSpaceTimeComponentSolution.TestObservations(t=time)), 0.) # check samples are zero numpy.testing.assert_array_equal( 0., s.solution_observation_projected_sample( TestSpaceTimeComponentSolution.TestObservations(t=time))) # check default number of samples self.assertEqual( s.solution_observation_projected_sample( TestSpaceTimeComponentSolution.TestObservations( t=time)).shape[1], 666)
def test_mini_world_altitude(self): """Testing using altitude as a covariate""" # GENERATING OBSERVATIONS # Simulated locations: they will exactly sits on the grid points of the covariate datafile DEM = Dataset(self.altitude_datafile) latitude = DEM.variables['lat'][:] longitude = DEM.variables['lon'][:] altitude = DEM.variables['dem'][:] indices = numpy.stack( (numpy.array([1, 3, 267, 80, 10, 215, 17, 120]), numpy.array([2, 256, 9, 110, 290, 154, 34, 151])), axis=1) selected_location = [] altitude_observations = [] for couple in indices: selected_location.append([ latitude[couple[0], couple[1]], longitude[couple[0], couple[1]] ]) altitude_observations.append(altitude[couple[0], couple[1]]) DEM.close() locations = numpy.array(selected_location) # Simulated measurements: simple linear relation of type: y = PI*x measurement = numpy.pi * numpy.array(altitude_observations) # Simulated errors uncorrelatederror = 0.1 * numpy.ones(measurement.shape) # Simulated inputs simulated_input_loader = SimulatedInputLoader(locations, measurement, uncorrelatederror) # Simulate evaluation of this time index simulated_time_indices = [0] # GENERATING THE MODEL # Local component geography_covariate_element = GeographyBasedElement( self.altitude_datafile, 'lat', 'lon', 'dem', 1.0) geography_covariate_element.load() geography_based_component = SpatialComponent( ComponentStorage_InMemory( geography_covariate_element, CovariateHyperparameters(-0.5 * numpy.log(10.))), SpatialComponentSolutionStorage_InMemory()) # GENERATING THE ANALYSIS # Analysis system using the specified components, for the Tmean observable analysis_system = AnalysisSystem([geography_based_component], ObservationSource.TMEAN, log=StringIO()) # Update with data analysis_system.update([simulated_input_loader], simulated_time_indices) # Check state vector directly statevector = analysis_system.components[ 0].solutionstorage.partial_state_read(0).ravel() # These are the nodes where observations were put (see SimulatedObservationSource above) # - check they correspond to within 3 times the stated noise level self.assertAlmostEqual(numpy.pi, statevector[0], delta=0.3) # Also check entire state vector within outer bounds set by obs self.assertTrue(all(statevector < numpy.pi)) # And check output corresponds too # (evaluate result on output structure same as input) simulated_output_structure = SimulatedObservationStructure( 0, locations, None, None) result = analysis_system.evaluate_expected_value( 'MAP', simulated_output_structure, flag='POINTWISE') numpy.testing.assert_almost_equal( statevector[0] * numpy.array(altitude_observations), result)
def test_mini_world_altitude_with_latitude(self): """Testing using altitude as a covariate""" # GENERATING OBSERVATIONS # Simulated locations: they will exactly sits on the grid points of the covariate datafile DEM = Dataset(self.altitude_datafile) latitude = DEM.variables['lat'][:] longitude = DEM.variables['lon'][:] altitude = DEM.variables['dem'][:] indices = numpy.stack( (numpy.array([1, 3, 5, 7, 8, 9, 10, 11 ]), numpy.array([0, 0, 0, 0, 0, 0, 0, 0])), axis=1) selected_location = [] altitude_observations = [] for couple in indices: selected_location.append([ latitude[couple[0], couple[1]], longitude[couple[0], couple[1]] ]) altitude_observations.append(altitude[couple[0], couple[1]]) DEM.close() locations = numpy.array(selected_location) # Simulated model is y = z + a*cos(2x) + c*cos(4*x) + b*sin(2x) + d*sin(4*x), with z = altitude, x = latitude, a=b=c=d=0 slope = 1e-3 measurement = slope * numpy.array(altitude_observations) # Simulated errors uncorrelatederror = 0.1 * numpy.ones(measurement.shape) # Simulated inputs simulated_input_loader = SimulatedInputLoader(locations, measurement, uncorrelatederror) # Simulate evaluation of this time index simulated_time_indices = [0] # GENERATING THE MODEL # Local component geography_covariate_element = GeographyBasedElement( self.altitude_datafile, 'lat', 'lon', 'dem', 1.0) geography_covariate_element.load() combined_element = CombinationElement( [geography_covariate_element, LatitudeHarmonicsElement()]) combined_hyperparamters = CombinationHyperparameters([ CovariateHyperparameters(-0.5 * numpy.log(10.)), CombinationHyperparameters([ CovariateHyperparameters(-0.5 * numpy.log(p)) for p in [10.0, 10.0, 10.0, 10.0] ]) ]) combined_component = SpatialComponent( ComponentStorage_InMemory(combined_element, combined_hyperparamters), SpatialComponentSolutionStorage_InMemory()) # GENERATING THE ANALYSIS # Analysis system using the specified components, for the Tmean observable analysis_system = AnalysisSystem([combined_component], ObservationSource.TMEAN, log=StringIO()) # Update with data analysis_system.update([simulated_input_loader], simulated_time_indices) # Check state vector directly statevector = analysis_system.components[ 0].solutionstorage.partial_state_read(0).ravel() # These are the nodes where observations were put (see SimulatedObservationSource above) # - check they correspond to within 3 times the stated noise level self.assertAlmostEqual(slope, statevector[0], delta=0.3) self.assertAlmostEqual(0., statevector[1], delta=0.3) self.assertAlmostEqual(0., statevector[2], delta=0.3) self.assertAlmostEqual(0., statevector[3], delta=0.3) self.assertAlmostEqual(0., statevector[4], delta=0.3) # And check output corresponds too # (evaluate result on output structure same as input) simulated_output_structure = SimulatedObservationStructure( 0, locations, None, None) result = analysis_system.evaluate_expected_value( 'MAP', simulated_output_structure, flag='POINTWISE') expected = statevector[0]*numpy.array(altitude_observations)\ + statevector[1]*LatitudeFunction(numpy.cos, 2.0).compute(locations[:,0]).ravel()\ + statevector[2]*LatitudeFunction(numpy.sin, 2.0).compute(locations[:,0]).ravel()\ + statevector[3]*LatitudeFunction(numpy.cos, 4.0).compute(locations[:,0]).ravel()\ + statevector[4]*LatitudeFunction(numpy.sin, 2.0).compute(locations[:,0]).ravel() numpy.testing.assert_almost_equal(expected, result) # test output gridding, pointwise limit outputstructure = OutputRectilinearGridStructure( 2, epoch_plus_days(2), latitudes=numpy.linspace(-60., 60., num=5), longitudes=numpy.linspace(-90., 90, num=10)) pointwise_result = analysis_system.evaluate_expected_value( 'MAP', outputstructure, 'POINTWISE') pointwise_limit_result = analysis_system.evaluate_expected_value( 'MAP', outputstructure, 'GRID_CELL_AREA_AVERAGE', [1, 1], 10) numpy.testing.assert_array_almost_equal(pointwise_result, pointwise_limit_result)
def test_mini_world_noiseless(self): # Use a number of time steps number_of_simulated_time_steps = 1 # Build system element = SpaceTimeFactorElement( n_triangulation_divisions=3, alpha=2, starttime=0, endtime=number_of_simulated_time_steps + 1, n_nodes=number_of_simulated_time_steps + 2, overlap_factor=2.5, H=1) initial_hyperparameters = SpaceTimeSPDEHyperparameters( space_log_sigma=0.0, space_log_rho=numpy.log(numpy.radians(45.0)), time_log_rho=numpy.log(3.0 / 365.0)) component = SpaceTimeComponent( ComponentStorage_InMemory(element, initial_hyperparameters), SpaceTimeComponentSolutionStorage_InMemory()) analysis_system = AnalysisSystem([component], ObservationSource.TMEAN, log=StringIO()) # use fixed locations from icosahedron fixed_locations = cartesian_to_polar2d( MeshIcosahedronSubdivision.build(3).points) # random measurement at each location numpy.random.seed(8976) field_basis = 10.0 * numpy.random.randn(fixed_locations.shape[0]) # some time function that varies over a year time_basis = numpy.cos( numpy.linspace(0.1, 1.75 * numpy.pi, number_of_simulated_time_steps)) # kronecker product of the two measurement = numpy.kron(field_basis, numpy.expand_dims(time_basis, 1)) # Simulated inputs simulated_input_loader = SimulatedInputLoader(fixed_locations, measurement, 0.01) # Simulate evaluation of this time index simulated_time_indices = range(number_of_simulated_time_steps) # Iterate for iteration in range(5): analysis_system.update([simulated_input_loader], simulated_time_indices) # Get all results result = numpy.zeros(measurement.shape) for t in range(number_of_simulated_time_steps): result[t, :] = analysis_system.evaluate_expected_value( 'MAP', SimulatedObservationStructure(t, fixed_locations, None, None), flag='POINTWISE') # Should be very close to original because specified noise is low numpy.testing.assert_almost_equal(result, measurement) max_disparity = (numpy.abs(result - measurement)).ravel().max() self.assertTrue(max_disparity < 1E-5)
def test_process_observations_compute_uncertainties(self): # Our test system is: # # ( [ 2.0 0.0 ] + [ -1.5 0.0 ] [ 10.0 3.0 ] [ -1.5 2.2 ] ) x = [ -1.5 0.0 ] [ 10.0 3.0 ] [ 7.0 - 2.0 ] # ( [ 0.0 2.0 ] [ 2.2 3.3 ] [ 3.0 10.0 ] [ 0.0 3.3 ] ) [ 2.2 3.3 ] [ 3.0 10.0 ] [ 9.0 - 3.0 ] # # [ 24.5 -47.85 ] x = [ -102.0 ] # [ -47.85 202.86 ] [ 397.1 ] # # => x = [ -0.63067268 ] # [ 1.80874649 ] # # Input data test_offset = numpy.array([2.0, 3.0]) test_obs = TestSpatialComponentSolution.TestObservations() # Make component and check it's of the correct class c = SpatialComponent(ComponentStorage_InMemory( TestSpatialComponentSolution.TestElement(), CovariateHyperparameters(-0.5 * numpy.log(2.0))), SpatialComponentSolutionStorage_InMemory(), compute_uncertainties=True, sample_size=666) s = c.component_solution() self.assertIsInstance(s, SpatialComponentSolution) self.assertTrue(s.compute_uncertainties) self.assertFalse(s.compute_sample) # Do the processing s.process_observations(test_obs, test_offset) s.update_time_step() # In this case we are considering the last iteration of model solving, hence marginal variances should have been stored expected_marginal_std = numpy.sqrt( numpy.diag( numpy.linalg.inv( numpy.array([[24.5, -47.85], [-47.85, 202.86]])))) numpy.testing.assert_array_almost_equal( s.solutionstorage.partial_state_marginal_std_read(21), expected_marginal_std) self.assertEqual(len(s.solutionstorage.state_marginal_std_at_time), 1) self.assertListEqual( s.solutionstorage.state_marginal_std_at_time.keys(), [21]) # We also test the computation of prior marginal variances, and their projection onto observations expected_prior_std = numpy.sqrt( numpy.diag(numpy.linalg.inv(numpy.array([[2., 0.], [0., 2.]])))) numpy.testing.assert_array_almost_equal(s.solution_prior_std(100), expected_prior_std, decimal=1) numpy.testing.assert_array_almost_equal(s.solution_prior_std(10000), expected_prior_std, decimal=2) design_matrix = numpy.array([[-1.5, 2.2], [0.0, 3.3]]) expected_prior_std_projection = numpy.dot(design_matrix, expected_prior_std) numpy.testing.assert_array_almost_equal( s.solution_observation_prior_uncertainties(None), expected_prior_std_projection, decimal=1) # In this case we did not allowed the computation of uncertainties samples self.assertEqual(s.solutionstorage.partial_state_sample_read(21), None) self.assertEqual(s.solutionstorage.state_sample_at_time, {}) numpy.testing.assert_almost_equal( s.solution_observation_projected_sample( TestSpatialComponentSolution.TestObservations()), 0.) self.assertEqual( 666, s.solution_observation_projected_sample( TestSpatialComponentSolution.TestObservations()).shape[1])
def test_mini_world_noiseless(self): number_of_simulated_time_steps = 1 # Build system element = SeasonalElement(n_triangulation_divisions=3, n_harmonics=5, include_local_mean=True) hyperparameters = SeasonalHyperparameters(n_spatial_components=6, common_log_sigma=0.0, common_log_rho=0.0) component = SpaceTimeComponent( ComponentStorage_InMemory(element, hyperparameters), SpaceTimeComponentSolutionStorage_InMemory()) analysis_system = AnalysisSystem([component], ObservationSource.TMEAN, log=StringIO()) # use fixed locations from icosahedron fixed_locations = cartesian_to_polar2d( MeshIcosahedronSubdivision.build(3).points) # random measurement at each location numpy.random.seed(8976) field_basis = numpy.random.randn(fixed_locations.shape[0]) #print(field_basis.shape) #time_basis = numpy.array(harmonics_list) # some time function that varies over a year #decimal_years = numpy.array([datetime_to_decimal_year(epoch_plus_days(step)) for step in range(number_of_simulated_time_steps)]) time_basis = numpy.cos( numpy.linspace(0.1, 1.75 * numpy.pi, number_of_simulated_time_steps)) # kronecker product of the two #print(numpy.expand_dims(time_basis, 1)) measurement = numpy.kron(field_basis, numpy.expand_dims( time_basis, 1)) #numpy.expand_dims(time_basis, 1)) #print(measurement.shape) # Simulated inputs simulated_input_loader = SimulatedInputLoader(fixed_locations, measurement, 0.0001) # Simulate evaluation of this time index simulated_time_indices = range(number_of_simulated_time_steps) # Iterate for iteration in range(5): analysis_system.update([simulated_input_loader], simulated_time_indices) # Get all results result = numpy.zeros(measurement.shape) for t in range(number_of_simulated_time_steps): result[t, :] = analysis_system.evaluate_expected_value( 'MAP', SimulatedObservationStructure(t, fixed_locations, None, None), flag='POINTWISE') # Should be very close to original because specified noise is low numpy.testing.assert_almost_equal(result, measurement) max_disparity = (numpy.abs(result - measurement)).ravel().max() self.assertTrue(max_disparity < 1E-5) # test output gridding, pointwise limit outputstructure = OutputRectilinearGridStructure( 2, epoch_plus_days(2), latitudes=numpy.linspace(-60., 60., num=5), longitudes=numpy.linspace(-90., 90, num=10)) pointwise_result = analysis_system.evaluate_expected_value( 'MAP', outputstructure, 'POINTWISE') pointwise_limit_result = analysis_system.evaluate_expected_value( 'MAP', outputstructure, 'GRID_CELL_AREA_AVERAGE', [1, 1], 10) numpy.testing.assert_array_almost_equal(pointwise_result, pointwise_limit_result)
def test_process_observations(self): # Our test system is: # # ( [ 2.0 0.0 ] + [ -1.5 0.0 ] [ 10.0 3.0 ] [ -1.5 2.2 ] ) x = [ -1.5 0.0 ] [ 10.0 3.0 ] [ 7.0 - 2.0 ] # ( [ 0.0 2.0 ] [ 2.2 3.3 ] [ 3.0 10.0 ] [ 0.0 3.3 ] ) [ 2.2 3.3 ] [ 3.0 10.0 ] [ 9.0 - 3.0 ] # # [ 24.5 -47.85 ] x = [ -102.0 ] # [ -47.85 202.86 ] [ 397.1 ] # # => x = [ -0.63067268 ] # [ 1.80874649 ] # # Input data test_offset = numpy.array([2.0, 3.0]) test_obs = TestSpatialComponentSolution.TestObservations() # Make component and check it's of the correct class c = SpatialComponent( ComponentStorage_InMemory( TestSpatialComponentSolution.TestElement(), CovariateHyperparameters(-0.5 * numpy.log(2.0))), SpatialComponentSolutionStorage_InMemory()) s = c.component_solution() self.assertIsInstance(s, SpatialComponentSolution) self.assertFalse(s.compute_uncertainties) self.assertFalse(s.compute_sample) # Do the processing s.process_observations(test_obs, test_offset) s.update_time_step() # Should be the value of x (see matrices in comment above) numpy.testing.assert_almost_equal( s.solutionstorage.partial_state_read(21), numpy.array([[-0.63067268], [1.80874649]])) # Should be the value of x multiplied by the design matrix numpy.testing.assert_almost_equal( s.solution_observation_expected_value( TestSpatialComponentSolution.TestObservations()), numpy.array([[-1.5 * -0.63067268 + 2.2 * 1.80874649], [3.3 * 1.80874649]])) # This should be zero as we have no information for other times numpy.testing.assert_almost_equal( s.solution_observation_expected_value( TestSpatialComponentSolution.TestSomeOtherTime()), numpy.zeros((7, ))) # In this case we did not allowed the computation of uncertainties or samples, hence the state of marginal variances should be None, while the expected uncertainties zero self.assertEqual(s.solutionstorage.partial_state_marginal_std_read(21), None) self.assertEqual(s.solutionstorage.state_marginal_std_at_time, {}) numpy.testing.assert_almost_equal( s.solution_observation_expected_uncertainties( TestSpatialComponentSolution.TestObservations()), 0.) self.assertEqual(s.solutionstorage.partial_state_sample_read(21), None) self.assertEqual(s.solutionstorage.state_sample_at_time, {}) numpy.testing.assert_almost_equal( s.solution_observation_projected_sample( TestSpatialComponentSolution.TestObservations()), 0.)
def main(): print 'Advanced standard example using a few days of EUSTACE data' parser = argparse.ArgumentParser( description='Advanced standard example using a few days of EUSTACE data' ) parser.add_argument('outpath', help='directory where the output should be redirected') parser.add_argument( '--json_descriptor', default=None, help= 'a json descriptor containing the covariates to include in the climatology model' ) parser.add_argument('--land_biases', action='store_true', help='include insitu land homogenization bias terms') parser.add_argument('--global_biases', action='store_true', help='include global satellite bias terms') parser.add_argument('--n_iterations', type=int, default=5, help='number of solving iterations') args = parser.parse_args() # Input data path basepath = os.path.join('/work/scratch/eustace/rawbinary3') # Days to process #time_indices = range(int(days_since_epoch(datetime(2006, 2, 1))), int(days_since_epoch(datetime(2006, 2, 2)))) #time_indices = range(int(days_since_epoch(datetime(1906, 2, 1))), int(days_since_epoch(datetime(1906, 2, 2)))) date_list = [ datetime(2006, 1, 1) + relativedelta(days=k) for k in range(3) ] #backwards_list = [date_list[i] for i in range(11, -1, -1)] #date_list = backwards_list time_indices = [int(days_since_epoch(date)) for date in date_list] # Sources to use sources = [ 'surfaceairmodel_land', 'surfaceairmodel_ocean', 'surfaceairmodel_ice', 'insitu_land', 'insitu_ocean' ] sources = ['insitu_land', 'insitu_ocean'] #sources = [ 'surfaceairmodel_land' ] # CLIMATOLOGY COMPONENT: combining the seasonal core along with latitude harmonics, altitude and coastal effects if args.json_descriptor is not None: loader = LoadCovariateElement(args.json_descriptor) loader.check_keys() covariate_elements, covariate_hyperparameters = loader.load_covariates_and_hyperparameters( ) print( 'The following fields have been added as covariates of the climatology model' ) print(loader.data.keys()) else: covariate_elements, covariate_hyperparameters = [], [] #climatology_element = CombinationElement( [SeasonalElement(n_triangulation_divisions=2, n_harmonics=2, include_local_mean=False), GrandMeanElement()]+covariate_elements) #climatology_hyperparameters = CombinationHyperparameters( [SeasonalHyperparameters(n_spatial_components=2, common_log_sigma=0.0, common_log_rho=0.0), CovariateHyperparameters(numpy.log(15.0))] + covariate_hyperparameters ) climatology_element = CombinationElement([ GrandMeanElement(), ] + covariate_elements) climatology_hyperparameters = CombinationHyperparameters([ CovariateHyperparameters(numpy.log(15.0)), ] + covariate_hyperparameters) #climatology_element =SeasonalElement(n_triangulation_divisions=2, n_harmonics=2, include_local_mean=False) #climatology_hyperparameters = SeasonalHyperparameters(n_spatial_components=2, common_log_sigma=0.0, common_log_rho=0.0) climatology_component = SpaceTimeComponent( ComponentStorage_InMemory(climatology_element, climatology_hyperparameters), SpaceTimeComponentSolutionStorage_InMemory(), compute_uncertainties=True, method='APPROXIMATED') # LARGE SCALE (kronecker product) COMPONENT: combining large scale trends with bias terms accounting for homogeneization effects if args.land_biases: bias_element, bias_hyperparameters = [ InsituLandBiasElement(BREAKPOINTS_FILE) ], [CovariateHyperparameters(numpy.log(.9))] print('Adding bias terms for insitu land homogenization') else: bias_element, bias_hyperparameters = [], [] large_scale_element = CombinationElement([ SpaceTimeKroneckerElement(n_triangulation_divisions=2, alpha=2, starttime=-30, endtime=365 * 1 + 30, n_nodes=12 * 1 + 2, overlap_factor=2.5, H=1) ] + bias_element) large_scale_hyperparameters = CombinationHyperparameters([ SpaceTimeSPDEHyperparameters(space_log_sigma=0.0, space_log_rho=numpy.log( numpy.radians(15.0)), time_log_rho=numpy.log(15.0)) ] + bias_hyperparameters) large_scale_component = SpaceTimeComponent( ComponentStorage_InMemory(large_scale_element, large_scale_hyperparameters), SpaceTimeComponentSolutionStorage_InMemory(), compute_uncertainties=True, method='APPROXIMATED') # LOCAL COMPONENT: combining local scale variations with global satellite bias terms if args.global_biases: bias_elements = [ BiasElement(groupname, 1) for groupname in GLOBAL_BIASES_GROUP_LIST ] bias_hyperparameters = [ CovariateHyperparameters(numpy.log(15.0)) for index in range(3) ] print('Adding global bias terms for all the surfaces') else: bias_elements, bias_hyperparameters = [], [] n_triangulation_divisions_local = 7 local_log_sigma = numpy.log(5) local_log_rho = numpy.log(numpy.radians(5.0)) local_element = NonStationaryLocal( n_triangulation_divisions=n_triangulation_divisions_local) n_local_nodes = local_element.spde.n_latent_variables() local_scale_element = CombinationElement([local_element] + bias_elements) local_hyperparameters = ExpandedLocalHyperparameters( log_sigma=numpy.repeat(local_log_sigma, n_local_nodes), log_rho=numpy.repeat(local_log_rho, n_local_nodes)) local_scale_hyperparameters = CombinationHyperparameters( [local_hyperparameters] + bias_hyperparameters) local_component = DelayedSpatialComponent( ComponentStorage_InMemory(local_scale_element, local_scale_hyperparameters), SpatialComponentSolutionStorage_InMemory(), compute_uncertainties=True, method='APPROXIMATED') print "hyperparameter storage:", local_component.storage.hyperparameters print 'Analysing inputs' # Analysis system using the specified components, for the Tmean observable ##analysis_system = AnalysisSystem( ## [ climatology_component, large_scale_component, local_component ], ## ObservationSource.TMEAN) analysis_system = OptimizationSystem( [climatology_component, local_component], ObservationSource.TMEAN) # Object to load raw binary inputs at time indices inputloaders = [ AnalysisSystemInputLoaderRawBinary_Sources(basepath, source, time_indices) for source in sources ] for iteration in range(args.n_iterations): message = 'Iteration {}'.format(iteration) print(message) # Update with data analysis_system.update(inputloaders, time_indices) ################################################## # Optimize local model hyperparameters # Loop over local regions, generate optimization systems, fit hyperparameters and save # split spde and bias models for local component into two components global_spde_sub_component_definition = ComponentStorage_InMemory( CombinationElement([local_element]), CombinationHyperparameters([local_hyperparameters])) global_spde_sub_component_storage_solution = SpatialComponentSolutionStorage_InMemory( ) global_spde_sub_component = DelayedSpatialComponent( global_spde_sub_component_definition, global_spde_sub_component_storage_solution) bias_sub_component_definition = ComponentStorage_InMemory( CombinationElement(bias_elements), CombinationHyperparameters(bias_hyperparameters)) bias_sub_component_storage_solution = SpatialComponentSolutionStorage_InMemory( ) bias_sub_component = DelayedSpatialComponent( bias_sub_component_definition, bias_sub_component_storage_solution) element_optimisation_flags = [True, False, False, False] # one spde, three biases for time_key in time_indices: split_states_time(local_component, global_spde_sub_component, bias_sub_component, element_optimisation_flags, time_key) # Define subregions and extract their states neighbourhood_level = 1 n_subregions = global_spde_sub_component.storage.element_read( ).combination[0].spde.n_triangles_at_level(neighbourhood_level) hyperparameter_file_template = "local_hyperparameters.%i.%i.%i.npy" fit_hyperparameters = True optimization_component_index = 2 if fit_hyperparameters: for region_index in range(n_subregions): # Setup model for local subregion of neighours with super triangle view_flags = [ True, ] region_element = CombinationElement([ LocalSubRegion(n_triangulation_divisions_local, neighbourhood_level, region_index) ]) region_hyperparameters = ExtendedCombinationHyperparameters([ LocalHyperparameters(log_sigma=local_log_sigma, log_rho=local_log_rho) ]) region_component_storage_solution = SpatialComponentSolutionStorage_InMemory( ) region_sub_component = DelayedSpatialComponent( ComponentStorage_InMemory(region_element, region_hyperparameters), region_component_storage_solution) for time_key in time_indices: print "region_index, time_key:", region_index, time_key extract_local_view_states_time(global_spde_sub_component, region_sub_component, view_flags, time_key) print "running optimization for region:", region_index region_optimization_system = OptimizationSystem([ climatology_component, bias_sub_component, region_sub_component ], ObservationSource.TMEAN) for time_key in time_indices: region_optimization_system.update_component_time( inputloaders, optimization_component_index, time_key) # commented version that works for few days inputs #region_optimization_system.components[optimization_component_index].component_solution().optimize() #region_optimization_system.components[optimization_component_index].storage.hyperparameters.get_array() #hyperparameter_file = os.path.join(args.outpath, hyperparameter_file_template % (n_triangulation_divisions_local, neighbourhood_level, region_index) ) #region_sub_component.storage.hyperparameters.values_to_npy_savefile( hyperparameter_file ) # replaced with version for full processing based json dump of input files - need to generate the input_descriptor dict hyperparameter_file = os.path.join( args.outpath, hyperparameter_file_template % (n_triangulation_divisions_local, neighbourhood_level, region_index)) region_optimization_system.process_inputs( input_descriptor, optimization_component_index, time_indices) region_optimization_system.optimize_component( optimization_component_index, hyperparameter_storage_file=hyperparameter_file) fitted_hyperparameters_converted = region_sub_component.storage.hyperparameters.get_array( ) fitted_hyperparameters_converted[0] = numpy.exp( fitted_hyperparameters_converted[0]) fitted_hyperparameters_converted[1] = numpy.exp( fitted_hyperparameters_converted[1]) * 180.0 / numpy.pi print 'fitted_hyperparameters_converted:', fitted_hyperparameters_converted # Setup model for the super triangle without neighbours for hyperparameter merging region_spdes = [] region_hyperparameter_values = [] for region_index in range(n_subregions): # Redefine the region sub component as a supertriangle rather than a neighbourhood region_element = CombinationElement([ LocalSuperTriangle(n_triangulation_divisions_local, neighbourhood_level, region_index) ]) region_hyperparameters = ExtendedCombinationHyperparameters([ LocalHyperparameters(log_sigma=local_log_sigma, log_rho=local_log_rho) ]) region_component_storage_solution = SpatialComponentSolutionStorage_InMemory( ) region_sub_component = DelayedSpatialComponent( ComponentStorage_InMemory(region_element, region_hyperparameters), region_component_storage_solution) # Read the optimized hyperparameters hyperparameter_file = os.path.join( args.outpath, hyperparameter_file_template % (n_triangulation_divisions_local, neighbourhood_level, region_index)) region_sub_component.storage.hyperparameters.values_from_npy_savefile( hyperparameter_file) # Append the spde model and hyperparameters to their lists for merging region_spdes.append(region_element.combination[0].spde) region_hyperparameter_values.append( region_sub_component.storage.hyperparameters.get_array()) # merge and save hyperparameters full_spde = local_element.spde new_hyperparameter_values, global_sigma_design, global_rho_design = full_spde.merge_local_parameterisations( region_spdes, region_hyperparameter_values, merge_method='exp_average') local_hyperparameters.set_array(new_hyperparameter_values) hyperparameter_file_merged = "merged_hyperparameters.%i.%i.npy" % ( n_triangulation_divisions_local, neighbourhood_level) local_hyperparameters.values_to_npy_savefile( os.path.join(args.outpath, hyperparameter_file_merged)) # Refit local model with the optimized hyperparameters analysis_system.update_component(inputloaders, 1, time_indices) ################################################## print 'Computing outputs' # Produce an output for each time index for time_index in time_indices: # Get date for output outputdate = inputloaders[0].datetime_at_time_index(time_index) print 'Evaluating output grid: ', outputdate #Configure output grid outputstructure = OutputRectilinearGridStructure( time_index, outputdate, latitudes=numpy.linspace(-89.875, 89.875, num=definitions.GLOBAL_FIELD_SHAPE[1]), longitudes=numpy.linspace(-179.875, 179.875, num=definitions.GLOBAL_FIELD_SHAPE[2])) # print 'Size of grid : ', outputstructure.number_of_observations() # Evaluate expected value at these locations result_expected_value = analysis_system.evaluate_expected_value( 'MAP', outputstructure, 'POINTWISE') result_expected_uncertainties = analysis_system.evaluate_expected_value( 'post_STD', outputstructure, 'POINTWISE') # Make output filename pathname = 'eustace_example_output_{0:04d}{1:02d}{2:02d}.nc'.format( outputdate.year, outputdate.month, outputdate.day) pathname = os.path.join(args.outpath, pathname) print 'Saving: ', pathname # Save results filebuilder = FileBuilderGlobalField( pathname, time_index, 'Infilling Example', 'UNVERSIONED', definitions.TAS.name, '', 'Example data only', 'eustace.analysis.advanced_standard.examples.example_eustace_few_days', '') filebuilder.add_global_field( definitions.TAS, result_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.add_global_field( definitions.TASUNCERTAINTY, result_expected_uncertainties.reshape( definitions.GLOBAL_FIELD_SHAPE)) filebuilder.save_and_close() print 'Complete'