def test_get_array(self): h = SeasonalHyperparameters(n_spatial_components=3, common_log_sigma=0.8, common_log_rho=1.0) numpy.testing.assert_equal(h.get_array(), [0.8, 1.0, 0.8, 1.0, 0.8, 1.0])
def test_harmonic_index_for_spatial_component(self): prior_a = SeasonalElementPrior( SeasonalHyperparameters(n_spatial_components=3, common_log_sigma=0.8, common_log_rho=1.0), spde=SphereMeshSPDE(level=1, sparse_format=SPARSEFORMAT), n_harmonics=2, include_local_mean=True, alpha=2) self.assertEqual(0, prior_a.harmonic_index_for_spatial_component(0)) self.assertEqual(1, prior_a.harmonic_index_for_spatial_component(1)) self.assertEqual(1, prior_a.harmonic_index_for_spatial_component(2)) self.assertEqual(2, prior_a.harmonic_index_for_spatial_component(3)) self.assertEqual(2, prior_a.harmonic_index_for_spatial_component(4)) prior_b = SeasonalElementPrior( SeasonalHyperparameters(n_spatial_components=2, common_log_sigma=0.8, common_log_rho=1.0), spde=SphereMeshSPDE(level=1, sparse_format=SPARSEFORMAT), n_harmonics=2, include_local_mean=False, alpha=2) self.assertEqual(0, prior_b.harmonic_index_for_spatial_component(0)) self.assertEqual(0, prior_b.harmonic_index_for_spatial_component(1)) self.assertEqual(1, prior_b.harmonic_index_for_spatial_component(2)) self.assertEqual(1, prior_b.harmonic_index_for_spatial_component(3))
def test_set_array(self): h = SeasonalHyperparameters(n_spatial_components=3, common_log_sigma=0.8, common_log_rho=1.0) h.set_array([88.2, 103.4, 0.2, 0.3, 0.4, 0.5]) numpy.testing.assert_equal(h.harmonics, [88.2, 103.4, 0.2, 0.3, 0.4, 0.5])
def __init__(self): super(ClimatologyDefinition, self).__init__( SeasonalElement(n_triangulation_divisions=5, n_harmonics=5, include_local_mean=True), SeasonalHyperparameters(n_spatial_components=6, common_log_sigma=1.0, common_log_rho=0.0))
def test_n_spatial_components(self): # With local mean 2 harmonics plus mean = (1 + 2*2) = 5 parameters self.assertEqual( 5, SeasonalElementPrior( SeasonalHyperparameters(n_spatial_components=3, common_log_sigma=0.8, common_log_rho=1.0), spde=SphereMeshSPDE(level=1, sparse_format=SPARSEFORMAT), n_harmonics=2, include_local_mean=True, alpha=2).n_spatial_components()) # Without local mean 2 harmonics = 2*2 = 4 parameters self.assertEqual( 4, SeasonalElementPrior( SeasonalHyperparameters(n_spatial_components=2, common_log_sigma=0.8, common_log_rho=1.0), spde=SphereMeshSPDE(level=1, sparse_format=SPARSEFORMAT), n_harmonics=2, include_local_mean=False, alpha=2).n_spatial_components()) # Also check state parameters self.assertEqual( 210, SeasonalElementPrior( SeasonalHyperparameters(n_spatial_components=3, common_log_sigma=0.8, common_log_rho=1.0), spde=SphereMeshSPDE(level=1, sparse_format=SPARSEFORMAT), n_harmonics=2, include_local_mean=True, alpha=2).prior_number_of_state_parameters())
def test_harmonic_parameters(self): h = SeasonalHyperparameters(n_spatial_components=3, common_log_sigma=0.8, common_log_rho=1.0) h.set_array([88.2, 103.4, 0.2, 0.3, 0.4, 0.5]) self.assertEqual(0.2, h.harmonic_parameters(1).log_sigma) self.assertEqual(103.4, h.harmonic_parameters(0).log_rho) self.assertEqual({ 'log_sigma': 0.4, 'log_rho': 0.5 }, h.harmonic_parameters(2).get_dictionary())
def test_element_prior(self): element = SeasonalElement(n_triangulation_divisions=1, n_harmonics=3, include_local_mean=False) prior = element.element_prior( SeasonalHyperparameters(n_spatial_components=3, common_log_sigma=0.5, common_log_rho=0.7)) self.assertTrue(isinstance(prior, SeasonalElementPrior)) numpy.testing.assert_equal([0.5, 0.7, 0.5, 0.7, 0.5, 0.7], prior.hyperparameters.get_array()) self.assertEqual(3, prior.n_harmonics) self.assertEqual(False, prior.include_local_mean) self.assertEqual(2, prior.alpha)
def test_prior_precision_derivative(self): prior = SeasonalElementPrior( SeasonalHyperparameters(n_spatial_components=3, common_log_sigma=0.8, common_log_rho=1.0), spde=SphereMeshSPDE(level=1, sparse_format=SPARSEFORMAT), n_harmonics=2, include_local_mean=True, alpha=2) dQ = prior.prior_precision_derivative(1) # For now just check size and format of result self.assertEqual(SPARSEFORMAT, dQ.getformat()) self.assertEqual((210, 210), dQ.shape)
def test_init(self): prior = SeasonalElementPrior( SeasonalHyperparameters(n_spatial_components=3, common_log_sigma=0.8, common_log_rho=1.0), spde=SphereMeshSPDE(level=1, sparse_format=SPARSEFORMAT), n_harmonics=2, include_local_mean=True, alpha=3) numpy.testing.assert_equal(prior.hyperparameters.get_array(), [0.8, 1.0, 0.8, 1.0, 0.8, 1.0]) self.assertEqual(2, prior.n_harmonics) self.assertTrue(prior.include_local_mean) self.assertEqual(3, prior.alpha)
def __init__(self, covariates_descriptor): if covariates_descriptor is not None: loader = LoadCovariateElement(covariates_descriptor) loader.check_keys() covariate_elements, covariate_hyperparameters = loader.load_covariates_and_hyperparameters() print('The following fields have been added as covariates of the climatology model') print(loader.data.keys()) else: covariate_elements, covariate_hyperparameters = [], [] setup = ClimatologySetup() super(ClimatologyDefinition, self).__init__( CombinationElement( [SeasonalElement(setup.n_triangulation_divisions, setup.n_harmonics, include_local_mean=True), GrandMeanElement()] + covariate_elements), CombinationHyperparameters( [SeasonalHyperparameters(setup.n_spatial_components, numpy.log(setup.amplitude), numpy.log(numpy.radians(setup.space_length_scale))), CovariateHyperparameters(numpy.log(setup.grandmean_amplitude))] + covariate_hyperparameters))
def main(): print 'Advanced standard example using a few days of EUSTACE data' parser = argparse.ArgumentParser(description='Advanced standard example using a few days of EUSTACE data') parser.add_argument('outpath', help='directory where the output should be redirected') parser.add_argument('--json_descriptor', default = None, help='a json descriptor containing the covariates to include in the climatology model') parser.add_argument('--land_biases', action='store_true', help='include insitu land homogenization bias terms') parser.add_argument('--global_biases', action='store_true', help='include global satellite bias terms') parser.add_argument('--n_iterations', type=int, default=5, help='number of solving iterations') args = parser.parse_args() # Input data path basepath = os.path.join('/work/scratch/eustace/rawbinary3') # Days to process time_indices = range(int(days_since_epoch(datetime(2006, 2, 1))), int(days_since_epoch(datetime(2006, 2, 2)))) # Sources to use sources = [ 'surfaceairmodel_land', 'surfaceairmodel_ocean', 'surfaceairmodel_ice', 'insitu_land', 'insitu_ocean' ] #SETUP # setup for the seasonal core: climatology covariates setup read from file seasonal_setup = {'n_triangulation_divisions':5, 'n_harmonics':4, 'n_spatial_components':6, 'amplitude':2., 'space_length_scale':5., # length scale in units of degrees } grandmean_amplitude = 15.0 # setup for the large scale component spacetime_setup = {'n_triangulation_divisions':2, 'alpha':2, 'starttime':0, 'endtime':10., 'n_nodes':2, 'overlap_factor':2.5, 'H':1, 'amplitude':1., 'space_lenght_scale':15.0, # length scale in units of degrees 'time_length_scale':15.0 # length scal in units of days } bias_amplitude = .9 # setup for the local component local_setup = {'n_triangulation_divisions':6, 'amplitude':2., 'space_length_scale':2. # length scale in units of degrees } globalbias_amplitude = 15.0 # CLIMATOLOGY COMPONENT: combining the seasonal core along with latitude harmonics, altitude and coastal effects if args.json_descriptor is not None: loader = LoadCovariateElement(args.json_descriptor) loader.check_keys() covariate_elements, covariate_hyperparameters = loader.load_covariates_and_hyperparameters() print('The following fields have been added as covariates of the climatology model') print(loader.data.keys()) else: covariate_elements, covariate_hyperparameters = [], [] climatology_element = CombinationElement( [SeasonalElement(n_triangulation_divisions=seasonal_setup['n_triangulation_divisions'], n_harmonics=seasonal_setup['n_harmonics'], include_local_mean=True), GrandMeanElement()]+covariate_elements) climatology_hyperparameters = CombinationHyperparameters( [SeasonalHyperparameters(n_spatial_components=seasonal_setup['n_spatial_components'], common_log_sigma=numpy.log(seasonal_setup['amplitude']), common_log_rho=numpy.log(numpy.radians(seasonal_setup['space_length_scale']))), CovariateHyperparameters(numpy.log(grandmean_amplitude))] + covariate_hyperparameters ) climatology_component = SpaceTimeComponent(ComponentStorage_InMemory(climatology_element, climatology_hyperparameters), SpaceTimeComponentSolutionStorage_InMemory(), compute_uncertainties=True, method='APPROXIMATED', compute_sample=True, sample_size=definitions.GLOBAL_SAMPLE_SHAPE[3]) # LARGE SCALE (kronecker product) COMPONENT: combining large scale trends with bias terms accounting for homogeneization effects if args.land_biases: bias_element, bias_hyperparameters = [InsituLandBiasElement(BREAKPOINTS_FILE)], [CovariateHyperparameters(numpy.log(bias_amplitude))] print('Adding bias terms for insitu land homogenization') else: bias_element, bias_hyperparameters = [], [] large_scale_element = CombinationElement( [SpaceTimeKroneckerElement(n_triangulation_divisions=spacetime_setup['n_triangulation_divisions'], alpha=spacetime_setup['alpha'], starttime=spacetime_setup['starttime'], endtime=spacetime_setup['endtime'], n_nodes=spacetime_setup['n_nodes'], overlap_factor=spacetime_setup['overlap_factor'], H=spacetime_setup['H'])] + bias_element) large_scale_hyperparameters = CombinationHyperparameters( [SpaceTimeSPDEHyperparameters(space_log_sigma=numpy.log(spacetime_setup['amplitude']), space_log_rho=numpy.log(numpy.radians(spacetime_setup['space_lenght_scale'])), time_log_rho=numpy.log(spacetime_setup['time_length_scale']))] + bias_hyperparameters) large_scale_component = SpaceTimeComponent(ComponentStorage_InMemory(large_scale_element, large_scale_hyperparameters), SpaceTimeComponentSolutionStorage_InMemory(), compute_uncertainties=True, method='APPROXIMATED', compute_sample=True, sample_size=definitions.GLOBAL_SAMPLE_SHAPE[3]) # LOCAL COMPONENT: combining local scale variations with global satellite bias terms if args.global_biases: bias_elements = [BiasElement(groupname, 1) for groupname in GLOBAL_BIASES_GROUP_LIST] bias_hyperparameters = [CovariateHyperparameters(numpy.log(globalbias_amplitude)) for index in range(len(GLOBAL_BIASES_GROUP_LIST))] print('Adding global bias terms for all the surfaces') else: bias_elements, bias_hyperparameters = [], [] local_scale_element = CombinationElement([LocalElement(n_triangulation_divisions=local_setup['n_triangulation_divisions'])] + bias_elements) local_scale_hyperparameters = CombinationHyperparameters([LocalHyperparameters(log_sigma=numpy.log(local_setup['amplitude']), log_rho=numpy.log(numpy.radians(local_setup['space_length_scale'])))] + bias_hyperparameters) local_component = SpatialComponent(ComponentStorage_InMemory(local_scale_element, local_scale_hyperparameters), SpatialComponentSolutionStorage_InMemory(), compute_uncertainties=True, method='APPROXIMATED', compute_sample=True, sample_size=definitions.GLOBAL_SAMPLE_SHAPE[3]) # Analysis system using the specified components, for the Tmean observable print 'Analysing inputs' analysis_system = AnalysisSystem( [ climatology_component, large_scale_component, local_component ], ObservationSource.TMEAN) # Object to load raw binary inputs at time indices inputloaders = [ AnalysisSystemInputLoaderRawBinary_Sources(basepath, source, time_indices) for source in sources ] for iteration in range(args.n_iterations): message = 'Iteration {}'.format(iteration) print(message) # Update with data analysis_system.update(inputloaders, time_indices) print 'Computing outputs' # Produce an output for each time index for time_index in time_indices: # Get date for output outputdate = inputloaders[0].datetime_at_time_index(time_index) print 'Evaluating output grid: ', outputdate #Configure output grid outputstructure = OutputRectilinearGridStructure( time_index, outputdate, latitudes=numpy.linspace(-90.+definitions.GLOBAL_FIELD_RESOLUTION/2., 90.-definitions.GLOBAL_FIELD_RESOLUTION/2., num=definitions.GLOBAL_FIELD_SHAPE[1]), longitudes=numpy.linspace(-180.+definitions.GLOBAL_FIELD_RESOLUTION/2., 180.-definitions.GLOBAL_FIELD_RESOLUTION/2., num=definitions.GLOBAL_FIELD_SHAPE[2])) # Evaluate expected value at these locations for field in ['MAP', 'post_STD']: print 'Evaluating: ',field result_expected_value = analysis_system.evaluate_expected_value('MAP', outputstructure, 'GRID_CELL_AREA_AVERAGE', [1,1], 1000) result_expected_uncertainties = analysis_system.evaluate_expected_value('post_STD', outputstructure, 'GRID_CELL_AREA_AVERAGE', [1,1], 1000) print 'Evaluating: climatology fraction' climatology_fraction = analysis_system.evaluate_climatology_fraction(outputstructure, [1,1], 1000) print 'Evaluating: the sample' sample = analysis_system.evaluate_projected_sample(outputstructure) # Make output filename pathname = 'eustace_example_output_{0:04d}{1:02d}{2:02d}.nc'.format(outputdate.year, outputdate.month, outputdate.day) pathname = os.path.join(args.outpath, pathname) print 'Saving: ', pathname # Save results filebuilder = FileBuilderGlobalField( pathname, time_index, 'Infilling Example', 'UNVERSIONED', definitions.TAS.name, '', 'Example data only', 'eustace.analysis.advanced_standard.examples.example_eustace_few_days', '') filebuilder.add_global_field(definitions.TAS, result_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.add_global_field(definitions.TASUNCERTAINTY, result_expected_uncertainties.reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.add_global_field(definitions.TAS_CLIMATOLOGY_FRACTION, climatology_fraction.reshape(definitions.GLOBAL_FIELD_SHAPE)) for index in range(definitions.GLOBAL_SAMPLE_SHAPE[3]): variable = copy.deepcopy(definitions.TASENSEMBLE) variable.name = variable.name + '_' + str(index) selected_sample = sample[:,index].ravel()+result_expected_value filebuilder.add_global_field(variable, selected_sample.reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.save_and_close() print 'Complete'
def test_mini_world_noiseless(self): number_of_simulated_time_steps = 1 # Build system element = SeasonalElement(n_triangulation_divisions=3, n_harmonics=5, include_local_mean=True) hyperparameters = SeasonalHyperparameters(n_spatial_components=6, common_log_sigma=0.0, common_log_rho=0.0) component = SpaceTimeComponent( ComponentStorage_InMemory(element, hyperparameters), SpaceTimeComponentSolutionStorage_InMemory()) analysis_system = AnalysisSystem([component], ObservationSource.TMEAN, log=StringIO()) # use fixed locations from icosahedron fixed_locations = cartesian_to_polar2d( MeshIcosahedronSubdivision.build(3).points) # random measurement at each location numpy.random.seed(8976) field_basis = numpy.random.randn(fixed_locations.shape[0]) #print(field_basis.shape) #time_basis = numpy.array(harmonics_list) # some time function that varies over a year #decimal_years = numpy.array([datetime_to_decimal_year(epoch_plus_days(step)) for step in range(number_of_simulated_time_steps)]) time_basis = numpy.cos( numpy.linspace(0.1, 1.75 * numpy.pi, number_of_simulated_time_steps)) # kronecker product of the two #print(numpy.expand_dims(time_basis, 1)) measurement = numpy.kron(field_basis, numpy.expand_dims( time_basis, 1)) #numpy.expand_dims(time_basis, 1)) #print(measurement.shape) # Simulated inputs simulated_input_loader = SimulatedInputLoader(fixed_locations, measurement, 0.0001) # Simulate evaluation of this time index simulated_time_indices = range(number_of_simulated_time_steps) # Iterate for iteration in range(5): analysis_system.update([simulated_input_loader], simulated_time_indices) # Get all results result = numpy.zeros(measurement.shape) for t in range(number_of_simulated_time_steps): result[t, :] = analysis_system.evaluate_expected_value( 'MAP', SimulatedObservationStructure(t, fixed_locations, None, None), flag='POINTWISE') # Should be very close to original because specified noise is low numpy.testing.assert_almost_equal(result, measurement) max_disparity = (numpy.abs(result - measurement)).ravel().max() self.assertTrue(max_disparity < 1E-5) # test output gridding, pointwise limit outputstructure = OutputRectilinearGridStructure( 2, epoch_plus_days(2), latitudes=numpy.linspace(-60., 60., num=5), longitudes=numpy.linspace(-90., 90, num=10)) pointwise_result = analysis_system.evaluate_expected_value( 'MAP', outputstructure, 'POINTWISE') pointwise_limit_result = analysis_system.evaluate_expected_value( 'MAP', outputstructure, 'GRID_CELL_AREA_AVERAGE', [1, 1], 10) numpy.testing.assert_array_almost_equal(pointwise_result, pointwise_limit_result)
def main(): print 'EUSTACE example using HadCRUT4 monthly data' # Input data path input_basepath = os.path.join(WORKSPACE_PATH, 'data/incoming/HadCRUT4.5.0.0') # Input filenames input_filenames = [ 'hadcrut4_median_netcdf.nc', 'hadcrut4_uncorrelated_supplementary.nc', 'hadcrut4_blended_uncorrelated.nc' ] # Months to process time_indices = range(2) # Climatology component climatology_component = SpaceTimeComponent(ComponentStorage_InMemory(SeasonalElement(n_triangulation_divisions=5, n_harmonics=5, include_local_mean=True), SeasonalHyperparameters(n_spatial_components=6, common_log_sigma=1.0, common_log_rho=0.0)), SpaceTimeComponentSolutionStorage_InMemory()) # Number of factors for large scale (factor analysis) component and initial hyperparameters n_factors = 5 factors = [ ] factor_hyperparameters = [ ] for factor_index in range(n_factors): factor_hyperparameters.append( SpaceTimeSPDEHyperparameters( space_log_sigma=0.0, space_log_rho=numpy.log(10.0 * numpy.pi/180 + 25.0 * numpy.pi/180 *(n_factors - factor_index) / n_factors), time_log_rho=numpy.log(1/12.0 + 6/12.0*(n_factors - factor_index) / n_factors)) ) factors.append( SpaceTimeFactorElement(n_triangulation_divisions=5, alpha=2, starttime=0, endtime=36, overlap_factor=2.5, H=1) ) # Large scale (factor analysis) component large_scale_component = SpaceTimeComponent(ComponentStorage_InMemory(CombinationElement(factors), CombinationHyperparameters(factor_hyperparameters)), SpaceTimeComponentSolutionStorage_InMemory()) # Local component local_component = SpatialComponent(ComponentStorage_InMemory(LocalElement(n_triangulation_divisions=4), LocalHyperparameters(log_sigma=0.0, log_rho=numpy.log(10.0 * numpy.pi/180))), SpatialComponentSolutionStorage_InMemory()) print 'Analysing inputs' # Analysis system using the specified components, for the Tmean observable analysis_system = AnalysisSystem( [ climatology_component, large_scale_component, local_component ], ObservationSource.TMEAN) # Make filelist input_filelist = [ os.path.join(input_basepath, filename) for filename in input_filenames ] # Object to load HadCRUT4 inputs at time indices inputloader = AnalysisSystemInputLoaderHadCRUT4(input_filelist) # Update with data analysis_system.update([ inputloader ], time_indices) print 'Computing outputs' # Produce an output for each time index for time_index in time_indices: # Make output filename outputdate = inputloader.datetime_at_time_index(time_index) pathname = 'example_output_{0:04d}{1:02d}.nc'.format(outputdate.year, outputdate.month) print 'Saving: ', pathname # Configure output grid outputstructure = OutputRectilinearGridStructure( time_index, outputdate, latitudes=numpy.linspace(-87.5, 87.5, num=36), longitudes=numpy.linspace(-177.5, 177.5, num=72)) # Evaluate expected value at these locations result_expected_value = analysis_system.evaluate_expected_value(outputstructure) # Save results filebuilder = FileBuilderHadCRUT4ExampleOutput(pathname, outputstructure) filebuilder.add_global_field(TAS_ANOMALY, result_expected_value.reshape(1,36,72)) filebuilder.save_and_close() print 'Complete'
def test_init(self): h = SeasonalHyperparameters(n_spatial_components=3, common_log_sigma=1.1, common_log_rho=1.2) numpy.testing.assert_equal(h.harmonics, [1.1, 1.2, 1.1, 1.2, 1.1, 1.2])
def __init__(self, covariates_descriptor): if covariates_descriptor is not None: loader = LoadCovariateElement(covariates_descriptor) loader.check_keys() covariate_elements, covariate_hyperparameters = loader.load_covariates_and_hyperparameters( ) print( 'The following fields have been added as covariates of the climatology model' ) print(loader.data.keys()) else: covariate_elements, covariate_hyperparameters = [], [] setup = MovingClimatologySetup() model_elements = CombinationElement([ AnnualKroneckerElement( setup.seasonal_spline_settings.n_triangulation_divisions, setup .seasonal_spline_settings.alpha, setup.seasonal_spline_settings .starttime, setup.seasonal_spline_settings.endtime, setup.seasonal_spline_settings.n_nodes, setup.seasonal_spline_settings.overlap_factor, setup.seasonal_spline_settings.H, setup.seasonal_spline_settings.wrap_dimensions), #SeasonalElement(setup.seasonal_settings.n_triangulation_divisions, #setup.seasonal_settings.n_harmonics, #include_local_mean=setup.seasonal_settings.include_local_mean), GrandMeanElement(), LatitudeSplineElement( setup.latitude_settings.alpha, setup.latitude_settings.n_nodes, setup.latitude_settings.overlap_factor, setup.latitude_settings.H, ), ] + covariate_elements) seasonal_hyperparameters = SeasonalHyperparameters( setup.seasonal_settings.n_spatial_components, numpy.log(setup.seasonal_settings.amplitude), numpy.log(numpy.radians( setup.seasonal_settings.space_length_scale))) seasonal_spline_hyperparameters = SpaceTimeSPDEHyperparameters( numpy.log(setup.seasonal_spline_settings.amplitude), numpy.log( numpy.radians( setup.seasonal_spline_settings.space_length_scale)), numpy.log(setup.seasonal_spline_settings.time_length_scale)) seasonal_params = zip( numpy.log(setup.seasonal_settings.harmonic_amplitudes), numpy.log( numpy.radians(setup.seasonal_settings.harmonic_length_scales))) seasonal_hyperparameters.set_array( [val for pair in seasonal_params for val in pair]) model_hyperparameters = CombinationHyperparameters([ seasonal_spline_hyperparameters, #seasonal_hyperparameters, CovariateHyperparameters( numpy.log(setup.covariate_settings.grandmean_amplitude)), #SpaceTimeSPDEHyperparameters(numpy.log(setup.slow_settings.amplitude), #numpy.log(numpy.radians(setup.slow_settings.space_length_scale)), #numpy.log(setup.slow_settings.time_length_scale)), LocalHyperparameters( numpy.log(setup.latitude_settings.amplitude), numpy.log(setup.latitude_settings.length_scale)) ] + covariate_hyperparameters) super(ClimatologyDefinition, self).__init__(model_elements, model_hyperparameters)