def test_mini_world_local(self): # Local component local_component = SpatialComponent( ComponentStorage_InMemory( LocalElement(n_triangulation_divisions=1), LocalHyperparameters(log_sigma=0.0, log_rho=numpy.log(1.0))), SpatialComponentSolutionStorage_InMemory(), compute_uncertainties=True, method='APPROXIMATED') # Analysis system using the specified components, for the Tmean observable analysis_system = AnalysisSystem([local_component], ObservationSource.TMEAN, log=StringIO()) # Simulated inputs simulated_input_loader = SimulatedInputLoader() # Simulate evaluation of this time index simulated_time_indices = [0] # Update with data analysis_system.update([simulated_input_loader], simulated_time_indices) # Check state vector directly statevector = analysis_system.components[ 0].solutionstorage.partial_state_read(0).ravel() # These are the nodes where observations were put (see SimulatedObservationSource above) # - check they correspond to within 3 times the stated noise level self.assertAlmostEqual(20.0, statevector[12], delta=0.3) self.assertAlmostEqual(-15.0, statevector[17], delta=0.3) self.assertAlmostEqual(5.0, statevector[41], delta=0.3) # Also check entire state vector within outer bounds set by obs self.assertTrue(all(statevector < 20.0)) self.assertTrue(all(statevector > -15.0)) # And check output corresponds too # (evaluate result on output structure same as input) simulated_output_structure = SimulatedObservationStructure(0) result = analysis_system.evaluate_expected_value( 'MAP', simulated_output_structure, flag='POINTWISE') numpy.testing.assert_almost_equal(statevector[[12, 17, 41]], result) # test output gridding, pointwise limit outputstructure = OutputRectilinearGridStructure( 2, epoch_plus_days(2), latitudes=numpy.linspace(-89.875, 89.875, num=10), longitudes=numpy.linspace(-179.875, 179.875, num=20)) pointwise_result = analysis_system.evaluate_expected_value( 'MAP', outputstructure, 'POINTWISE') pointwise_limit_result = analysis_system.evaluate_expected_value( 'MAP', outputstructure, 'GRID_CELL_AREA_AVERAGE', [1, 1], 3) numpy.testing.assert_array_almost_equal(pointwise_result, pointwise_limit_result)
def test_epoch_plus_days(self): result = epoch_plus_days(60573.647766204) self.assertTrue(isinstance(result, datetime)) self.assertEqual(2015, result.year) self.assertEqual(11, result.month) self.assertEqual(5, result.day) self.assertEqual(15, result.hour) self.assertEqual(32, result.minute) self.assertEqual(47, result.second)
def filename_mobile_locations(self, observable, daynumber): """Generate individual filename for locations.""" patterns = self.filename_patterns_mobile_locations(observable) return os.path.join(*[ datetime_numeric.build_from_pattern(pattern, epoch_plus_days(daynumber)) for pattern in patterns ])
def filename_observations(self, observable, daynumber): """Generate individual filename for data on specified day for the specified observable.""" patterns = self.filename_patterns_observations(observable) return os.path.join(*[ datetime_numeric.build_from_pattern(pattern, epoch_plus_days(daynumber)) for pattern in patterns ])
def filename_from_patterns(self, patterns, key): if type(key) is int: t = epoch.epoch_plus_days(key) paths = [ datetime_numeric.build_from_pattern(pattern, t) for pattern in patterns ] name = os.path.join(*paths) return name
def specify(manager, options): """Return the list of operations and expected output given the input catalogue.""" # Require many local variables here # pylint: disable=too-many-locals # parse input option list parser = argparse.ArgumentParser() parser.add_argument('--start', required=True) parser.add_argument('--end', required=True) args = parser.parse_args(options) # process a whole number of days startday = int(days_since_epoch(datetime_numeric.parse(args.start))) endday = int(days_since_epoch(datetime_numeric.parse(args.end))) # find items in catalogue grouped by day lstinput = manager.references_groupbyday(INPUTNAME, subsetindex=0) auxinput = manager.references_groupbyday(INPUTNAME, subsetindex=1) # the main output dataset dataset = manager.newdataset() # list of subsets (one for each command pattern) subsets = [dataset.newsubset([spec.outputpattern]) for spec in SUBSETSPECS] # iterate over subsets for subsetindex, subset in enumerate(subsets): # process each day for dayindex in range(startday, endday + 1): # convert to datetime object for formatting etc day = epoch_plus_days(dayindex) # loop over LST files for this day inputs = [] for lstreference in lstinput[dayindex]: # find AUX file matching the LST file lsttime = manager.match(lstreference).time print 'day {0} time {1}'.format(dayindex, lsttime) auxreference = next( reference for reference in auxinput[dayindex] if manager.match(reference).time == lsttime) # append pair inputs.extend([lstreference, auxreference]) # build output filename outputs = [subset.newfiletime(day)] # Append this operation to make it dataset.newoperation(inputs, outputs, SUBSETSPECS[subsetindex])
def update_time_index(self, time_index, keep_design=False): """Update time time information for the projector Set keep_design to True to allow reuse of a projection for a different time_index, e.g. for a EUSTACE analysis local component for a different time step. """ self.time_index = time_index self.corresponding_datetime = epoch_plus_days(time_index) if not keep_design: self.design_matrix = None print "New time index:", self.time_index
def __init__(self, centre_latitudes, centre_longitudes, grid_resolution, time_index, cell_sampling, blocking): self.outputstructure = None # TODO Allow for an output struture with no spatial location - pure covariate. self.grid_resolution = grid_resolution self.centre_latitudes = centre_latitudes self.centre_longitudes = centre_longitudes self.time_index = time_index self.corresponding_datetime = epoch_plus_days(time_index) self.cell_sampling = cell_sampling self.blocking = blocking self.component = None self.state_solution = None self.state_samples = None self.prior_samples = None self.design_matrix = None
def build_pathname(self, basepath, daynumber): """ Build whole pathname from basepath and daynumber. """ # Get pattern patterns = self.build_pathname_patterns() # Convert daynumber to datetime object fieldtime = epoch_plus_days(daynumber) # initialise pathname to basepath pathname = basepath # Fill date fields and append for pattern in patterns: pathname = os.path.join( pathname, datetime_numeric.build_from_pattern(pattern, fieldtime)) return pathname
def datetime_at_time_index(self, time_index): """Express a time index as days since epoch as a datetime""" print time_index return epoch_plus_days(time_index)
def datetime_at_time_index(self, time_index): """Use EUSTACE epoch.""" return epoch_plus_days(time_index)
def time_datetime(self): return epoch_plus_days(self.t)
def test_mini_world_altitude_with_latitude(self): """Testing using altitude as a covariate""" # GENERATING OBSERVATIONS # Simulated locations: they will exactly sits on the grid points of the covariate datafile DEM = Dataset(self.altitude_datafile) latitude = DEM.variables['lat'][:] longitude = DEM.variables['lon'][:] altitude = DEM.variables['dem'][:] indices = numpy.stack( (numpy.array([1, 3, 5, 7, 8, 9, 10, 11 ]), numpy.array([0, 0, 0, 0, 0, 0, 0, 0])), axis=1) selected_location = [] altitude_observations = [] for couple in indices: selected_location.append([ latitude[couple[0], couple[1]], longitude[couple[0], couple[1]] ]) altitude_observations.append(altitude[couple[0], couple[1]]) DEM.close() locations = numpy.array(selected_location) # Simulated model is y = z + a*cos(2x) + c*cos(4*x) + b*sin(2x) + d*sin(4*x), with z = altitude, x = latitude, a=b=c=d=0 slope = 1e-3 measurement = slope * numpy.array(altitude_observations) # Simulated errors uncorrelatederror = 0.1 * numpy.ones(measurement.shape) # Simulated inputs simulated_input_loader = SimulatedInputLoader(locations, measurement, uncorrelatederror) # Simulate evaluation of this time index simulated_time_indices = [0] # GENERATING THE MODEL # Local component geography_covariate_element = GeographyBasedElement( self.altitude_datafile, 'lat', 'lon', 'dem', 1.0) geography_covariate_element.load() combined_element = CombinationElement( [geography_covariate_element, LatitudeHarmonicsElement()]) combined_hyperparamters = CombinationHyperparameters([ CovariateHyperparameters(-0.5 * numpy.log(10.)), CombinationHyperparameters([ CovariateHyperparameters(-0.5 * numpy.log(p)) for p in [10.0, 10.0, 10.0, 10.0] ]) ]) combined_component = SpatialComponent( ComponentStorage_InMemory(combined_element, combined_hyperparamters), SpatialComponentSolutionStorage_InMemory()) # GENERATING THE ANALYSIS # Analysis system using the specified components, for the Tmean observable analysis_system = AnalysisSystem([combined_component], ObservationSource.TMEAN, log=StringIO()) # Update with data analysis_system.update([simulated_input_loader], simulated_time_indices) # Check state vector directly statevector = analysis_system.components[ 0].solutionstorage.partial_state_read(0).ravel() # These are the nodes where observations were put (see SimulatedObservationSource above) # - check they correspond to within 3 times the stated noise level self.assertAlmostEqual(slope, statevector[0], delta=0.3) self.assertAlmostEqual(0., statevector[1], delta=0.3) self.assertAlmostEqual(0., statevector[2], delta=0.3) self.assertAlmostEqual(0., statevector[3], delta=0.3) self.assertAlmostEqual(0., statevector[4], delta=0.3) # And check output corresponds too # (evaluate result on output structure same as input) simulated_output_structure = SimulatedObservationStructure( 0, locations, None, None) result = analysis_system.evaluate_expected_value( 'MAP', simulated_output_structure, flag='POINTWISE') expected = statevector[0]*numpy.array(altitude_observations)\ + statevector[1]*LatitudeFunction(numpy.cos, 2.0).compute(locations[:,0]).ravel()\ + statevector[2]*LatitudeFunction(numpy.sin, 2.0).compute(locations[:,0]).ravel()\ + statevector[3]*LatitudeFunction(numpy.cos, 4.0).compute(locations[:,0]).ravel()\ + statevector[4]*LatitudeFunction(numpy.sin, 2.0).compute(locations[:,0]).ravel() numpy.testing.assert_almost_equal(expected, result) # test output gridding, pointwise limit outputstructure = OutputRectilinearGridStructure( 2, epoch_plus_days(2), latitudes=numpy.linspace(-60., 60., num=5), longitudes=numpy.linspace(-90., 90, num=10)) pointwise_result = analysis_system.evaluate_expected_value( 'MAP', outputstructure, 'POINTWISE') pointwise_limit_result = analysis_system.evaluate_expected_value( 'MAP', outputstructure, 'GRID_CELL_AREA_AVERAGE', [1, 1], 10) numpy.testing.assert_array_almost_equal(pointwise_result, pointwise_limit_result)
def datetime_at_time_index(self, time_index): return epoch_plus_days(time_index)
def test_mini_world_noiseless(self): number_of_simulated_time_steps = 1 # Build system element = SeasonalElement(n_triangulation_divisions=3, n_harmonics=5, include_local_mean=True) hyperparameters = SeasonalHyperparameters(n_spatial_components=6, common_log_sigma=0.0, common_log_rho=0.0) component = SpaceTimeComponent( ComponentStorage_InMemory(element, hyperparameters), SpaceTimeComponentSolutionStorage_InMemory()) analysis_system = AnalysisSystem([component], ObservationSource.TMEAN, log=StringIO()) # use fixed locations from icosahedron fixed_locations = cartesian_to_polar2d( MeshIcosahedronSubdivision.build(3).points) # random measurement at each location numpy.random.seed(8976) field_basis = numpy.random.randn(fixed_locations.shape[0]) #print(field_basis.shape) #time_basis = numpy.array(harmonics_list) # some time function that varies over a year #decimal_years = numpy.array([datetime_to_decimal_year(epoch_plus_days(step)) for step in range(number_of_simulated_time_steps)]) time_basis = numpy.cos( numpy.linspace(0.1, 1.75 * numpy.pi, number_of_simulated_time_steps)) # kronecker product of the two #print(numpy.expand_dims(time_basis, 1)) measurement = numpy.kron(field_basis, numpy.expand_dims( time_basis, 1)) #numpy.expand_dims(time_basis, 1)) #print(measurement.shape) # Simulated inputs simulated_input_loader = SimulatedInputLoader(fixed_locations, measurement, 0.0001) # Simulate evaluation of this time index simulated_time_indices = range(number_of_simulated_time_steps) # Iterate for iteration in range(5): analysis_system.update([simulated_input_loader], simulated_time_indices) # Get all results result = numpy.zeros(measurement.shape) for t in range(number_of_simulated_time_steps): result[t, :] = analysis_system.evaluate_expected_value( 'MAP', SimulatedObservationStructure(t, fixed_locations, None, None), flag='POINTWISE') # Should be very close to original because specified noise is low numpy.testing.assert_almost_equal(result, measurement) max_disparity = (numpy.abs(result - measurement)).ravel().max() self.assertTrue(max_disparity < 1E-5) # test output gridding, pointwise limit outputstructure = OutputRectilinearGridStructure( 2, epoch_plus_days(2), latitudes=numpy.linspace(-60., 60., num=5), longitudes=numpy.linspace(-90., 90, num=10)) pointwise_result = analysis_system.evaluate_expected_value( 'MAP', outputstructure, 'POINTWISE') pointwise_limit_result = analysis_system.evaluate_expected_value( 'MAP', outputstructure, 'GRID_CELL_AREA_AVERAGE', [1, 1], 10) numpy.testing.assert_array_almost_equal(pointwise_result, pointwise_limit_result)