def output_grid(storage_climatology, storage_large_scale, storage_local, outputfile, processdate, time_index, covariates_descriptor, insitu_biases, breakpoints_file, global_biases, global_biases_group_list, compute_uncertainties, method, compute_sample, sample_size): print 'VERSION: {0}'.format(get_revision_id_for_module(eustace)) # Build analysis system analysissystem = AnalysisSystem_EUSTACE(storage_climatology, storage_large_scale, storage_local, covariates_descriptor, insitu_biases, breakpoints_file, global_biases, global_biases_group_list, compute_uncertainties, method, compute_sample, sample_size) #Configure output grid outputstructure = OutputRectilinearGridStructure( time_index, processdate, latitudes=numpy.linspace(-90.+definitions.GLOBAL_FIELD_RESOLUTION/2., 90.-definitions.GLOBAL_FIELD_RESOLUTION/2., num=definitions.GLOBAL_FIELD_SHAPE[1]), longitudes=numpy.linspace(-180.+definitions.GLOBAL_FIELD_RESOLUTION/2., 180.-definitions.GLOBAL_FIELD_RESOLUTION/2., num=definitions.GLOBAL_FIELD_SHAPE[2])) # Evaluate expected value at these locations for field in ['MAP', 'post_STD']: print 'Evaluating: ',field result_expected_value = analysissystem.evaluate_expected_value('MAP', outputstructure, 'GRID_CELL_AREA_AVERAGE', [1,1], 1000) result_expected_uncertainties = analysissystem.evaluate_expected_value('post_STD', outputstructure, 'GRID_CELL_AREA_AVERAGE', [1,1], 1000) print 'Evaluating: climatology fraction' climatology_fraction = analysissystem.evaluate_climatology_fraction(outputstructure, [1,1], 1000) print 'Evaluating: the sample' sample = analysissystem.evaluate_projected_sample(outputstructure) # Save results filebuilder = FileBuilderGlobalField( outputfile, eustace.timeutils.epoch.days_since_epoch(processdate), 'Infilling Example', get_revision_id_for_module(eustace), definitions.TAS.name, '', 'Example data only', __name__, '') filebuilder.add_global_field(definitions.TAS, result_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.add_global_field(definitions.TASUNCERTAINTY, result_expected_uncertainties.reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.add_global_field(definitions.TAS_CLIMATOLOGY_FRACTION, climatology_fraction.reshape(definitions.GLOBAL_FIELD_SHAPE)) for index in range(definitions.GLOBAL_SAMPLE_SHAPE[3]): variable = copy.deepcopy(definitions.TASENSEMBLE) variable.name = variable.name + '_' + str(index) selected_sample = sample[:,index].ravel()+result_expected_value filebuilder.add_global_field(variable, selected_sample.reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.save_and_close()
def versionlist(): """Get list of version identifiers for tools used.""" tools = [] if RECORD_JASMIN_TOOLCHAIN: tools.append(commandline(['rpm', '-q', 'jasmin-common-vm'])) tools.append('eustace-' + get_revision_id_for_module(eustace)) return tools
def output_grid_component(storage_climatology, storage_large_scale, storage_local, outputfile, processdate, time_index, covariates_descriptor, insitu_biases, breakpoints_file, global_biases, global_biases_group_list, compute_uncertainties, method): print 'VERSION: {0}'.format(get_revision_id_for_module(eustace)) # Build analysis system analysissystem = AnalysisSystem_EUSTACE(storage_climatology, storage_large_scale, storage_local, covariates_descriptor, insitu_biases, breakpoints_file, global_biases, global_biases_group_list, compute_uncertainties, method) # Configure output grid outputstructure = OutputRectilinearGridStructure( time_index, processdate, latitudes=numpy.linspace(-89.875, 89.875, num=definitions.GLOBAL_FIELD_SHAPE[1]), longitudes=numpy.linspace(-179.875, 179.875, num=definitions.GLOBAL_FIELD_SHAPE[2])) # Evaluate expected value at these locations for field in ['MAP', 'post_STD']: print 'Evaluating: ',field result_expected_value = analysissystem.evaluate_expected_value('MAP', outputstructure, 'GRID_CELL_AREA_AVERAGE', [1,1], 1000) result_expected_uncertainties = analysissystem.evaluate_expected_value('post_STD', outputstructure, 'GRID_CELL_AREA_AVERAGE', [1,1], 1000) # Save results filebuilder = FileBuilderGlobalField( outputfile, eustace.timeutils.epoch.days_since_epoch(processdate), 'Infilling Example', get_revision_id_for_module(eustace), definitions.TAS.name, '', 'Example data only', __name__, '') filebuilder.add_global_field(definitions.TAS, result_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.add_global_field(definitions.TASUNCERTAINTY, result_expected_uncertainties.reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.save_and_close()
def output_homog(storage_climatology, storage_large_scale, storage_local, outputfile, climatologyfile, largescalefile, localfile, processdate, time_index, covariates_descriptor, insitu_biases, breakpoints_file, global_biases, global_biases_group_list, compute_uncertainties, method, compute_sample, sample_size, compute_prior_sample): from eustace.preprocess.fileio.insitu_land_breakpoints import ObservationBreakPointSourceInsituLand from eustace.analysis.advanced_standard.fileio.output_projector import Projector print 'VERSION: {0}'.format(get_revision_id_for_module(eustace)) # Build analysis system analysissystem = AnalysisSystem_EUSTACE( storage_climatology, storage_large_scale, storage_local, covariates_descriptor, insitu_biases, breakpoints_file, global_biases, global_biases_group_list, compute_uncertainties, method) component = analysissystem.components[1] element = component.storage.element_read() hyperparameters = component.storage.hyperparameters_read() currentstate = component.solutionstorage.state_read() element_states = component.storage.element_read().element_prior( hyperparameters).element_states(currentstate) breakpoints_reader = ObservationBreakPointSourceInsituLand( breakpoints_file) station_locations = breakpoints_reader.observation_location_lookup() #station_indices = breakpoints_reader.observation_location_lookup() breakpoints_reader.dataset.close() for i, subelement in enumerate(element.combination): if isinstance(subelement, BiasElement): mystate = element_states[i] if hasattr(subelement, 'observed_breakpoints'): import pickle outdict = { 'breakpoints': subelement.observed_breakpoints, 'locations': station_locations.T, 'biases': mystate } with open( '/work/scratch/cmorice/advanced_standard/observed_breakpoints.pickle', 'w') as f: pickle.dump(outdict, f)
def save_flag_file(flag_values, processdate, outputfile): """Save the windowing flag information""" print "Saving: ", outputfile # Create directory structure if required if not os.path.exists(os.path.dirname(outputfile)): os.makedirs(os.path.dirname(outputfile)) # Make the QC flag file filebuilder = FileBuilderGlobalField( outputfile, eustace.timeutils.epoch.days_since_epoch(processdate), 'EUSTACE Analysis Flags', get_revision_id_for_module(eustace), definitions.TAS.name, '', 'Provisional output', __name__, '') # flag flag definition tas_qc_definition = OutputVariable( name='tas_qc', dtype=FLAG_TYPE, fill_value=default_fillvals[TYPE_NAME], standard_name='air_temperature status_flag', long_name='Quality control flags', #valid_range=numpy.array([FLAG_TYPE(0), FLAG_TYPE(1)<<FLAG_MAX_N], FLAG_TYPE), flag_masks=numpy.array([ FLAG_TYPE(1)<<n for n in range(FLAG_MAX_USED)], FLAG_TYPE), flag_meanings=' '.join(FLAG_MEANINGS)) # Combine flag information and populate the output file filebuilder.add_global_field( tas_qc_definition, flag_values.reshape(definitions.GLOBAL_FIELD_SHAPE) ) # Close the netCDF4 dataset filebuilder.save_and_close()
def apply_mask(analysis_directory, iteration, mask_directory, mask_directory2, flags_to_mask, processdate, output_directory): """Apply QC mask to analysis""" datestring = '%04i%02i%02i' % (processdate.year, processdate.month, processdate.day) # get mask defined by provided options qc_flags = load_flags(mask_directory, iteration, datestring).data[:,:,:] if mask_directory2 is not None: qc_flags = qc_flags | load_flags(mask_directory2, iteration, datestring).data[:,:,:] # set a new marine flag with a more flexible definition of marine (95% water rather than 100%) qc_flags = reset_marine_flag(qc_flags) combined_mask = compute_mask(qc_flags, flags_to_mask) # get analysis analysis, uncertainty, influence, ensemble_members = load_analysis(analysis_directory, iteration, datestring) # build masked output product file #outputfile = output_directory+'/eustace_analysis_'+str(iteration)+'/'+datestring[:4]+'/eustace_analysis_'+str(iteration)+'_masked_'+datestring+'.nc' #format as variable_collection_framework_realization_YYYYmmdd.nc so tas_global_eustace_0_YYYYmmdd.nc if iteration == 4: version_string = "R001400" rundate = '20190326' elif iteration == 9: version_string = "R001413" rundate = '20190405' else: print "unrecognised version" outputfile = output_directory+'/'+version_string+'/'+rundate+'/global/'+datestring[:4]+'/tas_global_eustace_0_'+datestring+'.nc' print outputfile filebuilder = FileBuilderGlobalField( outputfile, eustace.timeutils.epoch.days_since_epoch(processdate), 'EUSTACE Analysis', get_revision_id_for_module(eustace), definitions.TAS.name, 'Met Office', '', '', '') numpy.squeeze(analysis.data[:,:,:]) numpy.squeeze(uncertainty.data[:,:,:]) numpy.squeeze(influence.data[:,:,:]) filebuilder.add_global_field(definitions.TAS, numpy.ma.masked_where( combined_mask, analysis.data[:,:,:] ).reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.add_global_field(definitions.TASUNCERTAINTY, numpy.ma.masked_where( combined_mask, uncertainty.data[:,:,:] ).reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.add_global_field(definitions.TAS_OBSERVATION_INFLUENCE, numpy.ma.masked_where( combined_mask, influence .data[:,:,:] ).reshape(definitions.GLOBAL_FIELD_SHAPE)) # sort ensemble members and apply mask ensemble_members = sorted(ensemble_members, key=lambda item: int(item.var_name.split('_')[-1])) for index, ensemble_member in enumerate(ensemble_members): variable = copy.deepcopy(definitions.TASENSEMBLE) variable.name = variable.name + '_' + str(index) filebuilder.add_global_field(variable, numpy.ma.masked_where( combined_mask, ensemble_member.data[:,:,:] ).reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.save_and_close() # climatology only output analysis, uncertainty, influence = load_climatology(analysis_directory, iteration, datestring) climatologyfile = output_directory+'/eustace_climatology_component_'+str(iteration)+'/'+datestring[:4]+'/eustace_climatology_component_'+str(iteration)+'_masked_'+datestring+'.nc' filebuilder = FileBuilderGlobalField( climatologyfile, eustace.timeutils.epoch.days_since_epoch(processdate), 'EUSTACE Analysis', get_revision_id_for_module(eustace), definitions.TAS.name, '', 'Provisional component output - climatology', __name__, '') numpy.squeeze(analysis.data[:,:,:]) numpy.squeeze(uncertainty.data[:,:,:]) numpy.squeeze(influence.data[:,:,:]) filebuilder.add_global_field(definitions.TAS, numpy.ma.masked_where( combined_mask, analysis.data[:,:,:] ).reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.add_global_field(definitions.TASUNCERTAINTY, numpy.ma.masked_where( combined_mask, uncertainty.data[:,:,:] ).reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.add_global_field(definitions.TAS_OBSERVATION_INFLUENCE, numpy.ma.masked_where( combined_mask, influence .data[:,:,:] ).reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.save_and_close() # large scale only output analysis, uncertainty, influence = load_large_scale(analysis_directory, iteration, datestring) largescalefile = output_directory+'/eustace_large_scale_component_'+str(iteration)+'/'+datestring[:4]+'/eustace_large_scale_component_'+str(iteration)+'_masked_'+datestring+'.nc' filebuilder = FileBuilderGlobalField( largescalefile, eustace.timeutils.epoch.days_since_epoch(processdate), 'EUSTACE Analysis', get_revision_id_for_module(eustace), definitions.TAS.name, '', 'Provisional component output - large scale', __name__, '') numpy.squeeze(analysis.data[:,:,:]) numpy.squeeze(uncertainty.data[:,:,:]) numpy.squeeze(influence.data[:,:,:]) filebuilder.add_global_field(definitions.TASPERTURBATION, numpy.ma.masked_where( combined_mask, analysis.data[:,:,:] ).reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.add_global_field(definitions.TASUNCERTAINTY, numpy.ma.masked_where( combined_mask, uncertainty.data[:,:,:] ).reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.add_global_field(definitions.TAS_OBSERVATION_INFLUENCE, numpy.ma.masked_where( combined_mask, influence .data[:,:,:] ).reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.save_and_close() # local only output analysis, uncertainty, influence = load_local(analysis_directory, iteration, datestring) localfile = output_directory+'/eustace_local_component_'+str(iteration)+'/'+datestring[:4]+'/eustace_local_component_'+str(iteration)+'_masked_'+datestring+'.nc' filebuilder = FileBuilderGlobalField( localfile, eustace.timeutils.epoch.days_since_epoch(processdate), 'EUSTACE Analysis', get_revision_id_for_module(eustace), definitions.TAS.name, '', 'Provisional component output - local', __name__, '') numpy.squeeze(analysis.data[:,:,:]) numpy.squeeze(uncertainty.data[:,:,:]) numpy.squeeze(influence.data[:,:,:]) filebuilder.add_global_field(definitions.TASPERTURBATION, numpy.ma.masked_where( combined_mask, analysis.data[:,:,:] ).reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.add_global_field(definitions.TASUNCERTAINTY, numpy.ma.masked_where( combined_mask, uncertainty.data[:,:,:] ).reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.add_global_field(definitions.TAS_OBSERVATION_INFLUENCE, numpy.ma.masked_where( combined_mask, influence .data[:,:,:] ).reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.save_and_close()
def latent_variable_flag(input_directory, output_directory, iteration, processing_dates): # manually setup the analysis model for the R1413 run - Warning: the eustace svn revision must be correct for the global bias model interpretation to that run analysis storage_climatology = SpaceTimeComponentSolutionStorageBatched_Files( statefilename_read='/work/scratch/cmorice/advanced_standard/climatology_solution_9/climatology_solution_9.pickle', sample_filename_read='/work/scratch/cmorice/advanced_standard/climatology_solution_sample_9/climatology_solution_sample_9.pickle', prior_sample_filename_read='/work/scratch/cmorice/advanced_standard/climatology_solution_prior_sample_9/climatology_solution_prior_sample_9.pickle', keep_in_memory = True ) storage_large_scale = SpaceTimeComponentSolutionStorageBatched_Files( statefilename_read='/work/scratch/cmorice/advanced_standard/large_scale_solution_9/large_scale_solution_9.pickle', sample_filename_read='/work/scratch/cmorice/advanced_standard/large_scale_solution_sample_9/large_scale_solution_sample_9.pickle', prior_sample_filename_read='/work/scratch/cmorice/advanced_standard/large_scale_solution_prior_sample_9/large_scale_solution_prior_sample_9.pickle', keep_in_memory = True ) storage_local = eustace.analysis.advanced_standard.components.storage_files_batch.SpatialComponentSolutionStorageIndexed_Files() covariates_descriptor = "/gws/nopw/j04/eustace/data/internal/climatology_covariates/covariates.json" insitu_biases = True breakpoints_file = "/gws/nopw/j04/eustace/data/internal/D1.7/daily/eustace_stations_global_R001127_daily_status.nc" global_biases = True global_biases_group_list = ["surfaceairmodel_ice_global" , "surfaceairmodel_land_global", "surfaceairmodel_ocean_global"] compute_uncertainties = False method = 'EXACT' compute_sample = False sample_size = definitions.GLOBAL_SAMPLE_SHAPE[3] compute_prior_sample = False print 'VERSION: {0}'.format(get_revision_id_for_module(eustace)) # Build analysis system analysissystem = AnalysisSystem_EUSTACE(storage_climatology, storage_large_scale, storage_local, covariates_descriptor, insitu_biases, breakpoints_file, global_biases, global_biases_group_list, compute_uncertainties, method) grid_resolution = [180. / definitions.GLOBAL_FIELD_SHAPE[1], 360. / definitions.GLOBAL_FIELD_SHAPE[2]] latitudes=numpy.linspace(-90.+grid_resolution[0]/2., 90.-grid_resolution[0]/2, num=definitions.GLOBAL_FIELD_SHAPE[1]) longitudes=numpy.linspace(-180.+grid_resolution[1]/2., 180.-grid_resolution[1]/2, num=definitions.GLOBAL_FIELD_SHAPE[2]) #timebase = TimeBaseDays(eustace.timeutils.epoch.EPOCH) #processdates = [timebase.number_to_datetime(daynumber) for daynumber in time_indices] # get times as understood by the analysis sustem time_indices =[eustace.timeutils.epoch.days_since_epoch(t) for t in processing_dates] cell_sampling = [1, 1] blocking = 10 # thinned set of sample indices for inclusion in output product sample_indices = range(definitions.GLOBAL_SAMPLE_SHAPE[3]) climatology_projector = None large_scale_projector = None local_projector = None for ( inner_index, time_index, processdate ) in zip( range(len(time_indices)), time_indices, processing_dates ): print time_index # initialise flags flag_values = numpy.zeros( definitions.GLOBAL_FIELD_SHAPE[1:], FLAG_TYPE ) # Configure output grid outputstructure = OutputRectilinearGridStructure(time_index, processdate, latitudes=latitudes, longitudes=longitudes) # climatology component print 'Evaluating: climatology' if climatology_projector is None: climatology_projector = Projector(latitudes, longitudes, grid_resolution, time_index, cell_sampling, blocking) climatology_projector.set_component(analysissystem.components[0]) latent_climatology_constraint = evaluate_latent_variable_constraint(climatology_projector) climatology_projector.update_time_index(time_index, keep_design = False) climatology_projector.evaluate_design_matrix() climatology_statistic = evaluate_constraint_statistic(climatology_projector, latent_climatology_constraint, CONSTRAINT_THRESHOLD).reshape(definitions.GLOBAL_FIELD_SHAPE[1:]) flag_values[climatology_statistic] = flag_values[climatology_statistic] | CLIMATOLOGY_LATENT_FLAG # large scale component print 'Evaluating: large-scale' if large_scale_projector is None: large_scale_projector = Projector(latitudes, longitudes, grid_resolution, time_index, cell_sampling, blocking) large_scale_projector.set_component(analysissystem.components[1]) latent_large_scale_constraint = evaluate_latent_variable_constraint(large_scale_projector) large_scale_projector.update_time_index(time_index, keep_design = False) large_scale_projector.evaluate_design_matrix() large_scale_statistic = evaluate_constraint_statistic(large_scale_projector, latent_large_scale_constraint, CONSTRAINT_THRESHOLD).reshape(definitions.GLOBAL_FIELD_SHAPE[1:]) flag_values[large_scale_statistic] = flag_values[large_scale_statistic] | LARGE_SCALE_LATENT_FLAG outputfile = os.path.join(output_directory, '{:04d}'.format(processdate.year), 'eustace_analysis_{:d}_qc_flags_{:04d}{:02d}{:02d}.nc'.format(iteration, processdate.year, processdate.month, processdate.day)) save_flag_file(flag_values, processdate, outputfile)
def early_look_grid_batch( storage_climatology, storage_large_scale, storage_local, outputfiles, time_indices, covariates_descriptor, insitu_biases, breakpoints_file, global_biases, global_biases_group_list, compute_uncertainties, method, compute_sample, sample_size, compute_prior_sample): """Produce 'early look' NetCDF output files without loading or gridding uncertainty information For inspection of analysis output prior to the final gridding step. """ from eustace.analysis.advanced_standard.fileio.output_projector import Projector print 'VERSION: {0}'.format(get_revision_id_for_module(eustace)) # Build analysis system analysissystem = AnalysisSystem_EUSTACE( storage_climatology, storage_large_scale, storage_local, covariates_descriptor, insitu_biases, breakpoints_file, global_biases, global_biases_group_list, compute_uncertainties, method) grid_resolution = [ 180. / definitions.GLOBAL_FIELD_SHAPE[1], 360. / definitions.GLOBAL_FIELD_SHAPE[2] ] latitudes = numpy.linspace(-90. + grid_resolution[0] / 2., 90. - grid_resolution[0] / 2, num=definitions.GLOBAL_FIELD_SHAPE[1]) longitudes = numpy.linspace(-180. + grid_resolution[1] / 2., 180. - grid_resolution[1] / 2, num=definitions.GLOBAL_FIELD_SHAPE[2]) timebase = TimeBaseDays(eustace.timeutils.epoch.EPOCH) processdates = [ timebase.number_to_datetime(daynumber) for daynumber in time_indices ] cell_sampling = [1, 1] blocking = 10 # thinned set of sample indices for inclusion in output product climatology_projector = None large_scale_projector = None local_projector = None for (inner_index, time_index, processdate) in zip(range(len(time_indices)), time_indices, processdates): print time_index # Configure output grid outputstructure = OutputRectilinearGridStructure(time_index, processdate, latitudes=latitudes, longitudes=longitudes) # climatology component print 'Evaluating: climatology' if climatology_projector is None: climatology_projector = Projector(latitudes, longitudes, grid_resolution, time_index, cell_sampling, blocking) climatology_projector.set_component(analysissystem.components[0]) climatology_projector.update_time_index(time_index, keep_design=False) climatology_projector.evaluate_design_matrix() climatology_expected_value = climatology_projector.project_expected_value( ).reshape((-1, 1)) # large scale component print 'Evaluating: large-scale' if large_scale_projector is None: large_scale_projector = Projector(latitudes, longitudes, grid_resolution, time_index, cell_sampling, blocking) large_scale_projector.set_component(analysissystem.components[1]) large_scale_projector.update_time_index(time_index, keep_design=False) large_scale_projector.evaluate_design_matrix() large_scale_expected_value = large_scale_projector.project_expected_value( ).reshape((-1, 1)) # local component - time handling updates state to new time but does not recompute the design matrix print 'Evaluating: local' if local_projector is None: local_projector = Projector(latitudes, longitudes, grid_resolution, time_index, cell_sampling, blocking) local_projector.set_component(analysissystem.components[2]) local_projector.evaluate_design_matrix() else: local_projector.update_time_index(time_index, keep_design=True) local_projector.set_component(analysissystem.components[2], keep_design=True) print analysissystem.components local_expected_value = local_projector.project_expected_value( ).reshape((-1, 1)) # Save results outputfile = outputfiles[inner_index] print outputfile # main merged product output files filebuilder = FileBuilderGlobalField( outputfile, eustace.timeutils.epoch.days_since_epoch(processdate), 'EUSTACE Analysis', get_revision_id_for_module(eustace), definitions.TAS.name, '', 'Provisional output', __name__, '') field_definition_tas = definitions.OutputVariable.from_template( definitions.TEMPLATE_TEMPERATURE, 'tas', quantity='average', cell_methods='time: mean') field_definition_tas_climatology = definitions.OutputVariable.from_template( definitions.TEMPLATE_TEMPERATURE, 'tas_climatology', quantity='average', cell_methods='time: mean') field_definition_tas_large_scale = definitions.OutputVariable.from_template( definitions.TEMPLATE_PERTURBATION, 'tas_large_scale', quantity='average', cell_methods='time: mean') field_definition_tas_daily_local = definitions.OutputVariable.from_template( definitions.TEMPLATE_PERTURBATION, 'tas_daily_local', quantity='average', cell_methods='time: mean') result_expected_value = climatology_expected_value + large_scale_expected_value + local_expected_value filebuilder.add_global_field( field_definition_tas, result_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.add_global_field( field_definition_tas_climatology, climatology_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.add_global_field( field_definition_tas_large_scale, large_scale_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.add_global_field( field_definition_tas_daily_local, local_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.save_and_close() print "Memory usage (MB):", psutil.Process( os.getpid()).memory_info().rss / (1024 * 1024)
def output_grid_batch(storage_climatology, storage_large_scale, storage_local, outputfiles, climatologyfiles, largescalefiles, localfiles, time_indices, covariates_descriptor, insitu_biases, breakpoints_file, global_biases, global_biases_group_list, compute_uncertainties, method, compute_sample, sample_size, compute_prior_sample): from eustace.analysis.advanced_standard.fileio.output_projector import Projector variance_ratio_upper_bound = 1.0 print 'VERSION: {0}'.format(get_revision_id_for_module(eustace)) # Build analysis system analysissystem = AnalysisSystem_EUSTACE( storage_climatology, storage_large_scale, storage_local, covariates_descriptor, insitu_biases, breakpoints_file, global_biases, global_biases_group_list, compute_uncertainties, method) grid_resolution = [ 180. / definitions.GLOBAL_FIELD_SHAPE[1], 360. / definitions.GLOBAL_FIELD_SHAPE[2] ] latitudes = numpy.linspace(-90. + grid_resolution[0] / 2., 90. - grid_resolution[0] / 2, num=definitions.GLOBAL_FIELD_SHAPE[1]) longitudes = numpy.linspace(-180. + grid_resolution[1] / 2., 180. - grid_resolution[1] / 2, num=definitions.GLOBAL_FIELD_SHAPE[2]) timebase = TimeBaseDays(eustace.timeutils.epoch.EPOCH) #processdates = [datetime_numeric.build( timebase.number_to_datetime(daynumber) ) for daynumber in time_indices] processdates = [ timebase.number_to_datetime(daynumber) for daynumber in time_indices ] cell_sampling = [1, 1] blocking = 10 # thinned set of sample indices for inclusion in output product sample_indices = range(definitions.GLOBAL_SAMPLE_SHAPE[3]) climatology_projector = None large_scale_projector = None local_projector = None for (inner_index, time_index, processdate) in zip(range(len(time_indices)), time_indices, processdates): print time_index # Configure output grid outputstructure = OutputRectilinearGridStructure(time_index, processdate, latitudes=latitudes, longitudes=longitudes) # climatology component print 'Evaluating: climatology' if climatology_projector is None: climatology_projector = Projector(latitudes, longitudes, grid_resolution, time_index, cell_sampling, blocking) climatology_projector.set_component(analysissystem.components[0]) climatology_projector.update_time_index(time_index, keep_design=False) climatology_projector.evaluate_design_matrix() climatology_expected_value = climatology_projector.project_expected_value( ).reshape((-1, 1)) climatology_uncertainties = climatology_projector.project_sample_deviation( ) climatology_samples = climatology_projector.project_sample_values( sample_indices=sample_indices) + climatology_expected_value climatology_unconstraint = numpy.minimum( climatology_uncertainties**2 / climatology_projector.project_sample_deviation(prior=True)**2, variance_ratio_upper_bound) # large scale component print 'Evaluating: large-scale' if large_scale_projector is None: large_scale_projector = Projector(latitudes, longitudes, grid_resolution, time_index, cell_sampling, blocking) large_scale_projector.set_component(analysissystem.components[1]) large_scale_projector.update_time_index(time_index, keep_design=False) large_scale_projector.evaluate_design_matrix() large_scale_expected_value = large_scale_projector.project_expected_value( ).reshape((-1, 1)) large_scale_uncertainties = large_scale_projector.project_sample_deviation( ) large_scale_samples = large_scale_projector.project_sample_values( sample_indices=sample_indices) + large_scale_expected_value large_scale_unconstraint = numpy.minimum( large_scale_uncertainties**2 / large_scale_projector.project_sample_deviation(prior=True)**2, variance_ratio_upper_bound) # local component - time handling updates state to new time but does not recompute the design matrix print 'Evaluating: local' if local_projector is None: local_projector = Projector(latitudes, longitudes, grid_resolution, time_index, cell_sampling, blocking) local_projector.set_component(analysissystem.components[2]) local_projector.evaluate_design_matrix() else: local_projector.update_time_index(time_index, keep_design=True) local_projector.set_component(analysissystem.components[2], keep_design=True) print analysissystem.components local_expected_value = local_projector.project_expected_value( ).reshape((-1, 1)) local_uncertainties = local_projector.project_sample_deviation() local_samples = local_projector.project_sample_values( sample_indices=sample_indices) + local_expected_value local_unconstraint = numpy.minimum( local_uncertainties**2 / local_projector.project_sample_deviation(prior=True)**2, variance_ratio_upper_bound) # Save results outputfile = outputfiles[inner_index] print outputfile # main merged product output files filebuilder = FileBuilderGlobalField( outputfile, eustace.timeutils.epoch.days_since_epoch(processdate), 'EUSTACE Analysis', get_revision_id_for_module(eustace), definitions.TAS.name, '', 'Provisional output', __name__, '') result_expected_value = climatology_expected_value + large_scale_expected_value + local_expected_value result_expected_uncertainties = numpy.sqrt( climatology_uncertainties**2 + large_scale_uncertainties**2 + local_uncertainties**2) climatology_fraction = local_unconstraint # defined as ratio of posterior to prior variance in local component filebuilder.add_global_field( definitions.TAS, result_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.add_global_field( definitions.TASUNCERTAINTY, result_expected_uncertainties.reshape( definitions.GLOBAL_FIELD_SHAPE)) filebuilder.add_global_field( definitions.TAS_OBSERVATION_INFLUENCE, 1.0 - climatology_fraction.reshape(definitions.GLOBAL_FIELD_SHAPE)) for index in range(definitions.GLOBAL_SAMPLE_SHAPE[3]): variable = copy.deepcopy(definitions.TASENSEMBLE) variable.name = variable.name + '_' + str(index) selected_sample = (climatology_samples[:, index] + large_scale_samples[:, index] + local_samples[:, index]).ravel() filebuilder.add_global_field( variable, selected_sample.reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.save_and_close() # climatology only output climatologyfile = climatologyfiles[inner_index] filebuilder = FileBuilderGlobalField( climatologyfile, eustace.timeutils.epoch.days_since_epoch(processdate), 'EUSTACE Analysis', get_revision_id_for_module(eustace), definitions.TAS.name, '', 'Provisional component output - climatology', __name__, '') filebuilder.add_global_field( definitions.TAS, climatology_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.add_global_field( definitions.TASUNCERTAINTY, climatology_uncertainties.reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.add_global_field( definitions.TAS_OBSERVATION_INFLUENCE, 1.0 - climatology_unconstraint.reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.save_and_close() # large scale only output largescalefile = largescalefiles[inner_index] filebuilder = FileBuilderGlobalField( largescalefile, eustace.timeutils.epoch.days_since_epoch(processdate), 'EUSTACE Analysis', get_revision_id_for_module(eustace), definitions.TAS.name, '', 'Provisional component output - large scale', __name__, '') filebuilder.add_global_field( definitions.TASPERTURBATION, large_scale_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.add_global_field( definitions.TASUNCERTAINTY, large_scale_uncertainties.reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.add_global_field( definitions.TAS_OBSERVATION_INFLUENCE, 1.0 - large_scale_unconstraint.reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.save_and_close() # local only output localfile = localfiles[inner_index] filebuilder = FileBuilderGlobalField( localfile, eustace.timeutils.epoch.days_since_epoch(processdate), 'EUSTACE Analysis', get_revision_id_for_module(eustace), definitions.TAS.name, '', 'Provisional component output - local', __name__, '') filebuilder.add_global_field( definitions.TASPERTURBATION, local_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.add_global_field( definitions.TASUNCERTAINTY, local_uncertainties.reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.add_global_field( definitions.TAS_OBSERVATION_INFLUENCE, 1.0 - local_unconstraint.reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.save_and_close() print "Memory usage (MB):", psutil.Process( os.getpid()).memory_info().rss / (1024 * 1024)
def output_grid(storage_climatology, storage_large_scale, storage_local, outputfile, climatologyfile, largescalefile, localfile, processdate, time_index, covariates_descriptor, insitu_biases, breakpoints_file, global_biases, global_biases_group_list, compute_uncertainties, method, compute_sample, sample_size, compute_prior_sample): from eustace.analysis.advanced_standard.fileio.output_projector import Projector print 'VERSION: {0}'.format(get_revision_id_for_module(eustace)) # Build analysis system analysissystem = AnalysisSystem_EUSTACE( storage_climatology, storage_large_scale, storage_local, covariates_descriptor, insitu_biases, breakpoints_file, global_biases, global_biases_group_list, compute_uncertainties, method) grid_resolution = [ 180. / definitions.GLOBAL_FIELD_SHAPE[1], 360. / definitions.GLOBAL_FIELD_SHAPE[2] ] latitudes = numpy.linspace(-90. + grid_resolution[0] / 2., 90. - grid_resolution[0] / 2, num=definitions.GLOBAL_FIELD_SHAPE[1]) longitudes = numpy.linspace(-180. + grid_resolution[1] / 2., 180. - grid_resolution[1] / 2, num=definitions.GLOBAL_FIELD_SHAPE[2]) cell_sampling = [1, 1] blocking = 10 # Configure output grid outputstructure = OutputRectilinearGridStructure(time_index, processdate, latitudes=latitudes, longitudes=longitudes) # thinned set of sample indices for inclusion in output product sample_indices = range(definitions.GLOBAL_SAMPLE_SHAPE[3]) # climatology component print 'Evaluating: climatology' climatology_projector = Projector(latitudes, longitudes, grid_resolution, time_index, cell_sampling, blocking) climatology_projector.set_component(analysissystem.components[0]) climatology_projector.evaluate_design_matrix() climatology_expected_value = climatology_projector.project_expected_value( ).reshape((-1, 1)) climatology_uncertainties = climatology_projector.project_sample_deviation( ) climatology_samples = climatology_projector.project_sample_values( sample_indices=sample_indices) + climatology_expected_value climatology_unconstraint = climatology_uncertainties**2 / climatology_projector.project_sample_deviation( prior=True)**2 climatology_projector = None # clear projector from memory print climatology_expected_value.shape, climatology_uncertainties.shape, climatology_samples.shape # large scale component print 'Evaluating: large-scale' large_scale_projector = Projector(latitudes, longitudes, grid_resolution, time_index, cell_sampling, blocking) large_scale_projector.set_component(analysissystem.components[1]) large_scale_projector.evaluate_design_matrix() large_scale_expected_value = large_scale_projector.project_expected_value( ).reshape((-1, 1)) large_scale_uncertainties = large_scale_projector.project_sample_deviation( ) large_scale_samples = large_scale_projector.project_sample_values( sample_indices=sample_indices) + large_scale_expected_value large_scale_unconstraint = large_scale_uncertainties**2 / large_scale_projector.project_sample_deviation( prior=True)**2 large_scale_projector = None # clear projector from memory print large_scale_expected_value.shape, large_scale_uncertainties.shape, large_scale_samples.shape # local component print 'Evaluating: local' local_projector = Projector(latitudes, longitudes, grid_resolution, time_index, cell_sampling, blocking) local_projector.set_component(analysissystem.components[2]) local_projector.evaluate_design_matrix() local_expected_value = local_projector.project_expected_value().reshape( (-1, 1)) local_uncertainties = local_projector.project_sample_deviation() local_samples = local_projector.project_sample_values( sample_indices=sample_indices) + local_expected_value local_unconstraint = local_uncertainties**2 / local_projector.project_sample_deviation( prior=True)**2 local_projector = None # clear projector from memory print local_expected_value.shape, local_uncertainties.shape, local_samples.shape # Save results print outputfile # main merged product output files filebuilder = FileBuilderGlobalField( outputfile, eustace.timeutils.epoch.days_since_epoch(processdate), 'EUSTACE Analysis', get_revision_id_for_module(eustace), definitions.TAS.name, '', 'Provisional output', __name__, '') climatology_fraction = local_unconstraint # defined as ratio of posterior to prior variance in local component result_expected_value = climatology_expected_value + large_scale_expected_value + local_expected_value result_expected_uncertainties = numpy.sqrt(climatology_uncertainties**2 + large_scale_uncertainties**2 + local_uncertainties**2) filebuilder.add_global_field( definitions.TAS, result_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.add_global_field( definitions.TASUNCERTAINTY, result_expected_uncertainties.reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.add_global_field( definitions.TAS_CLIMATOLOGY_FRACTION, climatology_fraction.reshape(definitions.GLOBAL_FIELD_SHAPE)) for index in range(definitions.GLOBAL_SAMPLE_SHAPE[3]): variable = copy.deepcopy(definitions.TASENSEMBLE) variable.name = variable.name + '_' + str(index) selected_sample = (climatology_samples[:, index] + large_scale_samples[:, index] + local_samples[:, index]).ravel() filebuilder.add_global_field( variable, selected_sample.reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.save_and_close() # climatology only output filebuilder = FileBuilderGlobalField( climatologyfile, eustace.timeutils.epoch.days_since_epoch(processdate), 'EUSTACE Analysis', get_revision_id_for_module(eustace), definitions.TAS.name, '', 'Provisional component output - climatology', __name__, '') filebuilder.add_global_field( definitions.TAS, climatology_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.add_global_field( definitions.TASUNCERTAINTY, climatology_uncertainties.reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.add_global_field( definitions.TAS_CLIMATOLOGY_FRACTION, climatology_unconstraint.reshape(definitions.GLOBAL_FIELD_SHAPE)) # large scale only output filebuilder = FileBuilderGlobalField( largescalefile, eustace.timeutils.epoch.days_since_epoch(processdate), 'EUSTACE Analysis', get_revision_id_for_module(eustace), definitions.TAS.name, '', 'Provisional component output - large scale', __name__, '') filebuilder.add_global_field( definitions.TASPERTURBATION, large_scale_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.add_global_field( definitions.TASUNCERTAINTY, large_scale_uncertainties.reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.add_global_field( definitions.TAS_CLIMATOLOGY_FRACTION, large_scale_unconstraint.reshape(definitions.GLOBAL_FIELD_SHAPE)) # local only output filebuilder = FileBuilderGlobalField( localfile, eustace.timeutils.epoch.days_since_epoch(processdate), 'EUSTACE Analysis', get_revision_id_for_module(eustace), definitions.TAS.name, '', 'Provisional component output - local', __name__, '') filebuilder.add_global_field( definitions.TASPERTURBATION, local_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.add_global_field( definitions.TASUNCERTAINTY, local_uncertainties.reshape(definitions.GLOBAL_FIELD_SHAPE)) filebuilder.add_global_field( definitions.TAS_CLIMATOLOGY_FRACTION, local_unconstraint.reshape(definitions.GLOBAL_FIELD_SHAPE))
#source_encoding = 'utf-8-sig' # The master toctree document. master_doc = 'index' # General information about the project. project = u'EUSTACE System' copyright = u'2019, Met Office' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. try: version = get_revision_id_for_module(eustace) except: version = "!!!UNVERSIONED!!!" # The full version, including alpha/beta/rc tags. release = version # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. #language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: #today = '' # Else, today_fmt is used as the format for a strftime call. #today_fmt = '%B %d, %Y'