def output_grid(storage_climatology, storage_large_scale, storage_local,
outputfile, processdate, time_index,
covariates_descriptor, insitu_biases, breakpoints_file, global_biases, global_biases_group_list,
compute_uncertainties, method,
compute_sample, sample_size):
print 'VERSION: {0}'.format(get_revision_id_for_module(eustace))
# Build analysis system
analysissystem = AnalysisSystem_EUSTACE(storage_climatology, storage_large_scale, storage_local,
covariates_descriptor, insitu_biases, breakpoints_file, global_biases, global_biases_group_list,
compute_uncertainties, method,
compute_sample, sample_size)
#Configure output grid
outputstructure = OutputRectilinearGridStructure(
time_index, processdate,
latitudes=numpy.linspace(-90.+definitions.GLOBAL_FIELD_RESOLUTION/2., 90.-definitions.GLOBAL_FIELD_RESOLUTION/2., num=definitions.GLOBAL_FIELD_SHAPE[1]),
longitudes=numpy.linspace(-180.+definitions.GLOBAL_FIELD_RESOLUTION/2., 180.-definitions.GLOBAL_FIELD_RESOLUTION/2., num=definitions.GLOBAL_FIELD_SHAPE[2]))
# Evaluate expected value at these locations
for field in ['MAP', 'post_STD']:
print 'Evaluating: ',field
result_expected_value = analysissystem.evaluate_expected_value('MAP', outputstructure, 'GRID_CELL_AREA_AVERAGE', [1,1], 1000)
result_expected_uncertainties = analysissystem.evaluate_expected_value('post_STD', outputstructure, 'GRID_CELL_AREA_AVERAGE', [1,1], 1000)
print 'Evaluating: climatology fraction'
climatology_fraction = analysissystem.evaluate_climatology_fraction(outputstructure, [1,1], 1000)
print 'Evaluating: the sample'
sample = analysissystem.evaluate_projected_sample(outputstructure)
# Save results
filebuilder = FileBuilderGlobalField(
outputfile,
eustace.timeutils.epoch.days_since_epoch(processdate),
'Infilling Example',
get_revision_id_for_module(eustace),
definitions.TAS.name,
'',
'Example data only',
__name__,
'')
filebuilder.add_global_field(definitions.TAS, result_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(definitions.TASUNCERTAINTY, result_expected_uncertainties.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(definitions.TAS_CLIMATOLOGY_FRACTION, climatology_fraction.reshape(definitions.GLOBAL_FIELD_SHAPE))
for index in range(definitions.GLOBAL_SAMPLE_SHAPE[3]):
variable = copy.deepcopy(definitions.TASENSEMBLE)
variable.name = variable.name + '_' + str(index)
selected_sample = sample[:,index].ravel()+result_expected_value
filebuilder.add_global_field(variable, selected_sample.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.save_and_close()
def versionlist():
"""Get list of version identifiers for tools used."""
tools = []
if RECORD_JASMIN_TOOLCHAIN:
tools.append(commandline(['rpm', '-q', 'jasmin-common-vm']))
tools.append('eustace-' + get_revision_id_for_module(eustace))
return tools
def output_grid_component(storage_climatology, storage_large_scale, storage_local,
outputfile, processdate, time_index,
covariates_descriptor, insitu_biases, breakpoints_file, global_biases, global_biases_group_list,
compute_uncertainties, method):
print 'VERSION: {0}'.format(get_revision_id_for_module(eustace))
# Build analysis system
analysissystem = AnalysisSystem_EUSTACE(storage_climatology, storage_large_scale, storage_local,
covariates_descriptor, insitu_biases, breakpoints_file, global_biases, global_biases_group_list,
compute_uncertainties, method)
# Configure output grid
outputstructure = OutputRectilinearGridStructure(
time_index, processdate,
latitudes=numpy.linspace(-89.875, 89.875, num=definitions.GLOBAL_FIELD_SHAPE[1]),
longitudes=numpy.linspace(-179.875, 179.875, num=definitions.GLOBAL_FIELD_SHAPE[2]))
# Evaluate expected value at these locations
for field in ['MAP', 'post_STD']:
print 'Evaluating: ',field
result_expected_value = analysissystem.evaluate_expected_value('MAP', outputstructure, 'GRID_CELL_AREA_AVERAGE', [1,1], 1000)
result_expected_uncertainties = analysissystem.evaluate_expected_value('post_STD', outputstructure, 'GRID_CELL_AREA_AVERAGE', [1,1], 1000)
# Save results
filebuilder = FileBuilderGlobalField(
outputfile,
eustace.timeutils.epoch.days_since_epoch(processdate),
'Infilling Example',
get_revision_id_for_module(eustace),
definitions.TAS.name,
'',
'Example data only',
__name__,
'')
filebuilder.add_global_field(definitions.TAS, result_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(definitions.TASUNCERTAINTY, result_expected_uncertainties.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.save_and_close()
def output_homog(storage_climatology, storage_large_scale, storage_local,
outputfile, climatologyfile, largescalefile, localfile,
processdate, time_index, covariates_descriptor, insitu_biases,
breakpoints_file, global_biases, global_biases_group_list,
compute_uncertainties, method, compute_sample, sample_size,
compute_prior_sample):
from eustace.preprocess.fileio.insitu_land_breakpoints import ObservationBreakPointSourceInsituLand
from eustace.analysis.advanced_standard.fileio.output_projector import Projector
print 'VERSION: {0}'.format(get_revision_id_for_module(eustace))
# Build analysis system
analysissystem = AnalysisSystem_EUSTACE(
storage_climatology, storage_large_scale, storage_local,
covariates_descriptor, insitu_biases, breakpoints_file, global_biases,
global_biases_group_list, compute_uncertainties, method)
component = analysissystem.components[1]
element = component.storage.element_read()
hyperparameters = component.storage.hyperparameters_read()
currentstate = component.solutionstorage.state_read()
element_states = component.storage.element_read().element_prior(
hyperparameters).element_states(currentstate)
breakpoints_reader = ObservationBreakPointSourceInsituLand(
breakpoints_file)
station_locations = breakpoints_reader.observation_location_lookup()
#station_indices = breakpoints_reader.observation_location_lookup()
breakpoints_reader.dataset.close()
for i, subelement in enumerate(element.combination):
if isinstance(subelement, BiasElement):
mystate = element_states[i]
if hasattr(subelement, 'observed_breakpoints'):
import pickle
outdict = {
'breakpoints': subelement.observed_breakpoints,
'locations': station_locations.T,
'biases': mystate
}
with open(
'/work/scratch/cmorice/advanced_standard/observed_breakpoints.pickle',
'w') as f:
pickle.dump(outdict, f)
def save_flag_file(flag_values, processdate, outputfile):
"""Save the windowing flag information"""
print "Saving: ", outputfile
# Create directory structure if required
if not os.path.exists(os.path.dirname(outputfile)):
os.makedirs(os.path.dirname(outputfile))
# Make the QC flag file
filebuilder = FileBuilderGlobalField(
outputfile,
eustace.timeutils.epoch.days_since_epoch(processdate),
'EUSTACE Analysis Flags',
get_revision_id_for_module(eustace),
definitions.TAS.name,
'',
'Provisional output',
__name__,
'')
# flag flag definition
tas_qc_definition = OutputVariable( name='tas_qc',
dtype=FLAG_TYPE,
fill_value=default_fillvals[TYPE_NAME],
standard_name='air_temperature status_flag',
long_name='Quality control flags',
#valid_range=numpy.array([FLAG_TYPE(0), FLAG_TYPE(1)<<FLAG_MAX_N], FLAG_TYPE),
flag_masks=numpy.array([ FLAG_TYPE(1)<<n for n in range(FLAG_MAX_USED)], FLAG_TYPE),
flag_meanings=' '.join(FLAG_MEANINGS))
# Combine flag information and populate the output file
filebuilder.add_global_field( tas_qc_definition, flag_values.reshape(definitions.GLOBAL_FIELD_SHAPE) )
# Close the netCDF4 dataset
filebuilder.save_and_close()
def apply_mask(analysis_directory, iteration, mask_directory, mask_directory2, flags_to_mask, processdate, output_directory):
"""Apply QC mask to analysis"""
datestring = '%04i%02i%02i' % (processdate.year, processdate.month, processdate.day)
# get mask defined by provided options
qc_flags = load_flags(mask_directory, iteration, datestring).data[:,:,:]
if mask_directory2 is not None:
qc_flags = qc_flags | load_flags(mask_directory2, iteration, datestring).data[:,:,:]
# set a new marine flag with a more flexible definition of marine (95% water rather than 100%)
qc_flags = reset_marine_flag(qc_flags)
combined_mask = compute_mask(qc_flags, flags_to_mask)
# get analysis
analysis, uncertainty, influence, ensemble_members = load_analysis(analysis_directory, iteration, datestring)
# build masked output product file
#outputfile = output_directory+'/eustace_analysis_'+str(iteration)+'/'+datestring[:4]+'/eustace_analysis_'+str(iteration)+'_masked_'+datestring+'.nc'
#format as variable_collection_framework_realization_YYYYmmdd.nc so tas_global_eustace_0_YYYYmmdd.nc
if iteration == 4:
version_string = "R001400"
rundate = '20190326'
elif iteration == 9:
version_string = "R001413"
rundate = '20190405'
else:
print "unrecognised version"
outputfile = output_directory+'/'+version_string+'/'+rundate+'/global/'+datestring[:4]+'/tas_global_eustace_0_'+datestring+'.nc'
print outputfile
filebuilder = FileBuilderGlobalField(
outputfile,
eustace.timeutils.epoch.days_since_epoch(processdate),
'EUSTACE Analysis',
get_revision_id_for_module(eustace),
definitions.TAS.name,
'Met Office',
'',
'',
'')
numpy.squeeze(analysis.data[:,:,:])
numpy.squeeze(uncertainty.data[:,:,:])
numpy.squeeze(influence.data[:,:,:])
filebuilder.add_global_field(definitions.TAS, numpy.ma.masked_where( combined_mask, analysis.data[:,:,:] ).reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(definitions.TASUNCERTAINTY, numpy.ma.masked_where( combined_mask, uncertainty.data[:,:,:] ).reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(definitions.TAS_OBSERVATION_INFLUENCE, numpy.ma.masked_where( combined_mask, influence .data[:,:,:] ).reshape(definitions.GLOBAL_FIELD_SHAPE))
# sort ensemble members and apply mask
ensemble_members = sorted(ensemble_members, key=lambda item: int(item.var_name.split('_')[-1]))
for index, ensemble_member in enumerate(ensemble_members):
variable = copy.deepcopy(definitions.TASENSEMBLE)
variable.name = variable.name + '_' + str(index)
filebuilder.add_global_field(variable, numpy.ma.masked_where( combined_mask, ensemble_member.data[:,:,:] ).reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.save_and_close()
# climatology only output
analysis, uncertainty, influence = load_climatology(analysis_directory, iteration, datestring)
climatologyfile = output_directory+'/eustace_climatology_component_'+str(iteration)+'/'+datestring[:4]+'/eustace_climatology_component_'+str(iteration)+'_masked_'+datestring+'.nc'
filebuilder = FileBuilderGlobalField(
climatologyfile,
eustace.timeutils.epoch.days_since_epoch(processdate),
'EUSTACE Analysis',
get_revision_id_for_module(eustace),
definitions.TAS.name,
'',
'Provisional component output - climatology',
__name__,
'')
numpy.squeeze(analysis.data[:,:,:])
numpy.squeeze(uncertainty.data[:,:,:])
numpy.squeeze(influence.data[:,:,:])
filebuilder.add_global_field(definitions.TAS, numpy.ma.masked_where( combined_mask, analysis.data[:,:,:] ).reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(definitions.TASUNCERTAINTY, numpy.ma.masked_where( combined_mask, uncertainty.data[:,:,:] ).reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(definitions.TAS_OBSERVATION_INFLUENCE, numpy.ma.masked_where( combined_mask, influence .data[:,:,:] ).reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.save_and_close()
# large scale only output
analysis, uncertainty, influence = load_large_scale(analysis_directory, iteration, datestring)
largescalefile = output_directory+'/eustace_large_scale_component_'+str(iteration)+'/'+datestring[:4]+'/eustace_large_scale_component_'+str(iteration)+'_masked_'+datestring+'.nc'
filebuilder = FileBuilderGlobalField(
largescalefile,
eustace.timeutils.epoch.days_since_epoch(processdate),
'EUSTACE Analysis',
get_revision_id_for_module(eustace),
definitions.TAS.name,
'',
'Provisional component output - large scale',
__name__,
'')
numpy.squeeze(analysis.data[:,:,:])
numpy.squeeze(uncertainty.data[:,:,:])
numpy.squeeze(influence.data[:,:,:])
filebuilder.add_global_field(definitions.TASPERTURBATION, numpy.ma.masked_where( combined_mask, analysis.data[:,:,:] ).reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(definitions.TASUNCERTAINTY, numpy.ma.masked_where( combined_mask, uncertainty.data[:,:,:] ).reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(definitions.TAS_OBSERVATION_INFLUENCE, numpy.ma.masked_where( combined_mask, influence .data[:,:,:] ).reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.save_and_close()
# local only output
analysis, uncertainty, influence = load_local(analysis_directory, iteration, datestring)
localfile = output_directory+'/eustace_local_component_'+str(iteration)+'/'+datestring[:4]+'/eustace_local_component_'+str(iteration)+'_masked_'+datestring+'.nc'
filebuilder = FileBuilderGlobalField(
localfile,
eustace.timeutils.epoch.days_since_epoch(processdate),
'EUSTACE Analysis',
get_revision_id_for_module(eustace),
definitions.TAS.name,
'',
'Provisional component output - local',
__name__,
'')
numpy.squeeze(analysis.data[:,:,:])
numpy.squeeze(uncertainty.data[:,:,:])
numpy.squeeze(influence.data[:,:,:])
filebuilder.add_global_field(definitions.TASPERTURBATION, numpy.ma.masked_where( combined_mask, analysis.data[:,:,:] ).reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(definitions.TASUNCERTAINTY, numpy.ma.masked_where( combined_mask, uncertainty.data[:,:,:] ).reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(definitions.TAS_OBSERVATION_INFLUENCE, numpy.ma.masked_where( combined_mask, influence .data[:,:,:] ).reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.save_and_close()
def latent_variable_flag(input_directory, output_directory, iteration, processing_dates):
# manually setup the analysis model for the R1413 run - Warning: the eustace svn revision must be correct for the global bias model interpretation to that run analysis
storage_climatology = SpaceTimeComponentSolutionStorageBatched_Files( statefilename_read='/work/scratch/cmorice/advanced_standard/climatology_solution_9/climatology_solution_9.pickle',
sample_filename_read='/work/scratch/cmorice/advanced_standard/climatology_solution_sample_9/climatology_solution_sample_9.pickle',
prior_sample_filename_read='/work/scratch/cmorice/advanced_standard/climatology_solution_prior_sample_9/climatology_solution_prior_sample_9.pickle',
keep_in_memory = True )
storage_large_scale = SpaceTimeComponentSolutionStorageBatched_Files( statefilename_read='/work/scratch/cmorice/advanced_standard/large_scale_solution_9/large_scale_solution_9.pickle',
sample_filename_read='/work/scratch/cmorice/advanced_standard/large_scale_solution_sample_9/large_scale_solution_sample_9.pickle',
prior_sample_filename_read='/work/scratch/cmorice/advanced_standard/large_scale_solution_prior_sample_9/large_scale_solution_prior_sample_9.pickle',
keep_in_memory = True )
storage_local = eustace.analysis.advanced_standard.components.storage_files_batch.SpatialComponentSolutionStorageIndexed_Files()
covariates_descriptor = "/gws/nopw/j04/eustace/data/internal/climatology_covariates/covariates.json"
insitu_biases = True
breakpoints_file = "/gws/nopw/j04/eustace/data/internal/D1.7/daily/eustace_stations_global_R001127_daily_status.nc"
global_biases = True
global_biases_group_list = ["surfaceairmodel_ice_global" , "surfaceairmodel_land_global", "surfaceairmodel_ocean_global"]
compute_uncertainties = False
method = 'EXACT'
compute_sample = False
sample_size = definitions.GLOBAL_SAMPLE_SHAPE[3]
compute_prior_sample = False
print 'VERSION: {0}'.format(get_revision_id_for_module(eustace))
# Build analysis system
analysissystem = AnalysisSystem_EUSTACE(storage_climatology, storage_large_scale, storage_local,
covariates_descriptor, insitu_biases, breakpoints_file, global_biases, global_biases_group_list,
compute_uncertainties, method)
grid_resolution = [180. / definitions.GLOBAL_FIELD_SHAPE[1], 360. / definitions.GLOBAL_FIELD_SHAPE[2]]
latitudes=numpy.linspace(-90.+grid_resolution[0]/2., 90.-grid_resolution[0]/2, num=definitions.GLOBAL_FIELD_SHAPE[1])
longitudes=numpy.linspace(-180.+grid_resolution[1]/2., 180.-grid_resolution[1]/2, num=definitions.GLOBAL_FIELD_SHAPE[2])
#timebase = TimeBaseDays(eustace.timeutils.epoch.EPOCH)
#processdates = [timebase.number_to_datetime(daynumber) for daynumber in time_indices]
# get times as understood by the analysis sustem
time_indices =[eustace.timeutils.epoch.days_since_epoch(t) for t in processing_dates]
cell_sampling = [1, 1]
blocking = 10
# thinned set of sample indices for inclusion in output product
sample_indices = range(definitions.GLOBAL_SAMPLE_SHAPE[3])
climatology_projector = None
large_scale_projector = None
local_projector = None
for ( inner_index, time_index, processdate ) in zip( range(len(time_indices)), time_indices, processing_dates ):
print time_index
# initialise flags
flag_values = numpy.zeros( definitions.GLOBAL_FIELD_SHAPE[1:], FLAG_TYPE )
# Configure output grid
outputstructure = OutputRectilinearGridStructure(time_index, processdate,
latitudes=latitudes,
longitudes=longitudes)
# climatology component
print 'Evaluating: climatology'
if climatology_projector is None:
climatology_projector = Projector(latitudes, longitudes, grid_resolution, time_index, cell_sampling, blocking)
climatology_projector.set_component(analysissystem.components[0])
latent_climatology_constraint = evaluate_latent_variable_constraint(climatology_projector)
climatology_projector.update_time_index(time_index, keep_design = False)
climatology_projector.evaluate_design_matrix()
climatology_statistic = evaluate_constraint_statistic(climatology_projector, latent_climatology_constraint, CONSTRAINT_THRESHOLD).reshape(definitions.GLOBAL_FIELD_SHAPE[1:])
flag_values[climatology_statistic] = flag_values[climatology_statistic] | CLIMATOLOGY_LATENT_FLAG
# large scale component
print 'Evaluating: large-scale'
if large_scale_projector is None:
large_scale_projector = Projector(latitudes, longitudes, grid_resolution, time_index, cell_sampling, blocking)
large_scale_projector.set_component(analysissystem.components[1])
latent_large_scale_constraint = evaluate_latent_variable_constraint(large_scale_projector)
large_scale_projector.update_time_index(time_index, keep_design = False)
large_scale_projector.evaluate_design_matrix()
large_scale_statistic = evaluate_constraint_statistic(large_scale_projector, latent_large_scale_constraint, CONSTRAINT_THRESHOLD).reshape(definitions.GLOBAL_FIELD_SHAPE[1:])
flag_values[large_scale_statistic] = flag_values[large_scale_statistic] | LARGE_SCALE_LATENT_FLAG
outputfile = os.path.join(output_directory, '{:04d}'.format(processdate.year), 'eustace_analysis_{:d}_qc_flags_{:04d}{:02d}{:02d}.nc'.format(iteration, processdate.year, processdate.month, processdate.day))
save_flag_file(flag_values, processdate, outputfile)
def early_look_grid_batch(
storage_climatology, storage_large_scale, storage_local, outputfiles,
time_indices, covariates_descriptor, insitu_biases, breakpoints_file,
global_biases, global_biases_group_list, compute_uncertainties, method,
compute_sample, sample_size, compute_prior_sample):
"""Produce 'early look' NetCDF output files without loading or gridding uncertainty information
For inspection of analysis output prior to the final gridding step.
"""
from eustace.analysis.advanced_standard.fileio.output_projector import Projector
print 'VERSION: {0}'.format(get_revision_id_for_module(eustace))
# Build analysis system
analysissystem = AnalysisSystem_EUSTACE(
storage_climatology, storage_large_scale, storage_local,
covariates_descriptor, insitu_biases, breakpoints_file, global_biases,
global_biases_group_list, compute_uncertainties, method)
grid_resolution = [
180. / definitions.GLOBAL_FIELD_SHAPE[1],
360. / definitions.GLOBAL_FIELD_SHAPE[2]
]
latitudes = numpy.linspace(-90. + grid_resolution[0] / 2.,
90. - grid_resolution[0] / 2,
num=definitions.GLOBAL_FIELD_SHAPE[1])
longitudes = numpy.linspace(-180. + grid_resolution[1] / 2.,
180. - grid_resolution[1] / 2,
num=definitions.GLOBAL_FIELD_SHAPE[2])
timebase = TimeBaseDays(eustace.timeutils.epoch.EPOCH)
processdates = [
timebase.number_to_datetime(daynumber) for daynumber in time_indices
]
cell_sampling = [1, 1]
blocking = 10
# thinned set of sample indices for inclusion in output product
climatology_projector = None
large_scale_projector = None
local_projector = None
for (inner_index, time_index, processdate) in zip(range(len(time_indices)),
time_indices,
processdates):
print time_index
# Configure output grid
outputstructure = OutputRectilinearGridStructure(time_index,
processdate,
latitudes=latitudes,
longitudes=longitudes)
# climatology component
print 'Evaluating: climatology'
if climatology_projector is None:
climatology_projector = Projector(latitudes, longitudes,
grid_resolution, time_index,
cell_sampling, blocking)
climatology_projector.set_component(analysissystem.components[0])
climatology_projector.update_time_index(time_index, keep_design=False)
climatology_projector.evaluate_design_matrix()
climatology_expected_value = climatology_projector.project_expected_value(
).reshape((-1, 1))
# large scale component
print 'Evaluating: large-scale'
if large_scale_projector is None:
large_scale_projector = Projector(latitudes, longitudes,
grid_resolution, time_index,
cell_sampling, blocking)
large_scale_projector.set_component(analysissystem.components[1])
large_scale_projector.update_time_index(time_index, keep_design=False)
large_scale_projector.evaluate_design_matrix()
large_scale_expected_value = large_scale_projector.project_expected_value(
).reshape((-1, 1))
# local component - time handling updates state to new time but does not recompute the design matrix
print 'Evaluating: local'
if local_projector is None:
local_projector = Projector(latitudes, longitudes, grid_resolution,
time_index, cell_sampling, blocking)
local_projector.set_component(analysissystem.components[2])
local_projector.evaluate_design_matrix()
else:
local_projector.update_time_index(time_index, keep_design=True)
local_projector.set_component(analysissystem.components[2],
keep_design=True)
print analysissystem.components
local_expected_value = local_projector.project_expected_value(
).reshape((-1, 1))
# Save results
outputfile = outputfiles[inner_index]
print outputfile
# main merged product output files
filebuilder = FileBuilderGlobalField(
outputfile, eustace.timeutils.epoch.days_since_epoch(processdate),
'EUSTACE Analysis', get_revision_id_for_module(eustace),
definitions.TAS.name, '', 'Provisional output', __name__, '')
field_definition_tas = definitions.OutputVariable.from_template(
definitions.TEMPLATE_TEMPERATURE,
'tas',
quantity='average',
cell_methods='time: mean')
field_definition_tas_climatology = definitions.OutputVariable.from_template(
definitions.TEMPLATE_TEMPERATURE,
'tas_climatology',
quantity='average',
cell_methods='time: mean')
field_definition_tas_large_scale = definitions.OutputVariable.from_template(
definitions.TEMPLATE_PERTURBATION,
'tas_large_scale',
quantity='average',
cell_methods='time: mean')
field_definition_tas_daily_local = definitions.OutputVariable.from_template(
definitions.TEMPLATE_PERTURBATION,
'tas_daily_local',
quantity='average',
cell_methods='time: mean')
result_expected_value = climatology_expected_value + large_scale_expected_value + local_expected_value
filebuilder.add_global_field(
field_definition_tas,
result_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(
field_definition_tas_climatology,
climatology_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(
field_definition_tas_large_scale,
large_scale_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(
field_definition_tas_daily_local,
local_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.save_and_close()
print "Memory usage (MB):", psutil.Process(
os.getpid()).memory_info().rss / (1024 * 1024)
def output_grid_batch(storage_climatology, storage_large_scale, storage_local,
outputfiles, climatologyfiles, largescalefiles,
localfiles, time_indices, covariates_descriptor,
insitu_biases, breakpoints_file, global_biases,
global_biases_group_list, compute_uncertainties, method,
compute_sample, sample_size, compute_prior_sample):
from eustace.analysis.advanced_standard.fileio.output_projector import Projector
variance_ratio_upper_bound = 1.0
print 'VERSION: {0}'.format(get_revision_id_for_module(eustace))
# Build analysis system
analysissystem = AnalysisSystem_EUSTACE(
storage_climatology, storage_large_scale, storage_local,
covariates_descriptor, insitu_biases, breakpoints_file, global_biases,
global_biases_group_list, compute_uncertainties, method)
grid_resolution = [
180. / definitions.GLOBAL_FIELD_SHAPE[1],
360. / definitions.GLOBAL_FIELD_SHAPE[2]
]
latitudes = numpy.linspace(-90. + grid_resolution[0] / 2.,
90. - grid_resolution[0] / 2,
num=definitions.GLOBAL_FIELD_SHAPE[1])
longitudes = numpy.linspace(-180. + grid_resolution[1] / 2.,
180. - grid_resolution[1] / 2,
num=definitions.GLOBAL_FIELD_SHAPE[2])
timebase = TimeBaseDays(eustace.timeutils.epoch.EPOCH)
#processdates = [datetime_numeric.build( timebase.number_to_datetime(daynumber) ) for daynumber in time_indices]
processdates = [
timebase.number_to_datetime(daynumber) for daynumber in time_indices
]
cell_sampling = [1, 1]
blocking = 10
# thinned set of sample indices for inclusion in output product
sample_indices = range(definitions.GLOBAL_SAMPLE_SHAPE[3])
climatology_projector = None
large_scale_projector = None
local_projector = None
for (inner_index, time_index, processdate) in zip(range(len(time_indices)),
time_indices,
processdates):
print time_index
# Configure output grid
outputstructure = OutputRectilinearGridStructure(time_index,
processdate,
latitudes=latitudes,
longitudes=longitudes)
# climatology component
print 'Evaluating: climatology'
if climatology_projector is None:
climatology_projector = Projector(latitudes, longitudes,
grid_resolution, time_index,
cell_sampling, blocking)
climatology_projector.set_component(analysissystem.components[0])
climatology_projector.update_time_index(time_index, keep_design=False)
climatology_projector.evaluate_design_matrix()
climatology_expected_value = climatology_projector.project_expected_value(
).reshape((-1, 1))
climatology_uncertainties = climatology_projector.project_sample_deviation(
)
climatology_samples = climatology_projector.project_sample_values(
sample_indices=sample_indices) + climatology_expected_value
climatology_unconstraint = numpy.minimum(
climatology_uncertainties**2 /
climatology_projector.project_sample_deviation(prior=True)**2,
variance_ratio_upper_bound)
# large scale component
print 'Evaluating: large-scale'
if large_scale_projector is None:
large_scale_projector = Projector(latitudes, longitudes,
grid_resolution, time_index,
cell_sampling, blocking)
large_scale_projector.set_component(analysissystem.components[1])
large_scale_projector.update_time_index(time_index, keep_design=False)
large_scale_projector.evaluate_design_matrix()
large_scale_expected_value = large_scale_projector.project_expected_value(
).reshape((-1, 1))
large_scale_uncertainties = large_scale_projector.project_sample_deviation(
)
large_scale_samples = large_scale_projector.project_sample_values(
sample_indices=sample_indices) + large_scale_expected_value
large_scale_unconstraint = numpy.minimum(
large_scale_uncertainties**2 /
large_scale_projector.project_sample_deviation(prior=True)**2,
variance_ratio_upper_bound)
# local component - time handling updates state to new time but does not recompute the design matrix
print 'Evaluating: local'
if local_projector is None:
local_projector = Projector(latitudes, longitudes, grid_resolution,
time_index, cell_sampling, blocking)
local_projector.set_component(analysissystem.components[2])
local_projector.evaluate_design_matrix()
else:
local_projector.update_time_index(time_index, keep_design=True)
local_projector.set_component(analysissystem.components[2],
keep_design=True)
print analysissystem.components
local_expected_value = local_projector.project_expected_value(
).reshape((-1, 1))
local_uncertainties = local_projector.project_sample_deviation()
local_samples = local_projector.project_sample_values(
sample_indices=sample_indices) + local_expected_value
local_unconstraint = numpy.minimum(
local_uncertainties**2 /
local_projector.project_sample_deviation(prior=True)**2,
variance_ratio_upper_bound)
# Save results
outputfile = outputfiles[inner_index]
print outputfile
# main merged product output files
filebuilder = FileBuilderGlobalField(
outputfile, eustace.timeutils.epoch.days_since_epoch(processdate),
'EUSTACE Analysis', get_revision_id_for_module(eustace),
definitions.TAS.name, '', 'Provisional output', __name__, '')
result_expected_value = climatology_expected_value + large_scale_expected_value + local_expected_value
result_expected_uncertainties = numpy.sqrt(
climatology_uncertainties**2 + large_scale_uncertainties**2 +
local_uncertainties**2)
climatology_fraction = local_unconstraint # defined as ratio of posterior to prior variance in local component
filebuilder.add_global_field(
definitions.TAS,
result_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(
definitions.TASUNCERTAINTY,
result_expected_uncertainties.reshape(
definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(
definitions.TAS_OBSERVATION_INFLUENCE,
1.0 - climatology_fraction.reshape(definitions.GLOBAL_FIELD_SHAPE))
for index in range(definitions.GLOBAL_SAMPLE_SHAPE[3]):
variable = copy.deepcopy(definitions.TASENSEMBLE)
variable.name = variable.name + '_' + str(index)
selected_sample = (climatology_samples[:, index] +
large_scale_samples[:, index] +
local_samples[:, index]).ravel()
filebuilder.add_global_field(
variable,
selected_sample.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.save_and_close()
# climatology only output
climatologyfile = climatologyfiles[inner_index]
filebuilder = FileBuilderGlobalField(
climatologyfile,
eustace.timeutils.epoch.days_since_epoch(processdate),
'EUSTACE Analysis', get_revision_id_for_module(eustace),
definitions.TAS.name, '',
'Provisional component output - climatology', __name__, '')
filebuilder.add_global_field(
definitions.TAS,
climatology_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(
definitions.TASUNCERTAINTY,
climatology_uncertainties.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(
definitions.TAS_OBSERVATION_INFLUENCE, 1.0 -
climatology_unconstraint.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.save_and_close()
# large scale only output
largescalefile = largescalefiles[inner_index]
filebuilder = FileBuilderGlobalField(
largescalefile,
eustace.timeutils.epoch.days_since_epoch(processdate),
'EUSTACE Analysis', get_revision_id_for_module(eustace),
definitions.TAS.name, '',
'Provisional component output - large scale', __name__, '')
filebuilder.add_global_field(
definitions.TASPERTURBATION,
large_scale_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(
definitions.TASUNCERTAINTY,
large_scale_uncertainties.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(
definitions.TAS_OBSERVATION_INFLUENCE, 1.0 -
large_scale_unconstraint.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.save_and_close()
# local only output
localfile = localfiles[inner_index]
filebuilder = FileBuilderGlobalField(
localfile, eustace.timeutils.epoch.days_since_epoch(processdate),
'EUSTACE Analysis', get_revision_id_for_module(eustace),
definitions.TAS.name, '', 'Provisional component output - local',
__name__, '')
filebuilder.add_global_field(
definitions.TASPERTURBATION,
local_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(
definitions.TASUNCERTAINTY,
local_uncertainties.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(
definitions.TAS_OBSERVATION_INFLUENCE,
1.0 - local_unconstraint.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.save_and_close()
print "Memory usage (MB):", psutil.Process(
os.getpid()).memory_info().rss / (1024 * 1024)
def output_grid(storage_climatology, storage_large_scale, storage_local,
outputfile, climatologyfile, largescalefile, localfile,
processdate, time_index, covariates_descriptor, insitu_biases,
breakpoints_file, global_biases, global_biases_group_list,
compute_uncertainties, method, compute_sample, sample_size,
compute_prior_sample):
from eustace.analysis.advanced_standard.fileio.output_projector import Projector
print 'VERSION: {0}'.format(get_revision_id_for_module(eustace))
# Build analysis system
analysissystem = AnalysisSystem_EUSTACE(
storage_climatology, storage_large_scale, storage_local,
covariates_descriptor, insitu_biases, breakpoints_file, global_biases,
global_biases_group_list, compute_uncertainties, method)
grid_resolution = [
180. / definitions.GLOBAL_FIELD_SHAPE[1],
360. / definitions.GLOBAL_FIELD_SHAPE[2]
]
latitudes = numpy.linspace(-90. + grid_resolution[0] / 2.,
90. - grid_resolution[0] / 2,
num=definitions.GLOBAL_FIELD_SHAPE[1])
longitudes = numpy.linspace(-180. + grid_resolution[1] / 2.,
180. - grid_resolution[1] / 2,
num=definitions.GLOBAL_FIELD_SHAPE[2])
cell_sampling = [1, 1]
blocking = 10
# Configure output grid
outputstructure = OutputRectilinearGridStructure(time_index,
processdate,
latitudes=latitudes,
longitudes=longitudes)
# thinned set of sample indices for inclusion in output product
sample_indices = range(definitions.GLOBAL_SAMPLE_SHAPE[3])
# climatology component
print 'Evaluating: climatology'
climatology_projector = Projector(latitudes, longitudes, grid_resolution,
time_index, cell_sampling, blocking)
climatology_projector.set_component(analysissystem.components[0])
climatology_projector.evaluate_design_matrix()
climatology_expected_value = climatology_projector.project_expected_value(
).reshape((-1, 1))
climatology_uncertainties = climatology_projector.project_sample_deviation(
)
climatology_samples = climatology_projector.project_sample_values(
sample_indices=sample_indices) + climatology_expected_value
climatology_unconstraint = climatology_uncertainties**2 / climatology_projector.project_sample_deviation(
prior=True)**2
climatology_projector = None # clear projector from memory
print climatology_expected_value.shape, climatology_uncertainties.shape, climatology_samples.shape
# large scale component
print 'Evaluating: large-scale'
large_scale_projector = Projector(latitudes, longitudes, grid_resolution,
time_index, cell_sampling, blocking)
large_scale_projector.set_component(analysissystem.components[1])
large_scale_projector.evaluate_design_matrix()
large_scale_expected_value = large_scale_projector.project_expected_value(
).reshape((-1, 1))
large_scale_uncertainties = large_scale_projector.project_sample_deviation(
)
large_scale_samples = large_scale_projector.project_sample_values(
sample_indices=sample_indices) + large_scale_expected_value
large_scale_unconstraint = large_scale_uncertainties**2 / large_scale_projector.project_sample_deviation(
prior=True)**2
large_scale_projector = None # clear projector from memory
print large_scale_expected_value.shape, large_scale_uncertainties.shape, large_scale_samples.shape
# local component
print 'Evaluating: local'
local_projector = Projector(latitudes, longitudes, grid_resolution,
time_index, cell_sampling, blocking)
local_projector.set_component(analysissystem.components[2])
local_projector.evaluate_design_matrix()
local_expected_value = local_projector.project_expected_value().reshape(
(-1, 1))
local_uncertainties = local_projector.project_sample_deviation()
local_samples = local_projector.project_sample_values(
sample_indices=sample_indices) + local_expected_value
local_unconstraint = local_uncertainties**2 / local_projector.project_sample_deviation(
prior=True)**2
local_projector = None # clear projector from memory
print local_expected_value.shape, local_uncertainties.shape, local_samples.shape
# Save results
print outputfile
# main merged product output files
filebuilder = FileBuilderGlobalField(
outputfile, eustace.timeutils.epoch.days_since_epoch(processdate),
'EUSTACE Analysis', get_revision_id_for_module(eustace),
definitions.TAS.name, '', 'Provisional output', __name__, '')
climatology_fraction = local_unconstraint # defined as ratio of posterior to prior variance in local component
result_expected_value = climatology_expected_value + large_scale_expected_value + local_expected_value
result_expected_uncertainties = numpy.sqrt(climatology_uncertainties**2 +
large_scale_uncertainties**2 +
local_uncertainties**2)
filebuilder.add_global_field(
definitions.TAS,
result_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(
definitions.TASUNCERTAINTY,
result_expected_uncertainties.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(
definitions.TAS_CLIMATOLOGY_FRACTION,
climatology_fraction.reshape(definitions.GLOBAL_FIELD_SHAPE))
for index in range(definitions.GLOBAL_SAMPLE_SHAPE[3]):
variable = copy.deepcopy(definitions.TASENSEMBLE)
variable.name = variable.name + '_' + str(index)
selected_sample = (climatology_samples[:, index] +
large_scale_samples[:, index] +
local_samples[:, index]).ravel()
filebuilder.add_global_field(
variable, selected_sample.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.save_and_close()
# climatology only output
filebuilder = FileBuilderGlobalField(
climatologyfile, eustace.timeutils.epoch.days_since_epoch(processdate),
'EUSTACE Analysis', get_revision_id_for_module(eustace),
definitions.TAS.name, '', 'Provisional component output - climatology',
__name__, '')
filebuilder.add_global_field(
definitions.TAS,
climatology_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(
definitions.TASUNCERTAINTY,
climatology_uncertainties.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(
definitions.TAS_CLIMATOLOGY_FRACTION,
climatology_unconstraint.reshape(definitions.GLOBAL_FIELD_SHAPE))
# large scale only output
filebuilder = FileBuilderGlobalField(
largescalefile, eustace.timeutils.epoch.days_since_epoch(processdate),
'EUSTACE Analysis', get_revision_id_for_module(eustace),
definitions.TAS.name, '', 'Provisional component output - large scale',
__name__, '')
filebuilder.add_global_field(
definitions.TASPERTURBATION,
large_scale_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(
definitions.TASUNCERTAINTY,
large_scale_uncertainties.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(
definitions.TAS_CLIMATOLOGY_FRACTION,
large_scale_unconstraint.reshape(definitions.GLOBAL_FIELD_SHAPE))
# local only output
filebuilder = FileBuilderGlobalField(
localfile, eustace.timeutils.epoch.days_since_epoch(processdate),
'EUSTACE Analysis', get_revision_id_for_module(eustace),
definitions.TAS.name, '', 'Provisional component output - local',
__name__, '')
filebuilder.add_global_field(
definitions.TASPERTURBATION,
local_expected_value.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(
definitions.TASUNCERTAINTY,
local_uncertainties.reshape(definitions.GLOBAL_FIELD_SHAPE))
filebuilder.add_global_field(
definitions.TAS_CLIMATOLOGY_FRACTION,
local_unconstraint.reshape(definitions.GLOBAL_FIELD_SHAPE))
#source_encoding = 'utf-8-sig'
# The master toctree document.
master_doc = 'index'
# General information about the project.
project = u'EUSTACE System'
copyright = u'2019, Met Office'
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
try:
version = get_revision_id_for_module(eustace)
except:
version = "!!!UNVERSIONED!!!"
# The full version, including alpha/beta/rc tags.
release = version
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None
# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'