def run(outputfilename, inputs, processdate):
"""EUMOPPS run commands."""
# Need day number for inclusion in output
daynumber = numpy.int64(days_since_epoch(processdate))
# Grid onto this
axes = GlobalFieldAxes2DWithTime(daynumber).aslist()
outputgrid = DiagnosticGridBins(axes)
# Cache location lookup as we might refer to same one multiple times
locationlookup_cache = {}
# One set of results per available source [land|sea|ice|lakes|in-situ land|in-situ ocean] per available observable [Tmean|Tmax|Tmin]
for inputindex, descriptor in enumerate(inputs):
# Cached loading of location lookup
try:
# Attempt to find in cache in case it's already been loaded for other sources/observables
locationlookup = locationlookup_cache[descriptor.locationfilename]
except KeyError:
# Not found - load it for the first time this operation
locationlookup = LocationLookupRawBinaryReader().read(
descriptor.locationfilename)
locationlookup_cache[descriptor.locationfilename] = locationlookup
# Read correlation ranges for this item
ranges = LocalCorrelationRangeRawBinaryReader().read(
descriptor.correlationrangesfilename)
# Observation files for each observable
filespecs = {
descriptor.observable:
ObservableFileSpec(descriptor.observationfilename, ranges)
}
# Load as observation source
filesource = ObservationSourceSingleDayRawBinary(
locationlookup, filespecs, daynumber)
# Show stats
print_stats(filesource, descriptor.sourcename, descriptor.observable)
# Connector for gridding this
connector = DiagnosticGridObservationConnector(axes, filesource)
# Grid each observable
dailydata = connector.get_day(descriptor.observable, daynumber)
outputgrid.create_fields_from_sparse_observations(
descriptor.sourcename, dailydata)
outputgrid.compute_weighted_mean(descriptor.sourcename,
descriptor.observable)
# Store result
ensuredirectory(outputfilename)
saver = NetCDFSaverEUSTACE(outputfilename)
saver.write_cubes(outputgrid)
def write_day(self, pathname, location_uuid, obs, daynumber):
"""Write the specified object (one day of data)."""
# create directory if we don't already have it
ensuredirectory(pathname)
# open it
file_handle = open(pathname, 'wb')
# header
file_handle.write(self.formatid)
file_handle.write(CODEID)
file_handle.write(location_uuid.bytes)
# data
valid = numpy.nonzero(
numpy.logical_and(obs.mask == False,
obs.time.astype(numpy.int32) == daynumber))[0]
obscount = valid.shape[0]
corcount = len(obs.locallycorrelatederror)
# allocate output buffer for writing
result = numpy.empty(obscount,
dtype=ObservationRawBinaryWriter.build_dtype(
self.basecategory, corcount))
# populate output structure from observation info
self.populate_output_structure(result, obs, valid, corcount)
# store
result.tofile(file_handle)
# done
file_handle.close()
del file_handle
def write(self, pathname, ranges):
"""Write locations to file handle."""
ensuredirectory(pathname)
file_handle = open(pathname, 'wb')
file_handle.write(FORMATID_LOCAL_CORRELATION_RANGE)
file_handle.write(CODEID)
result = numpy.empty(len(ranges),
dtype=DTYPE[CATEGORY_LOCAL_CORRELATION_RANGE])
result['range'] = ranges
result.tofile(file_handle)
file_handle.close()
def input_summary(time, output, inputsources):
"""Extract and store information required to produce a set of inputloaders at a time step
Stored in a json file as [isotime, [inputs_0,...,input_n]]
"""
from eumopps.catalogue.fileio.formatjson import CatalogueWriterJSON
ensuredirectory(output)
outputfile = open(output, 'w')
CatalogueWriterJSON().save(output, [time.isoformat(), inputsources])
def merge_input_summaries(summary_files, output):
"""Concatenates lists of inputs into a single json file"""
from eumopps.catalogue.fileio.formatjson import CatalogueReaderJSON, CatalogueWriterJSON
list_of_inputs = []
for filename in summary_files:
input_date, input_definition = CatalogueReaderJSON().load(filename)
list_of_inputs.append( [input_date, input_definition] )
ensuredirectory(output)
CatalogueWriterJSON().save(output, list_of_inputs)
def submit_jobs():
#eustace, par-single, high-mem
level = 7
neighbourhood_level = 3
regionspec = 'LocalSubRegion'
#number_of_regions = 1
number_of_regions = n_operations(neighbourhood_level)
#number_of_regions = 20
outdir = '/work/scratch/cmorice/advanced_standard/optimise/'
ensuredirectory(outdir)
for region_index in range(number_of_regions):
logfile = os.path.join(outdir, 'optimize.'+str(level)+'.'+str(neighbourhood_level)+'.'+str(region_index)+'.out')
errfile = os.path.join(outdir, 'optimize.'+str(level)+'.'+str(neighbourhood_level)+'.'+str(region_index)+'.err')
jobfile = os.path.join(outdir, 'optimize.'+str(level)+'.'+str(neighbourhood_level)+'.'+str(region_index)+'.sh')
pycommand = ' '.join([ 'python2.7',
#'-m eustace.analysis.advanced_standard.examples.example_eustace_few_days_optimization',
'-m eustace.analysis.advanced_standard.examples.optimise',
'--neighbourhood_level', str(neighbourhood_level),
'--region_index', str(region_index),
'--regionspec', regionspec])
job_spec = ''.join([ '#!/bin/bash\n',
'#BSUB -c 2:00\n',
'#BSUB -W 2:00\n',
#'#BSUB -R "rusage[mem=10]"',
'#BSUB -n 1\n',
'#BSUB -q short-serial\n',
'#BSUB -oo', logfile+'\n',
'#BSUB -eo', errfile+'\n',
'module load intel/cce/15.0.090\n',
'module load intel/mkl/11.3.1.150\n',
pycommand+'\n',])
with open(jobfile, 'w') as f:
f.write(job_spec)
run_string = 'bsub < '+jobfile
print "runing:"
print run_string
os.system(run_string)
def write(self, pathname, data):
"""Write locations to file handle."""
ensuredirectory(pathname)
# open it
file_handle = open(pathname, 'wb')
# header including lookup table UUID
file_handle.write(FORMATID_LOCATION_LOOKUP)
file_handle.write(CODEID)
file_handle.write(data.uuid.bytes)
# lookup table
result = numpy.empty(data.lookuptable.shape[1],
dtype=DTYPE[CATEGORY_LOCATION_LOOKUP])
result['latitude'] = data.lookuptable[0, :]
result['longitude'] = data.lookuptable[1, :]
result.tofile(file_handle)
# done
file_handle.close()
del file_handle
def writepickle(quantity, filename):
ensuredirectory(filename)
with open(filename, 'wb') as fh:
cPickle.dump(quantity, fh, protocol=2)
def save_model(outdict, modelfile):
# load a bias model pickle file
ensuredirectory(modelfile)
with open(modelfile, 'w') as f:
pickle.dump(outdict, f)
def local_optimisation(neighbourhood_level=0,
region_index=0,
regionspec='LocalSubRegion'):
"""Run the optimisation outside of EUMOPPS with manually setup analysis system"""
from eustace.analysis.advanced_standard.components.storage_files import SpaceTimeComponentSolutionStorage_Files, DelayedSpatialComponentSolutionStorageFlexible_Files
from eustace.analysis.advanced_standard.components.storage_inmemory import SpatialComponentSolutionStorage_InMemory
# climatology storage
climatology_dir = "/work/scratch/cmorice/advanced_standard/solution_climatology/"
climatology_state_file = os.path.join(climatology_dir,
"solution_climatology.pickle")
#climatology_state_file = None
storage_climatology = SpaceTimeComponentSolutionStorage_Files(
statefilename_read=climatology_state_file)
# large scale storage
large_scale_dir = "/work/scratch/cmorice/advanced_standard/solution_large_scale/"
large_scale_state_file = os.path.join(large_scale_dir,
"solution_large_scale.pickle")
#large_scale_state_file = None
storage_large_scale = SpaceTimeComponentSolutionStorage_Files(
statefilename_read=large_scale_state_file)
# local bias storage
local_bias_read_pattern = [
"/work/scratch/cmorice/advanced_standard/optimisation_local_bias/%Y/",
"solution_bias_%Y%m%d.pickle"
]
storage_local_bias = DelayedSpatialComponentSolutionStorageFlexible_Files(
state_read_pattern=local_bias_read_pattern)
# local spde storage
storage_region_spde = SpatialComponentSolutionStorage_InMemory()
# climatology covariates
covariates_descriptor = "/gws/nopw/j04/eustace/data/internal/climatology_covariates/covariates.json"
# bias terms
insitu_biases = 1
breakpoints_file = "/gws/nopw/j04/eustace/data/internal/D1.7/daily/eustace_stations_global_R001127_daily_status.nc"
global_biases = 1
global_biases_group_list = [
"surfaceairmodel_ice_global", "surfaceairmodel_land_global",
"surfaceairmodel_ocean_global"
]
# uncertainty calculation
compute_uncertainties = False
method = 'APPROXIMATED'
# observation descriptor json file
input_dir = "/work/scratch/cmorice/advanced_standard/local_hyperparameters/"
inputdescriptor_file = os.path.join(input_dir, "merged_input_summary.json")
inputdescriptor = read_inputdescriptor(inputdescriptor_file)
#print inputdescriptor
# list of time steps used in optimisation
time_keys = [
"2009-12-25T00:00:00",
"2009-12-26T00:00:00",
"2009-12-27T00:00:00",
] # "2009-12-28T00:00:00", "2009-12-29T00:00:00", "2009-12-30T00:00:00", "2009-12-31T00:00:00"]
time_keys = [
"2003-01-01T00:00:00",
"2003-02-01T00:00:00",
"2003-03-01T00:00:00",
"2003-04-01T00:00:00",
"2003-05-01T00:00:00",
"2003-06-01T00:00:00",
"2003-07-01T00:00:00",
"2003-08-01T00:00:00",
"2003-09-01T00:00:00",
"2003-10-01T00:00:00",
"2003-11-01T00:00:00",
"2003-12-01T00:00:00",
"2004-01-01T00:00:00",
"2004-02-01T00:00:00",
"2004-03-01T00:00:00",
"2004-04-01T00:00:00",
"2004-05-01T00:00:00",
"2004-06-01T00:00:00",
"2004-07-01T00:00:00",
"2004-08-01T00:00:00",
"2004-09-01T00:00:00",
"2004-10-01T00:00:00",
"2004-11-01T00:00:00",
"2004-12-01T00:00:00",
]
# output hyperparameter file
output_dir = "/work/scratch/cmorice/advanced_standard/local_optimisation/"
ensuredirectory(output_dir)
local_setup = moving_climatology.ShortScaleSetup()
hyperparameter_storage_file = os.path.join(
output_dir, "regional_hyperparameters.%i.%i.%i.npy" %
(local_setup.local_settings.n_triangulation_divisions,
neighbourhood_level, region_index))
compute_sample = False
sample_size = 0
# run the optimisation for this region - model configuration is a combination of storage and ancilliary files specified here and that imported by optimize_region
optimize_region(storage_climatology, storage_large_scale,
storage_local_bias, storage_region_spde,
covariates_descriptor, insitu_biases, breakpoints_file,
global_biases, global_biases_group_list,
compute_uncertainties, method, compute_sample, sample_size,
neighbourhood_level, region_index, regionspec,
inputdescriptor, time_keys, hyperparameter_storage_file)
