def test_update_single_file(self):
# Testing the update of single files
outputs = [
tempfile.NamedTemporaryFile(
prefix=
'eustace.outputformats.test.FIELD_SURFACE_eustace_PRODUCTIONUMBER_LABEL_XXXX',
suffix='.nc',
delete=False) for index in range(2)
]
dataset = [Dataset(output.name, 'w', 'NETCDF4') for output in outputs]
for files in dataset:
files.comment = ''
files.close()
v = OutputVariable.from_template(t,
'bob',
'Bob\'s House',
ancillary_variables='Bob\'s Shed')
merger = FilesMerger(['a', 'b', 'c'], ['d', 'e', 'f'],
numpy.array([1, 2]), [v], [v], [v], [[1.]])
merger.update_single_file([outputs[0].name], [outputs[1].name])
new_outputs = [output.name.replace('_LABEL', '') for output in outputs]
dataset = [Dataset(name, 'r', 'NETCDF4') for name in new_outputs]
for files in dataset:
self.assertEqual(files.ncattrs(), ['comment'])
self.assertEqual(
files.getncattr(files.ncattrs()[0]),
u'input data preprocessed from LABEL retrievals.')
files.close()
for name in new_outputs:
os.remove(name)
def test_init(self):
#Test the correcteness of the init procedure
v = OutputVariable.from_template(t,
'bob',
'Bob\'s House',
ancillary_variables='Bob\'s Shed')
self.assertRaises(ValueError, FilesMerger, 1, ['a'], numpy.array([1]),
[v], [v], [v], [[1.]])
self.assertRaises(ValueError, FilesMerger, ['a'], 1, numpy.array([1]),
[v], [v], [v], [[1.]])
self.assertRaises(ValueError, FilesMerger, ['a'], ['a'], [1], [v], [v],
[v], [v])
self.assertRaises(ValueError, FilesMerger, [1], ['a'],
numpy.array([1]), [v], [v], [v], [[1.]])
self.assertRaises(ValueError, FilesMerger, ['a'], [1],
numpy.array([1]), [v], [v], [v], [[1.]])
self.assertRaises(ValueError, FilesMerger, ['a'], ['a'], ['a'], [v],
[v], [v], [[1.]])
self.assertRaises(ValueError, FilesMerger, ['a'], ['a'], ['a'], [v],
[v], [v], [[1.]])
self.assertRaises(ValueError, FilesMerger, ['a'], ['a', 'b'],
numpy.array([1]), [v], [v], [v], [[1.]])
self.assertRaises(ValueError, FilesMerger, ['a'], ['a', 'b'],
numpy.array([1]), [v], [v], [v], [[1.]])
self.assertRaises(ValueError, FilesMerger, ['a', 'b'], ['a', 'b'],
numpy.array([1]), [v], [v], [v], [[1.]])
self.assertRaises(ValueError, FilesMerger, ['a'], ['a', 'b'],
numpy.array([1]), [v], [v], [v], [[1.]])
self.assertRaises(ValueError, FilesMerger, ['a', 'b'], ['a', 'b'],
numpy.array([1]), [v], [v], [v], [[1.]])
self.assertRaises(ValueError, FilesMerger, ['a', 'b'], ['a', 'b'],
numpy.array([2]), 1, [v], [v], [[1.]])
self.assertRaises(ValueError, FilesMerger, ['a', 'b'], ['a', 'b'],
numpy.array([2]), [v], 1, [v], [[1.]])
self.assertRaises(ValueError, FilesMerger, ['a', 'b'], ['a', 'b'],
numpy.array([2]), [v], [v], 1, [[1.]])
self.assertRaises(ValueError, FilesMerger, ['a', 'b'], ['a', 'b'],
numpy.array([2]), [v], [v], [v], 1.)
self.assertRaises(ValueError, FilesMerger, ['a', 'b'], ['a', 'b'],
numpy.array([2]), [v], [v], [v], [[True]])
merger = FilesMerger(['a', 'b', 'c'], ['d', 'e', 'f'],
numpy.array([1, 2]), [v], [v], [v], [[1.]])
self.assertEqual(merger.outputs_primary, ['a', 'b', 'c'])
self.assertEqual(merger.outputs_ancillary, ['d', 'e', 'f'])
self.assertEqual(merger.list_of_daily_sources[0], 1)
self.assertEqual(merger.list_of_daily_sources[1], 2)
self.assertEqual(merger.primary_fields, [v])
self.assertEqual(merger.primary_uncertainties, [v])
self.assertEqual(merger.ancillary_fields, [v])
self.assertEqual(merger.list_of_correlation_indexes, [[1.]])
self.assertEqual(merger.list_of_merged_main_outputs, [])
self.assertEqual(merger.list_of_merged_ancillary_outputs, [])
def test_read_fields(self):
# Make test data
testfile = tempfile.NamedTemporaryFile(
prefix='eustace.surfaceairmodel.fileio.test.test_remap.',
suffix='.nc')
testdata = FileBuilderGlobalField(testfile.name, 10000, 'testdata',
'NOVERSION', 'jam', 'someplace', '',
'', '')
testdata.add_global_field(
OutputVariable('jam', numpy.float32, -2000),
182.2 * numpy.ones(definitions.GLOBAL_FIELD_SHAPE))
testdata.add_global_field(
OutputVariable('apples', numpy.float32, -2000),
37.8 * numpy.ones(definitions.GLOBAL_FIELD_SHAPE))
# Remapper
remapper = RemapNetCDF([
RemapNetCDFSpecDayNumber(),
RemapNetCDFSpecCopy('cream', 'jam', -29.1),
RemapNetCDFSpecCopy('oranges', 'apples')
])
# Get results
results = remapper.read_fields(testfile.name)
# Should have 3 fields (oranges, cream, and the daynumber)
self.assertEqual(3, len(results))
# Check daynumber
self.assertEqual(10000, results['daynumber'])
# Check global fields are as expected
numpy.testing.assert_almost_equal(
results['cream'],
153.1 * numpy.ones(definitions.GLOBAL_FIELD_SHAPE[1:]),
decimal=4)
numpy.testing.assert_almost_equal(
results['oranges'],
37.8 * numpy.ones(definitions.GLOBAL_FIELD_SHAPE[1:]),
decimal=4)
def test_update_correlated_uncertainty_ranges(self):
A = CorrelationRanges(['test', 'b'], ['g', 'h'], [1.2, 3.67], [3, 4.4])
TEST_VARIABLE = TIME = OutputVariable(name='test',
dtype=int,
fill_value=0)
A.update_correlated_uncertainty_ranges(TEST_VARIABLE)
self.assertEqual(TEST_VARIABLE.length_scale, 1.2)
self.assertEqual(TEST_VARIABLE.time_scale, 3)
self.assertEqual(TEST_VARIABLE.length_scale_units, 'g')
self.assertEqual(TEST_VARIABLE.time_scale_units, 'h')
def add_uncertainty_parameter(self, name, long_name, values):
"""Add uncertainty parameter (must have global coverage)."""
self.add_variable(
OutputVariable(
name=name,
dtype=numpy.int16,
fill_value=definitions.TEMPERATURE_UNCERTAINTY_FILL_VALUE,
scale_factor=definitions.TEMPERATURE_UNCERTAINTY_SCALE_FACTOR,
add_offset=definitions.TEMPERATURE_UNCERTAINTY_ADD_OFFSET,
long_name=long_name,
units='K'),
(definitions.DIMENSION_NAME_TIME, definitions.DIMENSION_NAME_LATITUDE, definitions.DIMENSION_NAME_LONGITUDE),
values)
def add_variance_field(self, name, standard_name, long_name, values):
"""Add a field with units and precision set for variance (K2)."""
self.add_variable(OutputVariable(name=name,
dtype=numpy.int32,
fill_value=-1,
scale_factor=0.000005,
add_offset=0.0,
long_name=long_name,
standard_name=standard_name,
units='K2'),
(definitions.DIMENSION_NAME_LATITUDE,
definitions.DIMENSION_NAME_LONGITUDE),
numpy.ma.masked_array(
values, numpy.equal(values, FLAG_INVALID)),
complevel=self.complevel)
def add_field(self, name, standard_name, long_name, units, scale, offset,
values):
"""Add field (must have global coverage with respect to latitude and logitude axes)."""
# Allow a large number of arguments here
# - could consider having a namespace in future
# pylint: disable=too-many-arguments
self.add_variable(OutputVariable(name=name,
dtype=numpy.int16,
fill_value=-32768,
scale_factor=scale,
add_offset=offset,
long_name=long_name,
standard_name=standard_name,
units=units),
(definitions.DIMENSION_NAME_LATITUDE,
definitions.DIMENSION_NAME_LONGITUDE),
numpy.ma.masked_array(
values, numpy.equal(values, FLAG_INVALID)),
complevel=self.complevel)
def __init__(self, pathname, outputstructure):
"""Create with specified output structure."""
super(FileBuilder, self).__init__()
# get EUSTACE daynumber
daynumber = days_since_epoch(outputstructure.time_datetime())
# make the file
self.create(pathname, title='Test output', institution='', comment='Test output', history='', source='')
# time variable
time_variable = OutputVariable(
name='time',
dtype=numpy.float32,
fill_value=None,
standard_name='time',
long_name='Time',
units=definitions.TIME_UNITS_DAYS_SINCE_EPOCH,
calendar='gregorian',
axis='T')
# set time in days since the EUSTACE epoch [and set UNLIMITED]
self.add_dimension_and_variable(
dimensionname='time',
variable=time_variable,
values=numpy.array([daynumber], numpy.float32),
unlimited=True)
# global latitude axis
self.add_dimension_and_variable(
dimensionname=definitions.DIMENSION_NAME_LATITUDE,
variable=definitions.LATITUDE,
values=outputstructure.latitudes)
# global longitude axis
self.add_dimension_and_variable(
dimensionname=definitions.DIMENSION_NAME_LONGITUDE,
variable=definitions.LONGITUDE,
values=outputstructure.longitudes)
def save_flag_file(flag_values, processdate, outputfile):
"""Save the windowing flag information"""
print "Saving: ", outputfile
# Create directory structure if required
if not os.path.exists(os.path.dirname(outputfile)):
os.makedirs(os.path.dirname(outputfile))
# Make the QC flag file
filebuilder = FileBuilderGlobalField(
outputfile,
eustace.timeutils.epoch.days_since_epoch(processdate),
'EUSTACE Analysis Flags',
get_revision_id_for_module(eustace),
definitions.TAS.name,
'',
'Provisional output',
__name__,
'')
# flag flag definition
tas_qc_definition = OutputVariable( name='tas_qc',
dtype=FLAG_TYPE,
fill_value=default_fillvals[TYPE_NAME],
standard_name='air_temperature status_flag',
long_name='Quality control flags',
#valid_range=numpy.array([FLAG_TYPE(0), FLAG_TYPE(1)<<FLAG_MAX_N], FLAG_TYPE),
flag_masks=numpy.array([ FLAG_TYPE(1)<<n for n in range(FLAG_MAX_USED)], FLAG_TYPE),
flag_meanings=' '.join(FLAG_MEANINGS))
# Combine flag information and populate the output file
filebuilder.add_global_field( tas_qc_definition, flag_values.reshape(definitions.GLOBAL_FIELD_SHAPE) )
# Close the netCDF4 dataset
filebuilder.save_and_close()
"""Example output."""
from eustace.outputformats import definitions
from eustace.outputformats.filebuilder import FileBuilder
from eustace.outputformats.outputvariable import OutputVariable
from eustace.timeutils.epoch import days_since_epoch
import numpy
from netCDF4 import default_fillvals
TAS_ANOMALY = OutputVariable(
name='tas',
dtype=numpy.float32,
fill_value=default_fillvals['f4'],
standard_name='air_temperature',
long_name='near_surface_temperature_anomaly',
units='K')
class FileBuilderHadCRUT4ExampleOutput(FileBuilder):
"""Build a NetCDF file a bit like EUSTACE output format."""
def __init__(self, pathname, outputstructure):
"""Create with specified output structure."""
super(FileBuilder, self).__init__()
# get EUSTACE daynumber
daynumber = days_since_epoch(outputstructure.time_datetime())
# make the file
self.create(pathname, title='Test output', institution='', comment='Test output', history='', source='')
def setfields(self, fields):
"""Add entire dictionary of fields."""
# global latitude axis
self.add_dimension_and_variable(
dimensionname=definitions.DIMENSION_NAME_LATITUDE,
variable=definitions.LATITUDE,
values=fields[definitions.LATITUDE.name])
# global longitude axis
self.add_dimension_and_variable(
dimensionname=definitions.DIMENSION_NAME_LONGITUDE,
variable=definitions.LONGITUDE,
values=fields[definitions.LONGITUDE.name])
# count of observations used
self.add_variable(
OutputVariable(
name='ts_number_of_observations',
dtype=numpy.int16,
fill_value=None,
standard_name=None,
long_name='Number of observations of surface temperature',
units='1'), (definitions.DIMENSION_NAME_LATITUDE,
definitions.DIMENSION_NAME_LONGITUDE),
fields['ts_number_of_observations'], self.complevel)
# count of observations available (including cloudy)
self.add_variable(
OutputVariable(
name='total_number_of_observations',
dtype=numpy.int16,
fill_value=None,
standard_name=None,
long_name='Total number of satellite observations in grid box',
units='1'), (definitions.DIMENSION_NAME_LATITUDE,
definitions.DIMENSION_NAME_LONGITUDE),
fields['total_number_of_observations'], self.complevel)
# fields
self.add_temperature_field(name='tsmean',
standard_name='surface_temperature',
long_name='Mean surface temperature (K)',
values=fields['tsmean'])
self.add_temperature_field(name='tsmax',
standard_name='surface_temperature',
long_name='Maximum surface temperature (K)',
values=fields['tsmax'])
self.add_temperature_field(name='tsmin',
standard_name='surface_temperature',
long_name='Minimum surface temperature (K)',
values=fields['tsmin'])
self.add_variance_field(name='tsvariance',
standard_name=None,
long_name='Surface temperature variance (K2)',
values=fields['tsvariance'])
self.add_uncertainty_field(
name='tsmean_unc_ran',
standard_name=None,
long_name='Random uncertainty in mean surface temperature (K)',
values=fields['tsmean_unc_ran'])
self.add_uncertainty_field(
name='tsmean_unc_loc_atm',
standard_name=None,
long_name=
'Locally correlated uncertainty (atm component) in mean surface temperature (K)',
values=fields['tsmean_unc_loc_atm'])
self.add_uncertainty_field(
name='tsmean_unc_loc_sfc',
standard_name=None,
long_name=
'Locally correlated uncertainty (sfc component) in mean surface temperature (K)',
values=fields['tsmean_unc_loc_sfc'])
self.add_uncertainty_field(
name='tsmean_unc_sys',
standard_name=None,
long_name='Systematic uncertainty in mean surface temperature (K)',
values=fields['tsmean_unc_sys'])
self.add_uncertainty_field(
name='tsmean_unc_spl',
standard_name=None,
long_name='Sampling uncertainty in mean surface temperature (K)',
values=fields['tsmean_unc_spl'])
from eustace.outputformats.definitions import SURFACEAIRMODEL_ANCILLARY_OCEAN_TAS_UNCERTAINTY_LOCALLYCORRELATED2
from eustace.outputformats.definitions import SURFACEAIRMODEL_ANCILLARY_OCEAN_TAS_UNCERTAINTY_SYSTEMATIC2
from eustace.outputformats.definitions import SURFACEAIRMODEL_ANCILLARY_OCEAN_TAS_UNCERTAINTY_PARAMETER0
from eustace.outputformats.definitions import SURFACEAIRMODEL_ANCILLARY_OCEAN_TAS_UNCERTAINTY_PARAMETER1
from eustace.outputformats.definitions import SURFACEAIRMODEL_ANCILLARY_OCEAN_TAS_UNCERTAINTY_PARAMETER2
from eustace.outputformats.definitions import SURFACEAIRMODEL_ANCILLARY_OCEAN_TAS_UNCERTAINTY_PARAMETER3
from eustace.outputformats.definitions import SURFACEAIRMODEL_ANCILLARY_OCEAN_TAS_UNCERTAINTY_PARAMETER4
from netCDF4 import Dataset
from shutil import copyfile
# Output variable template
t = OutputVariableTemplate(numpy.float32, -3200, units='mm')
# Output variable objects created fro testing purposes
f1 = OutputVariable.from_template(t,
'field_1',
'first field',
ancillary_variables='Bob\'s Shed')
f2 = OutputVariable.from_template(t,
'field_2',
'second field',
ancillary_variables='Bob\'s Shed')
u1 = OutputVariable.from_template(t, 'total_uncertainty_field_1',
'first total uncertainty field')
u2 = OutputVariable.from_template(t, 'total_uncertainty_field_2',
'second total uncertainty field')
a11 = OutputVariable.from_template(t, 'ancillary_1_for_field_1',
'first ancillary variable for first field')
a12 = OutputVariable.from_template(
t, 'ancillary_1_for_field_2', 'second ancillary variable for first field')
a21 = OutputVariable.from_template(
t, 'ancillary_2_for_field_1', 'first ancillary variable for second field')