def col_cmd(main_arguments):
"""
Main routine for handling calls to the collocate ('col') command.
:param main_arguments: The command line arguments (minus the col command)
"""
from cis.collocation.col_framework import get_kernel
from cis.parse import check_boolean
# Read the sample data
missing_data_for_missing_sample = False
if main_arguments.samplevariable is not None:
sample_data = DataReader().read_data_list(main_arguments.samplefiles, main_arguments.samplevariable,
main_arguments.sampleproduct)[0]
missing_data_for_missing_sample = True
else:
sample_data = DataReader().read_coordinates(main_arguments.samplefiles, main_arguments.sampleproduct)
# Unpack the sample options
col_name, col_options = main_arguments.samplegroup.get('collocator', ('', {}))
kern_name, kern_options = main_arguments.samplegroup.get('kernel', ('', {}))
missing_data_for_missing_sample = check_boolean(col_options.pop('missing_data_for_missing_sample',
str(missing_data_for_missing_sample)), logging)
kernel = get_kernel(kern_name)(**kern_options) if kern_name else None
for input_group in main_arguments.datagroups:
# Then collocate each datagroup
data = DataReader().read_single_datagroup(input_group)
output = data.collocated_onto(sample_data, how=col_name, kernel=kernel,
missing_data_for_missing_sample=missing_data_for_missing_sample, **col_options)
output.save_data(main_arguments.output)
def _get_collocator_classes_for_method(self, method_name, kernel_name, sample_gridded, data_gridded):
"""
Gets the collocator, constraint and kernel classes corresponding to a specified collocation method and kernel
name.
:param method_name: collocation method name
:param kernel_name: kernel name
:param sample_gridded: True if sample points are gridded, otherwise False
:param data_gridded: True if data points are gridded, otherwise False
:return: CollocationOptions containing relevant classes
"""
import cis.collocation.col_implementations as ci
from cis.collocation.col_framework import get_kernel
method_name = self.get_default_collocator_name(method_name, sample_gridded, data_gridded)
key = '_'.join([method_name, str(sample_gridded), str(data_gridded)])
# Method Sample Data collocator constraint kernel
# name gridded gridded
options = {
'lin_False_False': None,
'lin_True_False': None,
'lin_False_True': [ci.GeneralUngriddedCollocator, None, ci.li],
'lin_True_True': [ci.GriddedCollocator, None, ci.gridded_gridded_li],
'nn_False_False': None,
'nn_True_False': None,
'nn_False_True': [ci.GeneralUngriddedCollocator, None, ci.nn_gridded],
'nn_True_True': [ci.GriddedCollocator, None, ci.gridded_gridded_nn],
'bin_False_False': None,
'bin_True_False': [ci.GeneralGriddedCollocator, ci.BinnedCubeCellOnlyConstraint, _GenericKernel],
'bin_False_True': None,
'bin_True_True': None,
'box_False_False': [ci.GeneralUngriddedCollocator, ci.SepConstraintKdtree, _GenericKernel],
'box_True_False': [ci.GeneralGriddedCollocator, ci.SepConstraintKdtree, _GenericKernel],
'box_False_True': [ci.GeneralUngriddedCollocator, ci.SepConstraintKdtree, _GenericKernel],
'box_True_True': [ci.GeneralGriddedCollocator, ci.SepConstraintKdtree, _GenericKernel],
'dummy_False_False': [ci.DummyCollocator, None, None],
'dummy_True_False': None,
'dummy_False_True': None,
'dummy_True_True': None
}
option = options.get(key)
if option is None:
raise InvalidCommandLineOptionError("Collocator/kernel/data type combination is not compatible")
if option[2] is _GenericKernel:
if kernel_name is None:
kernel_name = 'moments'
option[2] = get_kernel(kernel_name)
else:
if kernel_name is not None:
raise InvalidCommandLineOptionError(
'A kernel cannot be specified for collocator "{}"'.format(method_name))
return option
def test_GIVEN_ungridded_data_WHEN_collapse_THEN_calculations_correct(self):
grid = {'y': slice(-12.5, 12.5, 12.5)}
data = mock.make_regular_2d_ungridded_data()
kernel_class = get_kernel('moments')
kernel = kernel_class()
result = data.aggregate(how=kernel, **grid)
expected_means = numpy.array([3.5, 11])
expected_std_dev = numpy.array([numpy.sqrt(3.5), numpy.sqrt(7.5)])
expected_no = numpy.array([6, 9])
assert_that(len(result), is_(3))
assert_arrays_almost_equal(result[0].data.flatten(), expected_means)
assert_arrays_almost_equal(result[1].data.flatten(), expected_std_dev)
assert_that(numpy.array_equal(result[2].data.flatten(), expected_no))
def get_kernel(kernel, default=moments):
"""
Return a valid kernel instance from either an instance or a string, default is moments if no kernel is specified
:param str or cis.collocation.col_framework.Kernel kernel:
:param default:
:return cis.collocation.col_framework.Kernel:
"""
from cis.collocation.col_framework import get_kernel, Kernel
if not kernel:
kernel = default()
elif isinstance(kernel, six.string_types):
kernel = get_kernel(kernel)()
elif not isinstance(kernel, Kernel):
raise ValueError("Invalid kernel argument, it must be either a string or a Kernel instance")
return kernel
def get_kernel(kernel, default=moments):
"""
Return a valid kernel instance from either an instance or a string, default is moments if no kernel is specified
:param str or cis.collocation.col_framework.Kernel kernel:
:param default:
:return cis.collocation.col_framework.Kernel:
"""
from cis.collocation.col_framework import get_kernel, Kernel
if not kernel:
kernel = default()
elif isinstance(kernel, six.string_types):
kernel = get_kernel(kernel)()
elif not isinstance(kernel, Kernel):
raise ValueError(
"Invalid kernel argument, it must be either a string or a Kernel instance"
)
return kernel
def aggregate(self, variables, filenames, product=None, kernel=None):
"""
Aggregate the given variables based on the initialised grid
:param variables: One or more variables to read from the files
:type variables: string or list
:param filenames: One or more filenames of the files to read
:type filenames: string or list
:param str product: Name of data product to use (optional)
:param str kernel: Name of kernel to use (the default is 'moments')
"""
# Set the default kernel here rather than in the signature as the input_group dict passes None by default
kernel_name = kernel or 'moments'
# Read the input data - the parser limits the number of data groups to one for this command.
try:
# Read the data into a data object (either UngriddedData or Iris Cube), concatenating data from
# the specified files.
logging.info("Reading data for variables: %s", variables)
data = self._data_reader.read_data_list(filenames, variables, product)
except (IrisError, InvalidVariableError) as e:
raise CISError("There was an error reading in data: \n" + str(e))
except IOError as e:
raise CISError("There was an error reading one of the files: \n" + str(e))
aggregator = self._create_aggregator(data, self._grid)
if isinstance(data, iris.cube.CubeList):
kernel_inst = cube_aggregation_kernels[kernel_name]
data = aggregator.aggregate_gridded(kernel_inst)
else:
kernel_class = get_kernel(kernel_name)
kernel_inst = kernel_class()
data = aggregator.aggregate_ungridded(kernel_inst)
# TODO Tidy up output of grid in the history
history = "Aggregated using CIS version " + __version__ + \
"\n variables: " + str(variables) + \
"\n from files: " + str(filenames) + \
"\n using new grid: " + str(self._grid) + \
"\n with kernel: " + kernel_name + "."
data.add_history(history)
self._data_writer.write_data(data, self._output_file)
def col_cmd(main_arguments):
"""
Main routine for handling calls to the collocate ('col') command.
:param main_arguments: The command line arguments (minus the col command)
"""
from cis.collocation.col_framework import get_kernel
from cis.parse import check_boolean
# Read the sample data
missing_data_for_missing_sample = False
if main_arguments.samplevariable is not None:
sample_data = DataReader().read_data_list(
main_arguments.samplefiles, main_arguments.samplevariable,
main_arguments.sampleproduct)[0]
missing_data_for_missing_sample = True
else:
sample_data = DataReader().read_coordinates(
main_arguments.samplefiles, main_arguments.sampleproduct)
# Unpack the sample options
col_name, col_options = main_arguments.samplegroup.get(
'collocator', ('', {}))
kern_name, kern_options = main_arguments.samplegroup.get(
'kernel', ('', {}))
missing_data_for_missing_sample = check_boolean(
col_options.pop('missing_data_for_missing_sample',
str(missing_data_for_missing_sample)), logging)
kernel = get_kernel(kern_name)(**kern_options) if kern_name else None
for input_group in main_arguments.datagroups:
# Then collocate each datagroup
data = DataReader().read_single_datagroup(input_group)
output = data.collocated_onto(
sample_data,
how=col_name,
kernel=kernel,
missing_data_for_missing_sample=missing_data_for_missing_sample,
**col_options)
output.save_data(main_arguments.output)
def test_GIVEN_ungridded_data_WHEN_collapse_THEN_metadata_correct(self):
grid = {'y': slice(-10, 10, 10)}
data = mock.make_regular_2d_ungridded_data()
kernel_class = get_kernel('moments')
kernel = kernel_class()
result = data.aggregate(how=kernel, **grid)
mean, stddev, num = result
assert_that(mean.standard_name, is_('rainfall_rate'))
assert_that(stddev.standard_name, is_(None))
assert_that(num.standard_name, is_(None))
assert_that(mean.long_name, is_('TOTAL RAINFALL RATE: LS+CONV KG/M2/S'))
assert_that(stddev.long_name,
is_('Corrected sample standard deviation of TOTAL RAINFALL RATE: LS+CONV KG/M2/S'))
assert_that(num.long_name, is_('Number of points used to calculate the mean of '
'TOTAL RAINFALL RATE: LS+CONV KG/M2/S'))
assert_that(mean.var_name, is_('rain'))
assert_that(stddev.var_name, is_('rain_std_dev'))
assert_that(num.var_name, is_('rain_num_points'))
assert_that(mean.units, is_('kg m-2 s-1'))
assert_that(stddev.units, is_('kg m-2 s-1'))
assert_that(num.units, is_(None))
