def load_cubes(filenames, user_callback, loader, filter_function=None):
concrete_reference_targets = {}
results_needing_reference = []
if isinstance(filenames, six.string_types):
filenames = [filenames]
for filename in filenames:
for field in loader.field_generator(filename,
**loader.field_generator_kwargs):
# evaluate field against format specific desired attributes
# load if no format specific desired attributes are violated
if filter_function is not None and not filter_function(field):
continue
# Convert the field to a Cube.
cube, factories, references = _make_cube(field, loader.converter)
# Run any custom user-provided rules.
if loader.legacy_custom_rules:
warnings.warn('The `legacy_custom_rules` attribute of '
'the `loader` is deprecated.')
loader.legacy_custom_rules.verify(cube, field)
cube = iris.io.run_callback(user_callback, cube, field, filename)
if cube is None:
continue
# Cross referencing
for reference in references:
name = reference.name
# Register this cube as a source cube for the named
# reference.
target = concrete_reference_targets.get(name)
if target is None:
target = ConcreteReferenceTarget(name, reference.transform)
concrete_reference_targets[name] = target
target.add_cube(cube)
if factories:
results_needing_reference.append((cube, factories))
else:
yield cube
regrid_cache = {}
for cube, factories in results_needing_reference:
for factory in factories:
try:
args = _dereference_args(factory, concrete_reference_targets,
regrid_cache, cube)
except _ReferenceError as e:
msg = 'Unable to create instance of {factory}. ' + str(e)
factory_name = factory.factory_class.__name__
warnings.warn(
msg.format(filenames=filenames, factory=factory_name))
else:
aux_factory = factory.factory_class(*args)
cube.add_aux_factory(aux_factory)
yield cube
def load_cubes(filenames, user_callback, loader, filter_function=None):
concrete_reference_targets = {}
results_needing_reference = []
if isinstance(filenames, six.string_types):
filenames = [filenames]
for filename in filenames:
for field in loader.field_generator(filename, **loader.field_generator_kwargs):
# evaluate field against format specific desired attributes
# load if no format specific desired attributes are violated
if filter_function is not None and not filter_function(field):
continue
# Convert the field to a Cube.
cube, factories, references = _make_cube(field, loader.converter)
# Run any custom user-provided rules.
if loader.legacy_custom_rules:
warnings.warn('The `legacy_custom_rules` attribute of '
'the `loader` is deprecated.')
loader.legacy_custom_rules.verify(cube, field)
cube = iris.io.run_callback(user_callback, cube, field, filename)
if cube is None:
continue
# Cross referencing
for reference in references:
name = reference.name
# Register this cube as a source cube for the named
# reference.
target = concrete_reference_targets.get(name)
if target is None:
target = ConcreteReferenceTarget(name, reference.transform)
concrete_reference_targets[name] = target
target.add_cube(cube)
if factories:
results_needing_reference.append((cube, factories))
else:
yield cube
regrid_cache = {}
for cube, factories in results_needing_reference:
for factory in factories:
try:
args = _dereference_args(factory, concrete_reference_targets,
regrid_cache, cube)
except _ReferenceError as e:
msg = 'Unable to create instance of {factory}. ' + str(e)
factory_name = factory.factory_class.__name__
warnings.warn(msg.format(filenames=filenames,
factory=factory_name))
else:
aux_factory = factory.factory_class(*args)
cube.add_aux_factory(aux_factory)
yield cube
def load_cubes(filenames, user_callback, loader):
concrete_reference_targets = {}
results_needing_reference = []
if isinstance(filenames, basestring):
filenames = [filenames]
for filename in filenames:
for field in loader.field_generator(filename,
**loader.field_generator_kwargs):
# Convert the field to a Cube, logging the rules that were used
rules_result = loader.load_rules.result(field)
cube = rules_result.cube
log(loader.log_name, filename, rules_result.matching_rules)
cube = iris.io.run_callback(user_callback, cube, field, filename)
if cube is None:
continue
# Cross referencing
rules = loader.cross_ref_rules.matching_rules(field)
for rule in rules:
reference, = rule.run_actions(cube, field)
name = reference.name
# Register this cube as a source cube for the named
# reference.
target = concrete_reference_targets.get(name)
if target is None:
target = ConcreteReferenceTarget(name, reference.transform)
concrete_reference_targets[name] = target
target.add_cube(cube)
if rules_result.factories:
results_needing_reference.append(rules_result)
else:
yield cube
regrid_cache = {}
for result in results_needing_reference:
cube = result.cube
for factory in result.factories:
try:
args = _dereference_args(factory, concrete_reference_targets,
regrid_cache, cube)
except _ReferenceError as e:
msg = 'Unable to create instance of {factory}. ' + e.message
factory_name = factory.factory_class.__name__
warnings.warn(
msg.format(filenames=filenames, factory=factory_name))
else:
aux_factory = factory.factory_class(*args)
cube.add_aux_factory(aux_factory)
yield cube
def load_cubes(filenames, user_callback, loader):
concrete_reference_targets = {}
results_needing_reference = []
if isinstance(filenames, basestring):
filenames = [filenames]
for filename in filenames:
for field in loader.field_generator(filename):
# Convert the field to a Cube, logging the rules that were used
rules_result = loader.load_rules.result(field)
cube = rules_result.cube
log(loader.log_name, filename, rules_result.matching_rules)
cube = iris.io.run_callback(user_callback, cube, field, filename)
if cube is None:
continue
# Cross referencing
rules = loader.cross_ref_rules.matching_rules(field)
for rule in rules:
reference, = rule.run_actions(cube, field)
name = reference.name
# Register this cube as a source cube for the named
# reference.
target = concrete_reference_targets.get(name)
if target is None:
target = ConcreteReferenceTarget(name, reference.transform)
concrete_reference_targets[name] = target
target.add_cube(cube)
if rules_result.factories:
results_needing_reference.append(rules_result)
else:
yield cube
regrid_cache = {}
for result in results_needing_reference:
cube = result.cube
for factory in result.factories:
try:
args = _dereference_args(factory, concrete_reference_targets,
regrid_cache, cube)
except _ReferenceError as e:
msg = 'Unable to create instance of {factory}. ' + e.message
factory_name = factory.factory_class.__name__
warnings.warn(msg.format(filenames=filenames,
factory=factory_name))
else:
aux_factory = factory.factory_class(*args)
cube.add_aux_factory(aux_factory)
yield cube
def load_cubes(filenames, user_callback, loader):
concrete_reference_targets = {}
results_needing_reference = []
if isinstance(filenames, basestring):
filenames = [filenames]
for filename in filenames:
for field in loader.field_generator(filename,
**loader.field_generator_kwargs):
# Convert the field to a Cube.
cube, factories, references = _make_cube(field, loader.converter)
# Run any custom user-provided rules.
if loader.legacy_custom_rules:
loader.legacy_custom_rules.verify(cube, field)
cube = iris.io.run_callback(user_callback, cube, field, filename)
if cube is None:
continue
# Cross referencing
for reference in references:
name = reference.name
# Register this cube as a source cube for the named
# reference.
target = concrete_reference_targets.get(name)
if target is None:
target = ConcreteReferenceTarget(name, reference.transform)
concrete_reference_targets[name] = target
target.add_cube(cube)
if factories:
results_needing_reference.append((cube, factories))
else:
yield cube
regrid_cache = {}
for cube, factories in results_needing_reference:
for factory in factories:
try:
args = _dereference_args(factory, concrete_reference_targets,
regrid_cache, cube)
except _ReferenceError as e:
msg = 'Unable to create instance of {factory}. ' + e.message
factory_name = factory.factory_class.__name__
warnings.warn(
msg.format(filenames=filenames, factory=factory_name))
else:
aux_factory = factory.factory_class(*args)
cube.add_aux_factory(aux_factory)
yield cube
def load_cubes(filenames, user_callback, loader):
concrete_reference_targets = {}
results_needing_reference = []
if isinstance(filenames, basestring):
filenames = [filenames]
for filename in filenames:
for field in loader.field_generator(filename, **loader.field_generator_kwargs):
# Convert the field to a Cube.
cube, factories, references = _make_cube(field, loader.converter)
# Run any custom user-provided rules.
if loader.legacy_custom_rules:
loader.legacy_custom_rules.verify(cube, field)
cube = iris.io.run_callback(user_callback, cube, field, filename)
if cube is None:
continue
# Cross referencing
for reference in references:
name = reference.name
# Register this cube as a source cube for the named
# reference.
target = concrete_reference_targets.get(name)
if target is None:
target = ConcreteReferenceTarget(name, reference.transform)
concrete_reference_targets[name] = target
target.add_cube(cube)
if factories:
results_needing_reference.append((cube, factories))
else:
yield cube
regrid_cache = {}
for cube, factories in results_needing_reference:
for factory in factories:
try:
args = _dereference_args(factory, concrete_reference_targets,
regrid_cache, cube)
except _ReferenceError as e:
msg = 'Unable to create instance of {factory}. ' + e.message
factory_name = factory.factory_class.__name__
warnings.warn(msg.format(filenames=filenames,
factory=factory_name))
else:
aux_factory = factory.factory_class(*args)
cube.add_aux_factory(aux_factory)
yield cube
def _resolve_factory_references(cube, factories, concrete_reference_targets,
regrid_cache={}):
# Attach the factories for a cube, building them from references.
# Note: the regrid_cache argument lets us share and reuse regridded data
# across multiple result cubes.
for factory in factories:
try:
args = _dereference_args(factory, concrete_reference_targets,
regrid_cache, cube)
except _ReferenceError as e:
msg = 'Unable to create instance of {factory}. ' + str(e)
factory_name = factory.factory_class.__name__
warnings.warn(msg.format(factory=factory_name))
else:
aux_factory = factory.factory_class(*args)
cube.add_aux_factory(aux_factory)
def _resolve_factory_references(cube, factories, concrete_reference_targets,
regrid_cache={}):
# Attach the factories for a cube, building them from references.
# Note: the regrid_cache argument lets us share and reuse regridded data
# across multiple result cubes.
for factory in factories:
try:
args = _dereference_args(factory, concrete_reference_targets,
regrid_cache, cube)
except _ReferenceError as e:
msg = 'Unable to create instance of {factory}. ' + str(e)
factory_name = factory.factory_class.__name__
warnings.warn(msg.format(factory=factory_name))
else:
aux_factory = factory.factory_class(*args)
cube.add_aux_factory(aux_factory)
def _load_aux_factory(engine, cf, filename, cube):
"""
Convert any CF-netCDF dimensionless coordinate to an AuxCoordFactory.
"""
formula_type = engine.requires.get('formula_type')
if formula_type == 'atmosphere_hybrid_height_coordinate':
def coord_from_var_name(name):
mapping = engine.provides['coordinates']
for coord, cf_var_name in engine.provides['coordinates']:
if cf_var_name == name:
return coord
raise ValueError('Unable to find coordinate for variable {!r}'.format(name))
# Convert term names to coordinates (via netCDF variable names).
terms_to_var_names = engine.requires['formula_terms']
delta = coord_from_var_name(terms_to_var_names['a'])
sigma = coord_from_var_name(terms_to_var_names['b'])
orography = coord_from_var_name(terms_to_var_names['orog'])
factory = iris.aux_factory.HybridHeightFactory(delta, sigma, orography)
cube.add_aux_factory(factory)
def _get_cube(self):
"""
Returns a cube containing all its coordinates and appropriately shaped
data that corresponds to this ProtoCube.
All the values in the cube's data array are masked.
"""
signature = self._cube_signature
dim_coords_and_dims = [(deepcopy(coord), dim) for coord, dim in self._dim_coords_and_dims]
aux_coords_and_dims = [(deepcopy(coord), dims) for coord, dims in self._aux_coords_and_dims]
kwargs = dict(zip(iris.cube.CubeMetadata._fields, signature.defn))
# Create fully masked data, i.e. all missing.
# (The CubeML checksum doesn't respect the mask, so we zero the
# underlying data to ensure repeatable checksums.)
if signature.data_manager is None:
data = ma.MaskedArray(np.zeros(self._shape,
signature.data_type),
mask=np.ones(self._shape, 'bool'),
fill_value=signature.mdi)
else:
data = ma.MaskedArray(np.zeros(self._shape, 'object'),
mask=np.ones(self._shape, 'bool'))
cube = iris.cube.Cube(data,
dim_coords_and_dims=dim_coords_and_dims,
aux_coords_and_dims=aux_coords_and_dims,
data_manager=signature.data_manager, **kwargs)
# Add on any aux coord factories.
for factory_defn in self._coord_signature.factory_defns:
args = {}
for key, defn in factory_defn.dependency_defns:
coord = cube.coord(coord=defn)
args[key] = coord
factory = factory_defn.class_(**args)
cube.add_aux_factory(factory)
return cube
def _get_cube(self):
"""
Returns a cube containing all its coordinates and appropriately shaped
data that corresponds to this ProtoCube.
All the values in the cube's data array are masked.
"""
signature = self._cube_signature
dim_coords_and_dims = [(deepcopy(coord), dim) for coord, dim in self._dim_coords_and_dims]
aux_coords_and_dims = [(deepcopy(coord), dims) for coord, dims in self._aux_coords_and_dims]
kwargs = dict(zip(iris.cube.CubeMetadata._fields, signature.defn))
# Create fully masked data, i.e. all missing.
# (The CubeML checksum doesn't respect the mask, so we zero the
# underlying data to ensure repeatable checksums.)
if signature.data_manager is None:
data = ma.MaskedArray(np.zeros(self._shape,
signature.data_type),
mask=np.ones(self._shape, 'bool'),
fill_value=signature.mdi)
else:
data = ma.MaskedArray(np.zeros(self._shape, 'object'),
mask=np.ones(self._shape, 'bool'))
cube = iris.cube.Cube(data,
dim_coords_and_dims=dim_coords_and_dims,
aux_coords_and_dims=aux_coords_and_dims,
data_manager=signature.data_manager, **kwargs)
# Add on any aux coord factories.
for factory_defn in self._coord_signature.factory_defns:
args = {}
for key, defn in factory_defn.dependency_defns:
coord = cube.coord(coord=defn)
args[key] = coord
factory = factory_defn.class_(**args)
cube.add_aux_factory(factory)
return cube
def _load_aux_factory(engine, cf, filename, cube):
"""
Convert any CF-netCDF dimensionless coordinate to an AuxCoordFactory.
"""
formula_type = engine.requires.get('formula_type')
if formula_type == 'atmosphere_hybrid_height_coordinate':
def coord_from_var_name(name):
mapping = engine.provides['coordinates']
for coord, cf_var_name in engine.provides['coordinates']:
if cf_var_name == name:
return coord
raise ValueError('Unable to find coordinate for variable '
'{!r}'.format(name))
# Convert term names to coordinates (via netCDF variable names).
terms_to_var_names = engine.requires['formula_terms']
delta = coord_from_var_name(terms_to_var_names['a'])
sigma = coord_from_var_name(terms_to_var_names['b'])
orography = coord_from_var_name(terms_to_var_names['orog'])
factory = iris.aux_factory.HybridHeightFactory(delta, sigma, orography)
cube.add_aux_factory(factory)
def _get_cube(self):
"""
Returns a cube containing all its coordinates and appropriately shaped
data that corresponds to this ProtoCube.
All the values in the cube's data array are masked.
"""
signature = self._cube_signature
dim_coords_and_dims = [(deepcopy(coord), dim) for coord, dim in self._dim_coords_and_dims]
aux_coords_and_dims = [(deepcopy(coord), dims) for coord, dims in self._aux_coords_and_dims]
kwargs = dict(zip(iris.cube.CubeMetadata._fields, signature.defn))
# Create fully masked data i.e. all missing.
if signature.data_manager is None:
# Must zero the data in order to avoid random checksums.
data = numpy.ma.zeros(self._shape, dtype=signature.data_type)
data.fill_value = signature.mdi
else:
# With dtype=object, ma.empty DOES initialise the memory (all None).
data = numpy.ma.empty(self._shape, dtype=object)
data.mask = True
cube = iris.cube.Cube(data,
dim_coords_and_dims=dim_coords_and_dims,
aux_coords_and_dims=aux_coords_and_dims,
data_manager=signature.data_manager, **kwargs)
# Add on any aux coord factories.
for factory_defn in self._coord_signature.factory_defns:
args = {}
for key, defn in factory_defn.dependency_defns:
coord = cube.coord(coord=defn)
args[key] = coord
factory = factory_defn.class_(**args)
cube.add_aux_factory(factory)
return cube
