def load_masked_data_lazy(sub_tile_slice: Tuple[slice, slice, slice],
source_prod: DataSource,
reverse=False, src_idx=None, timer=None) -> xarray.Dataset:
data_fuse_func = import_function(source_prod.spec['fuse_func']) if 'fuse_func' in source_prod.spec else None
data_tile = source_prod.data[sub_tile_slice]
data_measurements = source_prod.spec.get('measurements')
mask_nodata = source_prod.spec.get('mask_nodata', True)
mask_inplace = source_prod.spec.get('mask_inplace', False)
masks = []
if 'masks' in source_prod.spec:
for mask_spec, mask_tile in zip(source_prod.spec['masks'], source_prod.masks):
flags = mask_spec['flags']
mask_fuse_func = import_function(mask_spec['fuse_func']) if 'fuse_func' in mask_spec else None
opts = dict(skip_broken_datasets=True,
fuse_func=mask_fuse_func,
measurements=[mask_spec['measurement']])
masks.append((mask_tile[sub_tile_slice], flags, opts))
return load_masked_tile_lazy(data_tile,
masks,
mask_nodata=mask_nodata,
mask_inplace=mask_inplace,
reverse=reverse,
src_idx=src_idx,
timer=timer,
fuse_func=data_fuse_func,
measurements=data_measurements,
skip_broken_datasets=True)
def load_masked_data(sub_tile_slice: Tuple[slice, slice, slice],
source_prod: DataSource,
geom=None) -> xarray.Dataset:
data_fuse_func = import_function(
source_prod.spec['fuse_func']
) if 'fuse_func' in source_prod.spec else None
data = GridWorkflow.load(source_prod.data[sub_tile_slice],
measurements=source_prod.spec.get('measurements'),
fuse_func=data_fuse_func,
skip_broken_datasets=True)
mask_inplace = source_prod.spec.get('mask_inplace', False)
mask_nodata = source_prod.spec.get('mask_nodata', True)
if mask_nodata:
data = sensible_mask_invalid_data(data)
# if all NaN
completely_empty = all(
ds for ds in xarray.ufuncs.isnan(data).all().data_vars.values())
if completely_empty:
# Discard empty slice
return None
if mask_inplace or not mask_nodata:
where = sensible_where_inplace
else:
where = sensible_where
if 'masks' in source_prod.spec:
for mask_spec, mask_tile in zip(source_prod.spec['masks'],
source_prod.masks):
if mask_tile is None:
# Discard data due to no mask data
return None
mask_fuse_func = import_function(
mask_spec['fuse_func']) if 'fuse_func' in mask_spec else None
mask = GridWorkflow.load(
mask_tile[sub_tile_slice],
measurements=[mask_spec['measurement']],
fuse_func=mask_fuse_func,
skip_broken_datasets=True)[mask_spec['measurement']]
data = where(data, make_mask_from_spec(mask, mask_spec))
del mask
if geom is not None:
data = where(data, geometry_mask([geom], data.geobox, invert=True))
if source_prod.source_index is not None:
data.coords['source'] = ('time',
np.repeat(source_prod.source_index,
data.time.size))
return data
def load_masked_data(sub_tile_slice: Tuple[slice, slice, slice],
source_prod: DataSource) -> xarray.Dataset:
data_fuse_func = import_function(source_prod.spec['fuse_func']) if 'fuse_func' in source_prod.spec else None
data = GridWorkflow.load(source_prod.data[sub_tile_slice],
measurements=source_prod.spec.get('measurements'),
fuse_func=data_fuse_func,
skip_broken_datasets=True)
mask_inplace = source_prod.spec.get('mask_inplace', False)
mask_nodata = source_prod.spec.get('mask_nodata', True)
if mask_nodata:
data = sensible_mask_invalid_data(data)
# if all NaN
completely_empty = all(ds for ds in xarray.ufuncs.isnan(data).all().data_vars.values())
if completely_empty:
# Discard empty slice
return None
if 'masks' in source_prod.spec:
for mask_spec, mask_tile in zip(source_prod.spec['masks'], source_prod.masks):
if mask_tile is None:
# Discard data due to no mask data
return None
mask_fuse_func = import_function(mask_spec['fuse_func']) if 'fuse_func' in mask_spec else None
mask = GridWorkflow.load(mask_tile[sub_tile_slice],
measurements=[mask_spec['measurement']],
fuse_func=mask_fuse_func,
skip_broken_datasets=True)[mask_spec['measurement']]
if mask_spec.get('flags') is not None:
mask = make_mask(mask, **mask_spec['flags'])
elif mask_spec.get('less_than') is not None:
less_than = float(mask_spec['less_than'])
mask = mask < less_than
elif mask_spec.get('greater_than') is not None:
greater_than = float(mask_spec['greater_than'])
mask = mask > greater_than
if mask_inplace:
data = sensible_where_inplace(data, mask)
else:
data = sensible_where(data, mask)
del mask
if source_prod.source_index is not None:
data.coords['source'] = ('time', np.repeat(source_prod.source_index, data.time.size))
return data
def resolve_func(key, value):
if key not in ['fuse_func', 'dataset_predicate']:
return value
if callable(value):
return value
try:
return import_function(value)
except (ImportError, AttributeError):
raise VirtualProductException("could not resolve function {} in {}".format(key, recipe))
def lookup(name, namespace=None, kind='transformation'):
if callable(name):
return name
if namespace is not None and namespace in self.lookup_table and name in self.lookup_table[namespace]:
result = self.lookup_table[namespace][name]
else:
try:
result = import_function(name)
except (ImportError, AttributeError):
msg = "could not resolve {} {} in {}".format(kind, name, recipe)
raise VirtualProductException(msg)
self._assert(callable(result), "{} not callable in {}".format(kind, recipe))
return result
def resolve_transform(cls_name):
if callable(cls_name):
return cls_name
if cls_name in self.lookup_table:
cls = self.lookup_table[cls_name]
else:
try:
cls = import_function(cls_name)
except (ImportError, AttributeError):
msg = "could not resolve transformation {} in {}".format(cls_name, recipe)
raise VirtualProductException(msg)
if not callable(cls):
raise VirtualProductException("transformation not callable in {}".format(recipe))
return cls
def create_process(process, input, **settings):
process_class = import_function(process)
return process_class(input=construct(**input), **settings)