def load_masked_data_lazy(sub_tile_slice: Tuple[slice, slice, slice], source_prod: DataSource, reverse=False, src_idx=None, timer=None) -> xarray.Dataset: data_fuse_func = import_function(source_prod.spec['fuse_func']) if 'fuse_func' in source_prod.spec else None data_tile = source_prod.data[sub_tile_slice] data_measurements = source_prod.spec.get('measurements') mask_nodata = source_prod.spec.get('mask_nodata', True) mask_inplace = source_prod.spec.get('mask_inplace', False) masks = [] if 'masks' in source_prod.spec: for mask_spec, mask_tile in zip(source_prod.spec['masks'], source_prod.masks): flags = mask_spec['flags'] mask_fuse_func = import_function(mask_spec['fuse_func']) if 'fuse_func' in mask_spec else None opts = dict(skip_broken_datasets=True, fuse_func=mask_fuse_func, measurements=[mask_spec['measurement']]) masks.append((mask_tile[sub_tile_slice], flags, opts)) return load_masked_tile_lazy(data_tile, masks, mask_nodata=mask_nodata, mask_inplace=mask_inplace, reverse=reverse, src_idx=src_idx, timer=timer, fuse_func=data_fuse_func, measurements=data_measurements, skip_broken_datasets=True)
def load_masked_data(sub_tile_slice: Tuple[slice, slice, slice], source_prod: DataSource, geom=None) -> xarray.Dataset: data_fuse_func = import_function( source_prod.spec['fuse_func'] ) if 'fuse_func' in source_prod.spec else None data = GridWorkflow.load(source_prod.data[sub_tile_slice], measurements=source_prod.spec.get('measurements'), fuse_func=data_fuse_func, skip_broken_datasets=True) mask_inplace = source_prod.spec.get('mask_inplace', False) mask_nodata = source_prod.spec.get('mask_nodata', True) if mask_nodata: data = sensible_mask_invalid_data(data) # if all NaN completely_empty = all( ds for ds in xarray.ufuncs.isnan(data).all().data_vars.values()) if completely_empty: # Discard empty slice return None if mask_inplace or not mask_nodata: where = sensible_where_inplace else: where = sensible_where if 'masks' in source_prod.spec: for mask_spec, mask_tile in zip(source_prod.spec['masks'], source_prod.masks): if mask_tile is None: # Discard data due to no mask data return None mask_fuse_func = import_function( mask_spec['fuse_func']) if 'fuse_func' in mask_spec else None mask = GridWorkflow.load( mask_tile[sub_tile_slice], measurements=[mask_spec['measurement']], fuse_func=mask_fuse_func, skip_broken_datasets=True)[mask_spec['measurement']] data = where(data, make_mask_from_spec(mask, mask_spec)) del mask if geom is not None: data = where(data, geometry_mask([geom], data.geobox, invert=True)) if source_prod.source_index is not None: data.coords['source'] = ('time', np.repeat(source_prod.source_index, data.time.size)) return data
def load_masked_data(sub_tile_slice: Tuple[slice, slice, slice], source_prod: DataSource) -> xarray.Dataset: data_fuse_func = import_function(source_prod.spec['fuse_func']) if 'fuse_func' in source_prod.spec else None data = GridWorkflow.load(source_prod.data[sub_tile_slice], measurements=source_prod.spec.get('measurements'), fuse_func=data_fuse_func, skip_broken_datasets=True) mask_inplace = source_prod.spec.get('mask_inplace', False) mask_nodata = source_prod.spec.get('mask_nodata', True) if mask_nodata: data = sensible_mask_invalid_data(data) # if all NaN completely_empty = all(ds for ds in xarray.ufuncs.isnan(data).all().data_vars.values()) if completely_empty: # Discard empty slice return None if 'masks' in source_prod.spec: for mask_spec, mask_tile in zip(source_prod.spec['masks'], source_prod.masks): if mask_tile is None: # Discard data due to no mask data return None mask_fuse_func = import_function(mask_spec['fuse_func']) if 'fuse_func' in mask_spec else None mask = GridWorkflow.load(mask_tile[sub_tile_slice], measurements=[mask_spec['measurement']], fuse_func=mask_fuse_func, skip_broken_datasets=True)[mask_spec['measurement']] if mask_spec.get('flags') is not None: mask = make_mask(mask, **mask_spec['flags']) elif mask_spec.get('less_than') is not None: less_than = float(mask_spec['less_than']) mask = mask < less_than elif mask_spec.get('greater_than') is not None: greater_than = float(mask_spec['greater_than']) mask = mask > greater_than if mask_inplace: data = sensible_where_inplace(data, mask) else: data = sensible_where(data, mask) del mask if source_prod.source_index is not None: data.coords['source'] = ('time', np.repeat(source_prod.source_index, data.time.size)) return data
def resolve_func(key, value): if key not in ['fuse_func', 'dataset_predicate']: return value if callable(value): return value try: return import_function(value) except (ImportError, AttributeError): raise VirtualProductException("could not resolve function {} in {}".format(key, recipe))
def lookup(name, namespace=None, kind='transformation'): if callable(name): return name if namespace is not None and namespace in self.lookup_table and name in self.lookup_table[namespace]: result = self.lookup_table[namespace][name] else: try: result = import_function(name) except (ImportError, AttributeError): msg = "could not resolve {} {} in {}".format(kind, name, recipe) raise VirtualProductException(msg) self._assert(callable(result), "{} not callable in {}".format(kind, recipe)) return result
def resolve_transform(cls_name): if callable(cls_name): return cls_name if cls_name in self.lookup_table: cls = self.lookup_table[cls_name] else: try: cls = import_function(cls_name) except (ImportError, AttributeError): msg = "could not resolve transformation {} in {}".format(cls_name, recipe) raise VirtualProductException(msg) if not callable(cls): raise VirtualProductException("transformation not callable in {}".format(recipe)) return cls
def create_process(process, input, **settings): process_class = import_function(process) return process_class(input=construct(**input), **settings)