def _generate_mask(self, *ds_slice):
"""
Generate multiplicative inclusion mask from exclusion layers.
Parameters
----------
ds_slice : int | slice | list | ndarray
What to extract from ds, each arg is for a sequential axis.
For example, (slice(0, 64), slice(0, 64)) will extract a 64x64
exclusions mask.
Returns
-------
mask : ndarray
Multiplicative inclusion mask with all layers multiplied together
("and" operation) such that 1 is included, 0 is excluded,
0.5 is half.
"""
mask = None
if len(ds_slice) == 1 & isinstance(ds_slice[0], tuple):
ds_slice = ds_slice[0]
if self._min_area is not None:
ds_slice, sub_slice = self._increase_mask_slice(ds_slice, n=1)
if self.layers:
force_include = []
for layer in self.layers:
if layer.force_include:
force_include.append(layer)
else:
logger.debug('Computing exclusions {}'.format(layer))
log_mem(logger, log_level='DEBUG')
layer_slice = (layer.layer, ) + ds_slice
layer_mask = layer[self.excl_h5[layer_slice]]
if mask is None:
mask = layer_mask
else:
mask = np.minimum(mask, layer_mask, dtype='float32')
if force_include:
logger.debug('Computing forced inclusions')
log_mem(logger, log_level='DEBUG')
mask = self._force_include(mask, force_include, ds_slice)
if self._min_area is not None:
mask = self._area_filter(mask,
min_area=self._min_area,
kernel=self._kernel)
mask = mask[sub_slice]
else:
if self._min_area is not None:
ds_slice = sub_slice
mask = self._generate_ones_mask(ds_slice)
return mask
def _collect(self):
"""Simple & robust serial collection optimized for low memory usage."""
with Outputs(self._h5_file, mode='a') as f_out:
for fp in self._source_files:
with Outputs(fp, mode='r') as f_source:
x = self._get_source_gid_chunks(f_source)
all_source_gids, source_gid_chunks = x
for source_gids in source_gid_chunks:
self._collect_chunk(all_source_gids, source_gids,
f_out, f_source, fp)
log_mem(logger, log_level='DEBUG')
def run(self, **kwargs):
"""
Run ParallelSmartJobs
Parameters
----------
kwargs : dict
Keyword arguments to be passed to obj.run(). Makes it easier to
have obj.run() as a @staticmethod.
"""
logger.info('Executing parallel run on a local cluster with '
'{0} workers over {1} total iterations.'.format(
self.n_workers, 1 + len(self.execution_iter)))
log_mem()
# initialize a client based on the input cluster.
with SpawnProcessPool(max_workers=self.n_workers) as executor:
futures = []
# iterate through split executions, submitting each to worker
for i, exec_slice in enumerate(self.execution_iter):
logger.debug(
'Kicking off serial worker #{0} for: {1}. '.format(
i, exec_slice))
# submit executions and append to futures list
futures.append(
executor.submit(self.obj.run, exec_slice, **kwargs))
# Take a pause after one complete set of workers
if (i + 1) % self.n_workers == 0:
futures = self.gather_and_flush(i, futures)
# All futures complete
self.gather_and_flush('END', futures, force_flush=True)
logger.debug('Smart parallel job complete. Returning execution '
'control to higher level processes.')
log_mem()
def execute_single(fun, input_obj, worker=0, **kwargs):
"""Execute a serial compute on a single core.
Parameters
----------
fun : function
Function to execute.
input_obj : object
Object passed as first argument to fun. Typically a project control
object that can be the result of iteration in the parallel execution
framework.
worker : int
Worker number (for debugging purposes).
**kwargs : dict
Key word arguments passed to fun.
"""
logger.debug(
'Running single serial execution on worker #{} for: {}'.format(
worker, input_obj))
out = fun(input_obj, **kwargs)
log_mem()
return out
def run_serial(excl_fpath, h5_fpath, tm_dset, *agg_dset,
agg_method='mean', excl_dict=None,
area_filter_kernel='queen', min_area=None,
check_excl_layers=False, resolution=64, excl_area=0.0081,
gids=None, gen_index=None):
"""
Standalone method to aggregate - can be parallelized.
Parameters
----------
excl_fpath : str
Filepath to exclusions h5 with techmap dataset.
h5_fpath : str
Filepath to .h5 file to aggregate
tm_dset : str
Dataset name in the techmap file containing the
exclusions-to-resource mapping data.
agg_dset : str
Dataset to aggreate, can supply multiple datasets
agg_method : str
Aggregation method, either mean or sum/aggregate
excl_dict : dict | None
Dictionary of exclusion LayerMask arugments {layer: {kwarg: value}}
area_filter_kernel : str
Contiguous area filter method to use on final exclusions mask
min_area : float | None
Minimum required contiguous area filter in sq-km
check_excl_layers : bool
Run a pre-flight check on each exclusion layer to ensure they
contain un-excluded values
resolution : int | None
SC resolution, must be input in combination with gid. Prefered
option is to use the row/col slices to define the SC point instead.
excl_area : float
Area of an exclusion cell (square km).
gids : list | None
List of gids to get summary for (can use to subset if running in
parallel), or None for all gids in the SC extent.
gen_index : np.ndarray
Array of generation gids with array index equal to resource gid.
Array value is -1 if the resource index was not used in the
generation run.
Returns
-------
agg_out : dict
Aggregated values for each aggregation dataset
"""
with SupplyCurveExtent(excl_fpath, resolution=resolution) as sc:
exclusion_shape = sc.exclusions.shape
if gids is None:
gids = sc.valid_sc_points(tm_dset)
# pre-extract handlers so they are not repeatedly initialized
file_kwargs = {'excl_dict': excl_dict,
'area_filter_kernel': area_filter_kernel,
'min_area': min_area,
'check_excl_layers': check_excl_layers}
dsets = agg_dset + ('meta', )
agg_out = {ds: [] for ds in dsets}
with AggFileHandler(excl_fpath, h5_fpath, **file_kwargs) as fh:
n_finished = 0
for gid in gids:
try:
gid_out = AggregationSupplyCurvePoint.run(
gid,
fh.exclusions,
fh.h5,
tm_dset,
*agg_dset,
agg_method=agg_method,
excl_dict=excl_dict,
resolution=resolution,
excl_area=excl_area,
exclusion_shape=exclusion_shape,
close=False,
gen_index=gen_index)
except EmptySupplyCurvePointError:
logger.debug('SC gid {} is fully excluded or does not '
'have any valid source data!'.format(gid))
except Exception:
logger.exception('SC gid {} failed!'.format(gid))
raise
else:
n_finished += 1
logger.debug('Serial aggregation: '
'{} out of {} points complete'
.format(n_finished, len(gids)))
log_mem(logger)
for k, v in gid_out.items():
agg_out[k].append(v)
return agg_out
def compute_statistics(self,
dataset,
sites=None,
diurnal=False,
month=False,
combinations=False,
max_workers=None,
chunks_per_worker=5,
lat_lon_only=True):
"""
Compute statistics
Parameters
----------
dataset : str
Dataset to extract stats for
sites : list | slice, optional
Subset of sites to extract, by default None or all sites
diurnal : bool, optional
Extract diurnal stats, by default False
month : bool, optional
Extract monthly stats, by default False
combinations : bool, optional
Extract all combinations of temporal stats, by default False
max_workers : None | int, optional
Number of workers to use, if 1 run in serial, if None use all
available cores, by default None
chunks_per_worker : int, optional
Number of chunks to extract on each worker, by default 5
lat_lon_only : bool, optional
Only append lat, lon coordinates to stats, by default True
Returns
-------
res_stats : pandas.DataFrame
DataFrame of desired statistics at desired time intervals
"""
if max_workers is None:
max_workers = os.cpu_count()
slices = self._get_slices(dataset,
sites,
chunks_per_slice=chunks_per_worker)
if len(slices) == 1:
max_workers = 1
if max_workers > 1:
msg = ('Extracting {} for {} in parallel using {} workers'.format(
list(self.statistics), dataset, max_workers))
logger.info(msg)
loggers = [__name__, 'rex']
with SpawnProcessPool(max_workers=max_workers,
loggers=loggers) as exe:
futures = []
for sites_slice in slices:
future = exe.submit(self._extract_stats,
self.res_h5,
self.statistics,
dataset,
res_cls=self.res_cls,
hsds=self._hsds,
time_index=self.time_index,
sites_slice=sites_slice,
diurnal=diurnal,
month=month,
combinations=combinations)
futures.append(future)
res_stats = []
for i, future in enumerate(as_completed(futures)):
res_stats.append(future.result())
logger.debug('Completed {} out of {} workers'.format(
(i + 1), len(futures)))
else:
msg = ('Extracting {} for {} in serial'.format(
self.statistics.keys(), dataset))
logger.info(msg)
res_stats = []
for i, sites_slice in enumerate(slices):
res_stats.append(
self._extract_stats(self.res_h5,
self.statistics,
dataset,
res_cls=self.res_cls,
hsds=self._hsds,
time_index=self.time_index,
sites_slice=sites_slice,
diurnal=diurnal,
month=month,
combinations=combinations))
logger.debug('Completed {} out of {} sets of sites'.format(
(i + 1), len(slices)))
gc.collect()
log_mem(logger)
res_stats = pd.concat(res_stats)
if lat_lon_only:
meta = self.lat_lon
else:
meta = self.meta
res_stats = meta.join(res_stats.sort_index(), how='inner')
return res_stats
def compute(self,
dset1,
dset2,
bins1,
bins2,
sites=None,
max_workers=None,
chunks_per_worker=5):
"""
Compute joint probability distribution between given datasets using
given bins for all sites.
Parameters
----------
dset1 : str
Dataset 1 to generate joint probability distribution for
dset2 : str
Dataset 2 to generate joint probabilty distribution for
bins1 : tuple
(start, stop, step) for dataset 1 bins. The stop value is
inclusive, so (0, 6, 2) would yield three bins with edges (0, 2, 4,
6). If the stop value is not perfectly divisible by the step, the
last bin will overshoot the stop value.
bins2 : tuple
(start, stop, step) for dataset 2 bins. The stop value is
inclusive, so (0, 6, 2) would yield three bins with edges (0, 2, 4,
6). If the stop value is not perfectly divisible by the step, the
last bin will overshoot the stop value.
sites : list | slice, optional
Subset of sites to extract, by default None or all sites
max_workers : None | int, optional
Number of workers to use, if 1 run in serial, if None use all
available cores, by default None
chunks_per_worker : int, optional
Number of chunks to extract on each worker, by default 5
Returns
-------
jpd: pandas.DataFrame
DataFrame of joint probability distribution between given datasets
with given bins
"""
if max_workers is None:
max_workers = os.cpu_count()
slices = self._get_slices(dset1,
dset2,
sites,
chunks_per_slice=chunks_per_worker)
if len(slices) == 1:
max_workers = 1
jpd = {}
if max_workers > 1:
msg = ('Computing the joint probability distribution between {} '
'and {} in parallel using {} workers'.format(
dset1, dset2, max_workers))
logger.info(msg)
loggers = [__name__, 'rex']
with SpawnProcessPool(max_workers=max_workers,
loggers=loggers) as exe:
futures = []
for sites_slice in slices:
future = exe.submit(self.compute_joint_pd,
self.res_h5,
dset1,
dset2,
bins1,
bins2,
res_cls=self.res_cls,
hsds=self._hsds,
sites_slice=sites_slice)
futures.append(future)
for i, future in enumerate(as_completed(futures)):
jpd.update(future.result())
logger.debug('Completed {} out of {} workers'.format(
(i + 1), len(futures)))
else:
msg = ('Computing the joint probability distribution between {} '
'and {} in serial.'.format(dset1, dset2))
logger.info(msg)
for i, sites_slice in enumerate(slices):
jpd.update(
self.compute_joint_pd(self.res_h5,
dset1,
dset2,
bins1,
bins2,
res_cls=self.res_cls,
hsds=self._hsds,
sites_slice=sites_slice))
logger.debug('Completed {} out of {} sets of sites'.format(
(i + 1), len(slices)))
gc.collect()
log_mem(logger)
bins1 = self._make_bins(*bins1)
bins2 = self._make_bins(*bins2)
index = np.meshgrid(bins1[:-1], bins2[:-1], indexing='ij')
index = np.array(index).T.reshape(-1, 2).astype(np.int16)
index = pd.MultiIndex.from_arrays(index.T, names=(dset1, dset2))
jpd = pd.DataFrame({k: v.flatten(order='F')
for k, v in jpd.items()},
index=index).sort_index(axis=1)
return jpd
