def run(self, estimator, *args, **kwargs): """ run estimator on the 'straps' Parameters ---------- *args : a list of indexable datasets. the datasets are divided into straps by the strap function. estimator: the estimator `estimator(*args)`. np : number of processes to use """ np = kwargs.pop('np', None) varstraps = [self.create_straps(v) for v in args] indstraps = [[int(i) for i in self.active_straps]]*len(args) cache = {} sizes_per_var = [{} for i in range(len(args))] with utils.MapReduce(np=np) as pool: def work(p): ind, vars = p n = [len(var) for var in vars] return ind, estimator(*vars), n def reduce(ind, r, n): cache[ind] = r for i, (ind1, n1) in enumerate(zip(ind, n)): sizes_per_var[i][ind1] = n1 items = [(ind, var) for ind, var in zip(outer(*indstraps), outer(*varstraps))] pool.map(work, items, reduce=reduce) result = StrappedResults(cache, sizes_per_var) return result
def run(self, estimator, *args, **kwargs): """ run estimator on the 'straps' Parameters ---------- *args : a list of indexable datasets. the datasets are divided into straps by the strap function. estimator: the estimator `estimator(*args)`. np : number of processes to use """ np = kwargs.pop('np', None) vars = [self.create_straps(v) for v in args] cache = {} with utils.MapReduce(np=np) as pool: def work(p): ind, var = p return ind, estimator(*var) def reduce(ind, r): cache[ind] = r items = [(ind, var) for ind, var in zip(outer(*([[int(i) for i in self.active_straps]]*len(args))), outer(*vars))] pool.map(work, items, reduce=reduce) sizes = [numpy.array([len(s) for s in v], dtype='intp') for v in vars] result = StrappedResults(cache, sizes) return result
def run(self, estimator, *args, **kwargs): """ run estimator on the 'straps' Parameters ---------- *args : a list of indexable datasets. the datasets are divided into straps by the strap function. estimator: the estimator `estimator(*args)`. np : number of processes to use """ np = kwargs.pop('np', None) vars = [self.create_straps(v) for v in args] cache = {} with utils.MapReduce(np=np) as pool: def work(p): ind, var = p return ind, estimator(*var) def reduce(ind, r): cache[ind] = r items = [(ind, var) for ind, var in zip( outer(*([range(len(self.active_straps))] * len(args))), outer(*vars))] pool.map(work, items, reduce=reduce) sizes = [numpy.array([len(s) for s in v], dtype='intp') for v in vars] result = StrappedResults(cache, sizes) return result
def resample(self, result, bootstrap=None, operator=lambda x, y : x + y): """ bootstrap is a list of strapids returnedy by self.bootstrap. operator is used to combine the result of straps into the resample. it shall be sort of addition but may have to be different if result of the function does not support the `+` operator. """ Nargs = len(result.sizes_per_var) # length for each bootstrapped resample dataset L = [sum([sizes_per_strap[i] for i in bootstrap]) for sizes_per_strap in result.sizes_per_var] # result is the sum of the submatrix R = reduce(operator, [result.cache[ind] for ind in outer(*([bootstrap] * Nargs))]) return L, R
def resample(self, result, bootstrap=None, operator=lambda x, y : x + y): """ bootstrap is a list of strapids returnedy by self.bootstrap. operator is used to combine the result of straps into the resample. it shall be sort of addition but may have to be different if result of the function does not support the `+` operator. """ Nargs = len(result.sizes) # length for each bootstrapped resample dataset L = [sum(s[bootstrap]) for s in result.sizes] # result is the sum of the submatrix R = reduce(operator, [result.cache[ind] for ind in outer(*([bootstrap] * Nargs))]) return L, R
def rectangular_partitioning(shape, steps): """ N-D rectangular batch generation. shape: [len(grid[dim]) for dim in range(ndim)] steps: [step_len[dim] for dim in range(ndim)] returns a list of batches, where each element (batch) is an ndim-list, where each element is an array of ordinates (length == #gridpoints in batch). # Example, with visualization: >>> shape = [4,13] >>> steps = [2,4] >>> batches = rectangular_partitioning(shape, steps) >>> m = np.prod(shape) >>> nB = len(batches) >>> values = np.random.choice(arange(nB),nB,0) >>> Z = zeros(shape) >>> for ib,b in enumerate(batches): >>> Z[b] = values[ib] >>> plt.imshow(Z) """ assert len(shape) == len(steps) ndim = len(steps) # An ndim list of (average) local grid lengths: nLocs = [round(n / d) for n, d in zip(shape, steps)] # An ndim list of (marginal) grid partitions [array_split() handles non-divisibility]: edge_partitions = [ np.array_split(np.arange(n), nLoc) for n, nLoc in zip(shape, nLocs) ] batches = [] from itertools import product as outer for batch_edges in outer(*edge_partitions): # The 'indexing' argument below is actually inconsequential: # it merely changes batch's internal ordering. batch_rect = np.meshgrid(*batch_edges, indexing='ij') coords = [ii.flatten() for ii in batch_rect] batches += [coords] return batches
import tempfile import glob import os import tensorflow as tf import pytest import biggan from biggan.config import base as cfg from itertools import product as outer from tensorflow.keras.mixed_precision import experimental as mp @pytest.mark.parametrize( "image_size,mixed_precision", list(outer(cfg.choices.image_size, [True, False])) ) def test_build_and_train_model(image_size, mixed_precision): if mixed_precision: mp.set_policy(mp.Policy("mixed_float16")) model = biggan.build_model( image_size=image_size, channels=4, num_classes=4, latent_dim=4, ) assert model.G.built and model.D.built def dummy_dataset():