def run(self, estimator, *args, **kwargs):
"""
run estimator on the 'straps'
Parameters
----------
*args : a list of indexable datasets.
the datasets are divided into straps by the strap function.
estimator: the estimator `estimator(*args)`.
np : number of processes to use
"""
np = kwargs.pop('np', None)
varstraps = [self.create_straps(v) for v in args]
indstraps = [[int(i) for i in self.active_straps]]*len(args)
cache = {}
sizes_per_var = [{} for i in range(len(args))]
with utils.MapReduce(np=np) as pool:
def work(p):
ind, vars = p
n = [len(var) for var in vars]
return ind, estimator(*vars), n
def reduce(ind, r, n):
cache[ind] = r
for i, (ind1, n1) in enumerate(zip(ind, n)):
sizes_per_var[i][ind1] = n1
items = [(ind, var) for ind, var in zip(outer(*indstraps), outer(*varstraps))]
pool.map(work, items, reduce=reduce)
result = StrappedResults(cache, sizes_per_var)
return result
def run(self, estimator, *args, **kwargs):
"""
run estimator on the 'straps'
Parameters
----------
*args : a list of indexable datasets.
the datasets are divided into straps by the strap function.
estimator: the estimator `estimator(*args)`.
np : number of processes to use
"""
np = kwargs.pop('np', None)
vars = [self.create_straps(v) for v in args]
cache = {}
with utils.MapReduce(np=np) as pool:
def work(p):
ind, var = p
return ind, estimator(*var)
def reduce(ind, r):
cache[ind] = r
items = [(ind, var) for ind, var in zip(outer(*([[int(i) for i in self.active_straps]]*len(args))),
outer(*vars))]
pool.map(work, items, reduce=reduce)
sizes = [numpy.array([len(s) for s in v], dtype='intp') for v in vars]
result = StrappedResults(cache, sizes)
return result
def run(self, estimator, *args, **kwargs):
"""
run estimator on the 'straps'
Parameters
----------
*args : a list of indexable datasets.
the datasets are divided into straps by the strap function.
estimator: the estimator `estimator(*args)`.
np : number of processes to use
"""
np = kwargs.pop('np', None)
vars = [self.create_straps(v) for v in args]
cache = {}
with utils.MapReduce(np=np) as pool:
def work(p):
ind, var = p
return ind, estimator(*var)
def reduce(ind, r):
cache[ind] = r
items = [(ind, var) for ind, var in zip(
outer(*([range(len(self.active_straps))] *
len(args))), outer(*vars))]
pool.map(work, items, reduce=reduce)
sizes = [numpy.array([len(s) for s in v], dtype='intp') for v in vars]
result = StrappedResults(cache, sizes)
return result
def resample(self, result, bootstrap=None, operator=lambda x, y : x + y):
"""
bootstrap is a list of strapids returnedy by self.bootstrap.
operator is used to combine the result of straps into the resample.
it shall be sort of addition but may have to be different if result
of the function does not support the `+` operator.
"""
Nargs = len(result.sizes_per_var)
# length for each bootstrapped resample dataset
L = [sum([sizes_per_strap[i] for i in bootstrap]) for sizes_per_strap in result.sizes_per_var]
# result is the sum of the submatrix
R = reduce(operator, [result.cache[ind] for ind in outer(*([bootstrap] * Nargs))])
return L, R
def resample(self, result, bootstrap=None, operator=lambda x, y : x + y):
"""
bootstrap is a list of strapids returnedy by self.bootstrap.
operator is used to combine the result of straps into the resample.
it shall be sort of addition but may have to be different if result
of the function does not support the `+` operator.
"""
Nargs = len(result.sizes)
# length for each bootstrapped resample dataset
L = [sum(s[bootstrap]) for s in result.sizes]
# result is the sum of the submatrix
R = reduce(operator, [result.cache[ind] for ind in outer(*([bootstrap] * Nargs))])
return L, R
def rectangular_partitioning(shape, steps):
"""
N-D rectangular batch generation.
shape: [len(grid[dim]) for dim in range(ndim)]
steps: [step_len[dim] for dim in range(ndim)]
returns a list of batches,
where each element (batch) is an ndim-list,
where each element is an array of ordinates (length == #gridpoints in batch).
# Example, with visualization:
>>> shape = [4,13]
>>> steps = [2,4]
>>> batches = rectangular_partitioning(shape, steps)
>>> m = np.prod(shape)
>>> nB = len(batches)
>>> values = np.random.choice(arange(nB),nB,0)
>>> Z = zeros(shape)
>>> for ib,b in enumerate(batches):
>>> Z[b] = values[ib]
>>> plt.imshow(Z)
"""
assert len(shape) == len(steps)
ndim = len(steps)
# An ndim list of (average) local grid lengths:
nLocs = [round(n / d) for n, d in zip(shape, steps)]
# An ndim list of (marginal) grid partitions [array_split() handles non-divisibility]:
edge_partitions = [
np.array_split(np.arange(n), nLoc) for n, nLoc in zip(shape, nLocs)
]
batches = []
from itertools import product as outer
for batch_edges in outer(*edge_partitions):
# The 'indexing' argument below is actually inconsequential:
# it merely changes batch's internal ordering.
batch_rect = np.meshgrid(*batch_edges, indexing='ij')
coords = [ii.flatten() for ii in batch_rect]
batches += [coords]
return batches
import tempfile
import glob
import os
import tensorflow as tf
import pytest
import biggan
from biggan.config import base as cfg
from itertools import product as outer
from tensorflow.keras.mixed_precision import experimental as mp
@pytest.mark.parametrize(
"image_size,mixed_precision",
list(outer(cfg.choices.image_size, [True, False]))
)
def test_build_and_train_model(image_size, mixed_precision):
if mixed_precision:
mp.set_policy(mp.Policy("mixed_float16"))
model = biggan.build_model(
image_size=image_size,
channels=4,
num_classes=4,
latent_dim=4,
)
assert model.G.built and model.D.built
def dummy_dataset():
