def test_augmentation_data_generator(tmpdir):
dist = DistributionCollection(examplary_tag_distribution())
dset_fname = str(tmpdir.join("dset.hdf5"))
samples = 6000
dset = DistributionHDF5Dataset(dset_fname, nb_samples=samples,
distribution=dist)
labels = dist.sample(samples)
labels = dist.normalize(labels)
fake = np.random.random((samples, 1, 8, 8))
discriminator = np.random.random((samples, 1))
dset.append(labels=labels, fake=fake, discriminator=discriminator)
dset.close()
dset = DistributionHDF5Dataset(dset_fname)
bs = 32
names = ['labels', 'fake']
assert 'labels' in next(dset.iter(bs, names))
assert next(dset.iter(bs))['labels'].dtype.names == tuple(dist.names)
dset_iters = [lambda bs: bit_split(dataset_iterator(dset, bs))]
data_gen = lambda bs: zip_dataset_iterators(dset_iters, bs)
label_names = ['bit_0', 'bit_11', 'x_rotation']
aug_gen = augmentation_data_generator(data_gen, 'fake', label_names)
outs = next(aug_gen(bs))
assert len(outs[0]) == 32
assert len(outs[1]) == len(label_names)
gen = aug_gen(bs)
for i, batch in enumerate(gen):
if i == 2*samples // bs:
break
assert batch is not None
assert batch[0].shape == (bs, 1, 8, 8)
assert len(batch[1]) == len(label_names)
def test_generated_3d_tags():
artist = BlackWhiteArtist(0, 255, 0, 1)
label_dist = DistributionCollection(examplary_tag_distribution())
bs = 8
labels, grids = generated_3d_tags(label_dist, batch_size=bs, artist=artist)
assert grids.shape == (bs, 1, 64, 64)
assert list(labels.dtype.names) == label_dist.names
def test_augmentation_data_generator(tmpdir):
dist = DistributionCollection(examplary_tag_distribution())
dset_fname = str(tmpdir.join("dset.hdf5"))
samples = 6000
dset = DistributionHDF5Dataset(dset_fname,
nb_samples=samples,
distribution=dist)
labels = dist.sample(samples)
labels = dist.normalize(labels)
fake = np.random.random((samples, 1, 8, 8))
discriminator = np.random.random((samples, 1))
dset.append(labels=labels, fake=fake, discriminator=discriminator)
dset.close()
dset = DistributionHDF5Dataset(dset_fname)
bs = 32
names = ['labels', 'fake']
assert 'labels' in next(dset.iter(bs, names))
assert next(dset.iter(bs))['labels'].dtype.names == tuple(dist.names)
dset_iters = [lambda bs: bit_split(dataset_iterator(dset, bs))]
data_gen = lambda bs: zip_dataset_iterators(dset_iters, bs)
label_names = ['bit_0', 'bit_11', 'x_rotation']
aug_gen = augmentation_data_generator(data_gen, 'fake', label_names)
outs = next(aug_gen(bs))
assert len(outs[0]) == 32
assert len(outs[1]) == len(label_names)
gen = aug_gen(bs)
for i, batch in enumerate(gen):
if i == 2 * samples // bs:
break
assert batch is not None
assert batch[0].shape == (bs, 1, 8, 8)
assert len(batch[1]) == len(label_names)
def test_generated_3d_tags():
artist = BlackWhiteArtist(0, 255, 0, 1)
label_dist = DistributionCollection(examplary_tag_distribution())
bs = 8
labels, grids = generated_3d_tags(label_dist, batch_size=bs, artist=artist)
assert grids.shape == (bs, 1, 64, 64)
assert list(labels.dtype.names) == label_dist.names
def test_store_samples(tmpdir):
dist = DistributionCollection(examplary_tag_distribution())
bs = 64
labels_record = dist.sample(bs)
labels_record = dist.normalize(labels_record)
labels = []
for name in labels_record.dtype.names:
labels.append(labels_record[name])
labels = np.concatenate(labels, axis=-1)
fakes = np.random.random((bs, 1, 8, 8))
store = StoreSamples(str(tmpdir), dist)
store.on_epoch_end(0, logs={'samples': {'labels': labels, 'fake': fakes}})
assert tmpdir.join("00000.hdf5").exists()
def default_tag_distribution():
tag_dist_params = examplary_tag_distribution()
angle = to_radians(60)
tag_dist_params["x_rotation"] = TruncNormal(-angle, angle, 0, angle / 2)
tag_dist_params["y_rotation"] = TruncNormal(-angle, angle, 0, angle / 2)
tag_dist_params["radius"] = TruncNormal(22, 26, 24.0, 1.3)
tag_dist_params["center"] = (Uniform(-16, 16), 2)
tag_dist_params["bulge_factor"] = Uniform(0.4, 0.8)
tag_dist_params["focal_length"] = Uniform(2, 4)
tag_dist_params["inner_ring_radius"] = Uniform(0.42, 0.48)
tag_dist_params["middle_ring_radius"] = Constant(0.8)
tag_dist_params["outer_ring_radius"] = Constant(1.)
return DistributionCollection(tag_dist_params)
def default_tag_distribution():
tag_dist_params = examplary_tag_distribution()
angle = to_radians(60)
tag_dist_params["x_rotation"] = TruncNormal(-angle, angle, 0, angle / 2)
tag_dist_params["y_rotation"] = TruncNormal(-angle, angle, 0, angle / 2)
tag_dist_params["radius"] = TruncNormal(22, 26, 24.0, 1.3)
tag_dist_params["center"] = (Uniform(-16, 16), 2)
tag_dist_params["bulge_factor"] = Uniform(0.4, 0.8)
tag_dist_params["focal_length"] = Uniform(2, 4)
tag_dist_params["inner_ring_radius"] = Uniform(0.42, 0.48)
tag_dist_params["middle_ring_radius"] = Constant(0.8)
tag_dist_params["outer_ring_radius"] = Constant(1.)
return DistributionCollection(tag_dist_params)
def test_distribution_hdf5_dataset(tmpdir):
with pytest.raises(Exception):
DistributionHDF5Dataset(
str(tmpdir.join('dataset_no_distribution.hdf5')), nb_samples=1000)
dist = DistributionCollection(examplary_tag_distribution(nb_bits=12))
labels = dist.sample(32)
image = np.random.random((32, 1, 8, 8))
dset = DistributionHDF5Dataset(
str(tmpdir.join('dataset.hdf5')), distribution=dist, nb_samples=1000)
dset.append(labels=labels, image=image)
for name in dist.names:
assert name in dset
for batch in dset.iter(batch_size=32):
for name in dist.names:
assert name not in batch
assert 'labels' in batch
assert batch['labels'].dtype == dist.norm_dtype
break
def test_distribution_hdf5_dataset(tmpdir):
with pytest.raises(Exception):
DistributionHDF5Dataset(str(
tmpdir.join('dataset_no_distribution.hdf5')),
nb_samples=1000)
dist = DistributionCollection(examplary_tag_distribution(nb_bits=12))
labels = dist.sample(32)
image = np.random.random((32, 1, 8, 8))
dset = DistributionHDF5Dataset(str(tmpdir.join('dataset.hdf5')),
distribution=dist,
nb_samples=1000)
dset.append(labels=labels, image=image)
for name in dist.names:
assert name in dset
for batch in dset.iter(batch_size=32):
for name in dist.names:
assert name not in batch
assert 'labels' in batch
assert batch['labels'].dtype == dist.norm_dtype
break
