def test_modelio_paths():
""" Test default path initialization """
# init all defaults
obj = modelio.ModelIO()
default_paths = obj.paths
# Check pathname default setup
path_names = [
'data', 'training', 'validation', 'evaluation', 'predict', 'genepool',
'genebank', 'model', 'state'
]
for test_path in path_names:
test_dict = {test_path: 'Tests123'}
expected_paths = deepcopy(default_paths)
if test_path == 'data':
expected_paths = {
key: 'Tests123' + expected_paths[key][4:]
for key in expected_paths
}
else:
expected_paths[test_path] = 'Tests123'
config = {'paths': test_dict}
obj = modelio.ModelIO(config)
assert obj.config['paths'] == expected_paths
assert obj.paths == expected_paths
def test_modelio_functions():
""" Test modelio functions """
config = {'paths': _PATHS}
obj = modelio.ModelIO(config)
parameters = ['jitter', 'edges', 'quality']
alt_values = {'jitter': True, 'edges': False, 'quality': 0.5}
# Just twiddle each item once this time
frameops.reset_cache(True)
for item in parameters:
config = deepcopy(obj.config)
config[item] = alt_values[item]
nobj = modelio.ModelIO(config)
# hack result so Alpha/Beta paths point to our mockup directories
nobj.alpha = 'DPX'
nobj.beta = 'PNG'
# always 2 images in DPX (alpha) and 1 in PNG (beta)
assert nobj.train_images_count() == int(nobj.quality *
nobj.tiles_per_image * 2)
assert nobj.val_images_count() == nobj.tiles_per_image * 2
assert nobj.eval_images_count() == nobj.tiles_per_image * 2
assert nobj.predict_images_count() == nobj.tiles_per_image * 2
# Loop through the assumed number of tiles twice, check they are aligned
gen = nobj.training_data_generator()
tiles1 = [next(gen) for _ in range(nobj.train_images_count())]
tiles2 = [next(gen) for _ in range(nobj.train_images_count())]
assert tiles1 == tiles2
gen = nobj.validation_data_generator()
tiles1 = [next(gen) for _ in range(nobj.val_images_count())]
tiles2 = [next(gen) for _ in range(nobj.val_images_count())]
assert tiles1 == tiles2
gen = nobj.evaluation_data_generator()
tiles1 = [next(gen) for _ in range(nobj.eval_images_count())]
tiles2 = [next(gen) for _ in range(nobj.eval_images_count())]
assert tiles1 == tiles2
gen = nobj.prediction_data_generator()
tiles1 = [next(gen) for _ in range(nobj.predict_images_count())]
tiles2 = [next(gen) for _ in range(nobj.predict_images_count())]
assert tiles1 == tiles2
def test_grout():
""" Test frameops.grout() """
frameops.reset_cache(False)
# the actual image we will be testing. Use PNG because it's 1920x1080
# so it won't be autoscaled
img = frameops.imread(_PNG)
# get the tiles
config = modelio.ModelIO({
'shuffle': False,
'jitter': False,
'skip': False,
'edges': True,
'residual': False
})
tiles = frameops.extract_tiles(_PNG, config)
assert len(tiles) == 432
# grout the tiles back together
grouted = frameops.grout(tiles, config)
assert np.shape(img) == np.shape(grouted)
assert np.array_equal(img, grouted)
# repeat check with no default borders
config = modelio.ModelIO({
'shuffle': False,
'jitter': False,
'skip': False,
'residual': False,
'edges': True,
'trim_left': 0,
'trim_right': 0
})
tiles = frameops.extract_tiles(_PNG, config)
grouted = frameops.grout(tiles, config)
assert np.shape(img) == np.shape(grouted)
assert np.array_equal(img, grouted)
def test_modelio_init():
""" Test initialization of a ModelIO class """
# init all defaults
obj = modelio.ModelIO()
objconfig = obj.config
check_modelio(obj, objconfig)
# for each item in our default config, twiddle it, create a new
# object, and check that it is consistent. Do only for the
# parameters that will be passed into an instantiation.
# the 'path' list is a special case handled later
for key in objconfig:
if key in PARAMETERS:
val = objconfig[key]
if isinstance(val, bool):
val = not val
elif isinstance(val, int):
val = val + 1 if val == 0 else val * 2
elif isinstance(val, float):
val = val + 0.1 if val == 0 else val / 2
elif isinstance(val, (list, tuple, dict)):
pass
elif val == 'BasicSR':
val = 'ExpansionSR'
elif val == 'constant':
val = 'valid'
elif val == 'dpx':
val = 'png'
else:
assert False, 'Whoops, got [{}]'.format(val)
#print('Testing', key, ':', objconfig[key], '->', val)
config = {key: val}
obj = modelio.ModelIO(config)
assert obj.config[key] == val
check_modelio(obj, config)
def test_modelio_computed():
""" Test modelio computed information """
obj = modelio.ModelIO()
parameters = [
'base_tile_width', 'base_tile_height', 'border', 'image_width',
'trim_left', 'trim_right', 'image_height', 'trim_top', 'trim_bottom',
'jitter', 'edges'
]
alt_values = {
'base_tile_width': 30,
'base_tile_height': 15,
'border': 10,
'image_width': 3840,
'trim_left': 0,
'trim_right': 0,
'image_height': 2160,
'trim_top': 240,
'trim_bottom': 240,
'jitter': True,
'edges': True
}
# Try all possible input permutations, just to be pedantic
for taken in range(len(parameters)):
for combination in itertools.combinations(parameters, taken):
config = deepcopy(obj.config)
for item in combination:
config[item] = alt_values[item]
nobj = modelio.ModelIO(config)
assert nobj.config['tile_width'] == nobj.tile_width
assert nobj.tile_width == nobj.border * 2 + nobj.base_tile_width
assert nobj.config['tile_height'] == nobj.tile_height
assert nobj.tile_height == nobj.border * 2 + nobj.base_tile_height
assert nobj.config['trimmed_width'] == nobj.trimmed_width
assert nobj.trimmed_width == nobj.image_width - (nobj.trim_left +
nobj.trim_right)
assert nobj.config['trimmed_height'] == nobj.trimmed_height
assert nobj.trimmed_height == nobj.image_height - (
nobj.trim_top + nobj.trim_bottom)
assert nobj.config['tiles_across'] == nobj.tiles_across
assert nobj.tiles_across == nobj.trimmed_width // nobj.base_tile_width
assert nobj.config['tiles_down'] == nobj.tiles_down
assert nobj.tiles_down == nobj.trimmed_height // nobj.base_tile_height
assert nobj.config['tiles_per_image'] == nobj.tiles_per_image
# This looks a little bass-ackwards, but I'm being deliberately pedantic
# so this isn't computed in the same way as in modelio.py
if nobj.edges:
tpi = nobj.tiles_across * nobj.tiles_down
if nobj.jitter:
tpi += (nobj.tiles_across - 1) * (nobj.tiles_down - 1)
else:
tpi = (nobj.tiles_across - 1) * (nobj.tiles_down - 1)
if nobj.jitter:
tpi += (nobj.tiles_across - 2) * (nobj.tiles_down - 2)
assert nobj.tiles_per_image == tpi
# Test theano
assert obj.config['image_shape'] == obj.image_shape
assert obj.config['theano'] == obj.theano
assert obj.image_shape[0 if obj.theano else 2] == 3
def test_tesselate_pair():
""" Test frameops.tesselate_pair(). Also tests update_cache_quality() """
# For ease of checking, we will test using the same image on both sides
# of the pair.
frameops.reset_cache(False)
# Check all options off
config = modelio.ModelIO({
'shuffle': False,
'jitter': False,
'skip': False,
'edges': True,
'quality': 1.0,
'residual': False
})
tiles = [tp for tp in frameops.tesselate_pair(_DPX, _DPX, config)]
assert len(tiles) == 432
assert all([np.array_equal(a, b) for a, b in tiles])
# confirm that residual generation works. Since both images are
# identical, the residuals will all be zero.
config = modelio.ModelIO({
'shuffle': False,
'jitter': False,
'skip': False,
'edges': True,
'quality': 1.0,
'residual': True
})
tiles = [tp for tp in frameops.tesselate_pair(_DPX, _DPX, config)]
assert len(tiles) == 432
assert not any([np.any(b) for _, b in tiles])
# check that shuffles are matched
config = modelio.ModelIO({
'shuffle': True,
'jitter': False,
'skip': False,
'edges': True,
'quality': 1.0,
'residual': False
})
tiles = [tp for tp in frameops.tesselate_pair(_DPX, _DPX, config)]
assert len(tiles) == 432
assert all([np.array_equal(a, b) for a, b in tiles])
# check that jitters are matched
config = modelio.ModelIO({
'shuffle': False,
'jitter': True,
'skip': False,
'edges': True,
'quality': 1.0,
'residual': False
})
tiles = [tp for tp in frameops.tesselate_pair(_DPX, _DPX, config)]
assert len(tiles) == 823
assert all([np.array_equal(a, b) for a, b in tiles])
# check that skips are matched
config = modelio.ModelIO({
'shuffle': False,
'jitter': False,
'skip': True,
'edges': True,
'quality': 1.0,
'residual': False
})
tiles = [tp for tp in frameops.tesselate_pair(_DPX, _DPX, config)]
assert len(tiles) < 432
assert all([np.array_equal(a, b) for a, b in tiles])
# check that quality reductions are matched.
config = modelio.ModelIO({
'shuffle': False,
'jitter': False,
'skip': False,
'edges': True,
'quality': 0.5,
'residual': False
})
tiles = [tp for tp in frameops.tesselate_pair(_DPX, _DPX, config)]
assert len(tiles) == 432 // 2
assert all([np.array_equal(a, b) for a, b in tiles])
# turn everything on
config = modelio.ModelIO({
'shuffle': True,
'jitter': True,
'skip': True,
'edges': True,
'quality': 0.5,
'residual': False
})
tiles = [tp for tp in frameops.tesselate_pair(_DPX, _DPX, config)]
assert len(tiles) < 823
assert all([np.array_equal(a, b) for a, b in tiles])
assert not frameops.CACHED_TILES
assert not frameops.CACHED_QUALITY
def test_caching():
""" Test that tile caching is working correctly """
# cache off
frameops.reset_cache(False)
config = modelio.ModelIO({
'shuffle': False,
'jitter': False,
'skip': False,
'edges': True
})
extracted = frameops.extract_tiles(_DPX, config)
tiles = [t for t in frameops.tesselate(_DPX, config)]
assert not frameops.CACHED_TILES
assert not frameops.CACHED_QUALITY
# cache on, quality 100%
frameops.reset_cache(True)
config = modelio.ModelIO({
'shuffle': False,
'jitter': False,
'skip': False,
'edges': True
})
extracted = frameops.extract_tiles(_DPX, config)
tiles = [t for t in frameops.tesselate(_DPX, config)]
assert frameops.CACHED_TILES
assert not frameops.CACHED_QUALITY
assert _DPX in frameops.CACHED_TILES
tiles = frameops.CACHED_TILES[_DPX]
assert len(tiles) == len(extracted)
assert all([np.array_equal(a, b) for a, b in zip(tiles, extracted)])
# cache on, quality 50%
frameops.reset_cache(True)
config = modelio.ModelIO({
'shuffle': False,
'jitter': False,
'skip': False,
'edges': True,
'quality': 0.50
})
extracted = frameops.extract_tiles(_DPX, config)
tiles = [t for t in frameops.tesselate(_DPX, config)]
assert frameops.CACHED_TILES
assert frameops.CACHED_QUALITY
assert _DPX in frameops.CACHED_TILES
assert _DPX in frameops.CACHED_QUALITY
tiles = frameops.CACHED_TILES[_DPX]
assert len(tiles) == len(extracted) // 2
for tile in tiles:
matches = [np.array_equal(t, tile) for t in extracted]
assert any(matches)
del extracted[matches.index(True)]
def test_tesselate():
""" Test frameops.tesselate() """
# do not want caching to happen because of all the testing we are doing.
frameops.reset_cache(False)
# set up a default model configuration
config = modelio.ModelIO({
'shuffle': False,
'jitter': False,
'skip': False,
'edges': True
})
# get actual tiles to check against
extracted = frameops.extract_tiles(_DPX, config)
# tesselate() is a generator (will listify _DPX btw...)
tiles = [t for t in frameops.tesselate(_DPX, config)]
assert len(tiles) == len(extracted)
assert all([np.array_equal(a, b) for a, b in zip(tiles, extracted)])
# test jitter
config = modelio.ModelIO({
'shuffle': False,
'jitter': True,
'skip': False,
'edges': True
})
extracted = frameops.extract_tiles(_DPX, config)
tiles = [t for t in frameops.tesselate(_DPX, config)]
assert len(tiles) == len(extracted)
assert all([np.array_equal(a, b) for a, b in zip(tiles, extracted)])
# test skip
config = modelio.ModelIO({
'shuffle': False,
'jitter': False,
'skip': True,
'edges': True
})
extracted = frameops.extract_tiles(_DPX, config)
tiles = [t for t in frameops.tesselate(_DPX, config)]
# Checking for matches is a little tricky, because x in y where y is a
# list of numpy arrays is ambiguous.
assert len(tiles) < len(extracted)
for tile in tiles:
matches = [np.array_equal(t, tile) for t in extracted]
assert any(matches)
del extracted[matches.index(True)]
# test shuffle
config = modelio.ModelIO({
'shuffle': True,
'jitter': False,
'skip': False,
'edges': True
})
extracted = frameops.extract_tiles(_DPX, config)
tiles = [t for t in frameops.tesselate(_DPX, config)]
assert len(tiles) == len(extracted)
for tile in tiles:
matches = [np.array_equal(t, tile) for t in extracted]
assert any(matches)
del extracted[matches.index(True)]
# test shuffle + jitter
config = modelio.ModelIO({
'shuffle': True,
'jitter': True,
'skip': False,
'edges': True
})
extracted = frameops.extract_tiles(_DPX, config)
tiles = [t for t in frameops.tesselate(_DPX, config)]
assert len(tiles) == len(extracted)
for tile in tiles:
matches = [np.array_equal(t, tile) for t in extracted]
assert any(matches)
del extracted[matches.index(True)]
# test shuffle + jitter + skip
config = modelio.ModelIO({
'shuffle': True,
'jitter': True,
'skip': True,
'edges': True
})
extracted = frameops.extract_tiles(_DPX, config)
tiles = [t for t in frameops.tesselate(_DPX, config)]
assert len(tiles) < len(extracted)
for tile in tiles:
matches = [np.array_equal(t, tile) for t in extracted]
assert any(matches)
del extracted[matches.index(True)]
def test_extract_tiles():
""" Test frameops.extract_tiles() """
# do not want caching to happen because of all the testing we are doing.
frameops.reset_cache(False)
# set up a default model configuration
config = modelio.ModelIO({
'shuffle': False,
'jitter': False,
'skip': False,
'edges': True
})
# extract from a 1920x1080 png with 4x3 clipping
tiles = frameops.extract_tiles(_PNG, config)
assert len(tiles) == 432
for tile in tiles:
assert np.shape(tile) == (64, 64, 3)
# extract from a 720x480 dpx (with implicit scaling)
tiles = frameops.extract_tiles(_DPX, config)
assert len(tiles) == 432
for tile in tiles:
assert np.shape(tile) == (64, 64, 3)
# extract from a 720x480 dpx with jittering
config = modelio.ModelIO({
'shuffle': False,
'jitter': True,
'skip': False,
'edges': True
})
tiles = frameops.extract_tiles(_DPX, config)
assert len(tiles) == 823
for tile in tiles:
assert np.shape(tile) == (64, 64, 3)
# extract from a 720x480 dpx with no edges
config = modelio.ModelIO({
'shuffle': False,
'jitter': False,
'skip': False,
'edges': False
})
tiles = frameops.extract_tiles(_DPX, config)
assert len(tiles) == 391
for tile in tiles:
assert np.shape(tile) == (64, 64, 3)
# extract from a 720x480 dpx with no edges and jittering
config = modelio.ModelIO({
'shuffle': False,
'jitter': True,
'skip': False,
'edges': False
})
tiles = frameops.extract_tiles(_DPX, config)
assert len(tiles) == 743
for tile in tiles:
assert np.shape(tile) == (64, 64, 3)