def test_minibatch_source_config_other_properties(tmpdir):
ctf = create_ctf_deserializer(tmpdir)
config = MinibatchSourceConfig([ctf])
config.is_multithreaded.set(True)
config.trace_level = TraceLevel.Info.value
config.is_frame_mode_enabled = True
dictionary = to_dictionary(config)
assert 8 == len(dictionary.keys())
assert TraceLevel.Info == dictionary['traceLevel']
assert dictionary['frameMode'] is True
assert dictionary['multiThreadedDeserialization'] is True
config.is_multithreaded.set(False)
config.trace_level = 0
config.truncation_length = 123
with pytest.raises(Exception):
# to_dictionary will validate the config
dictionary = to_dictionary(config)
config.is_frame_mode_enabled = False
dictionary = to_dictionary(config)
assert 10 == len(dictionary.keys())
assert 0 == dictionary['traceLevel']
assert dictionary['frameMode'] is False
assert dictionary['multiThreadedDeserialization'] is False
assert dictionary['truncated'] is True
assert 123 == dictionary['truncationLength']
def test_minibatch_source_config_constructor(tmpdir):
ctf = create_ctf_deserializer(tmpdir)
config = MinibatchSourceConfig([ctf], False)
dictionary = to_dictionary(config)
check_default_config_keys(dictionary)
assert 5 == len(dictionary.keys())
assert dictionary['randomize'] is False
config = MinibatchSourceConfig([ctf], True)
dictionary = to_dictionary(config)
check_default_config_keys(dictionary)
assert 8 == len(dictionary.keys())
assert dictionary['randomize'] is True
assert DEFAULT_RANDOMIZATION_WINDOW_IN_CHUNKS == dictionary['randomizationWindow']
assert False == dictionary['sampleBasedRandomizationWindow']
config = MinibatchSourceConfig([ctf]) # 'randomize' is omitted
dictionary = to_dictionary(config)
check_default_config_keys(dictionary)
assert 8 == len(dictionary.keys())
assert dictionary['randomize'] is True
assert DEFAULT_RANDOMIZATION_WINDOW_IN_CHUNKS == dictionary['randomizationWindow']
assert False == dictionary['sampleBasedRandomizationWindow']
def test_minibatch_source_config_randomization(tmpdir):
ctf = create_ctf_deserializer(tmpdir)
config = MinibatchSourceConfig([ctf])
dictionary = to_dictionary(config)
check_default_config_keys(dictionary)
assert dictionary['randomize'] is True
config.randomization_window_in_chunks = 0
dictionary = to_dictionary(config)
check_default_config_keys(dictionary)
assert dictionary['randomize'] is False
config.randomization_window_in_chunks = 10
dictionary = to_dictionary(config)
check_default_config_keys(dictionary)
assert dictionary['randomize'] is True
assert 10 == dictionary['randomizationWindow']
assert dictionary['sampleBasedRandomizationWindow'] is False
config.randomization_window_in_samples = 100
with pytest.raises(Exception):
# to_dictionary will validate the config
dictionary = to_dictionary(config)
config.randomization_window_in_chunks = 0
dictionary = to_dictionary(config)
check_default_config_keys(dictionary)
assert dictionary['randomize'] is True
assert 100 == dictionary['randomizationWindow']
assert dictionary['sampleBasedRandomizationWindow'] is True
def test_minibatch_source_config_constructor(tmpdir):
ctf = create_ctf_deserializer(tmpdir)
config = MinibatchSourceConfig([ctf], False)
dictionary = to_dictionary(config)
check_default_config_keys(dictionary)
assert 5 == len(dictionary.keys())
assert dictionary['randomize'] is False
config = MinibatchSourceConfig([ctf], True)
dictionary = to_dictionary(config)
check_default_config_keys(dictionary)
assert 8 == len(dictionary.keys())
assert dictionary['randomize'] is True
assert DEFAULT_RANDOMIZATION_WINDOW_IN_CHUNKS == dictionary['randomizationWindow']
assert False == dictionary['sampleBasedRandomizationWindow']
config = MinibatchSourceConfig([ctf]) # 'randomize' is omitted
dictionary = to_dictionary(config)
check_default_config_keys(dictionary)
assert 8 == len(dictionary.keys())
assert dictionary['randomize'] is True
assert DEFAULT_RANDOMIZATION_WINDOW_IN_CHUNKS == dictionary['randomizationWindow']
assert False == dictionary['sampleBasedRandomizationWindow']
def test_minibatch_source_config_other_properties(tmpdir):
ctf = create_ctf_deserializer(tmpdir)
config = MinibatchSourceConfig([ctf])
config.is_multithreaded = True
config.trace_level = TraceLevel.Info.value
config.is_frame_mode_enabled = True
dictionary = to_dictionary(config)
assert 8 == len(dictionary.keys())
assert TraceLevel.Info == dictionary['traceLevel']
assert dictionary['frameMode'] is True
assert dictionary['multiThreadedDeserialization'] is True
config.is_multithreaded = False
config.trace_level = 0
config.truncation_length = 123
with pytest.raises(Exception):
# to_dictionary will validate the config
dictionary = to_dictionary(config)
config.is_frame_mode_enabled = False
dictionary = to_dictionary(config)
assert 10 == len(dictionary.keys())
assert 0 == dictionary['traceLevel']
assert dictionary['frameMode'] is False
assert dictionary['multiThreadedDeserialization'] is False
assert dictionary['truncated'] is True
assert 123 == dictionary['truncationLength']
def test_minibatch_source_config_randomization(tmpdir):
ctf = create_ctf_deserializer(tmpdir)
config = MinibatchSourceConfig([ctf])
dictionary = to_dictionary(config)
check_default_config_keys(dictionary)
assert dictionary['randomize'] is True
config.randomization_window_in_chunks = 0
dictionary = to_dictionary(config)
check_default_config_keys(dictionary)
assert dictionary['randomize'] is False
config.randomization_window_in_chunks = 10
dictionary = to_dictionary(config)
check_default_config_keys(dictionary)
assert dictionary['randomize'] is True
assert 10 == dictionary['randomizationWindow']
assert dictionary['sampleBasedRandomizationWindow'] is False
config.randomization_window_in_samples = 100
with pytest.raises(Exception):
# to_dictionary will validate the config
dictionary = to_dictionary(config)
config.randomization_window_in_chunks = 0
dictionary = to_dictionary(config)
check_default_config_keys(dictionary)
assert dictionary['randomize'] is True
assert 100 == dictionary['randomizationWindow']
assert dictionary['sampleBasedRandomizationWindow'] is True
def test_image():
map_file = "input.txt"
mean_file = "mean.txt"
feature_name = "f"
image_width = 100
image_height = 200
num_channels = 3
label_name = "l"
num_classes = 7
transforms = [
xforms.crop(crop_type='randomside', side_ratio=0.5,
jitter_type='uniratio'),
xforms.scale(width=image_width, height=image_height,
channels=num_channels, interpolations='linear'),
xforms.mean(mean_file)]
defs = StreamDefs(f=StreamDef(field='image', transforms=transforms),
l=StreamDef(field='label', shape=num_classes))
image = ImageDeserializer(map_file, defs)
config = to_dictionary(MinibatchSourceConfig([image], randomize=False))
assert len(config['deserializers']) == 1
d = config['deserializers'][0]
assert d['type'] == 'ImageDeserializer'
assert d['file'] == map_file
assert set(d['input'].keys()) == {label_name, feature_name}
l = d['input'][label_name]
assert l['labelDim'] == num_classes
f = d['input'][feature_name]
assert set(f.keys()) == {'transforms'}
t0, t1, t2, _ = f['transforms']
assert t0['type'] == 'Crop'
assert t1['type'] == 'Scale'
assert t2['type'] == 'Mean'
assert t0['cropType'] == 'randomside'
assert t0['sideRatio'] == 0.5
assert t0['aspectRatio'] == 1.0
assert t0['jitterType'] == 'uniratio'
assert t1['width'] == image_width
assert t1['height'] == image_height
assert t1['channels'] == num_channels
assert t1['interpolations'] == 'linear'
assert t2['meanFile'] == mean_file
config = to_dictionary(MinibatchSourceConfig([image, image]))
assert len(config['deserializers']) == 2
config = to_dictionary(MinibatchSourceConfig([image, image, image]))
assert len(config['deserializers']) == 3
# TODO depends on ImageReader.dll
'''
def test_image_with_crop_range():
map_file = "input.txt"
feature_name = "f"
image_width = 100
image_height = 200
num_channels = 3
label_name = "l"
num_classes = 7
transforms = [
xforms.crop(crop_type='randomside',
crop_size=(512, 424),
side_ratio=(0.2, 0.5),
area_ratio=(0.1, 0.75),
aspect_ratio=(0.3, 0.8),
jitter_type='uniratio')
]
defs = StreamDefs(f=StreamDef(field='image', transforms=transforms),
l=StreamDef(field='label', shape=num_classes))
image = ImageDeserializer(map_file, defs)
config = to_dictionary(MinibatchSourceConfig([image], randomize=False))
assert len(config['deserializers']) == 1
d = config['deserializers'][0]
assert d['type'] == 'ImageDeserializer'
assert d['file'] == map_file
assert set(d['input'].keys()) == {label_name, feature_name}
l = d['input'][label_name]
assert l['labelDim'] == num_classes
f = d['input'][feature_name]
assert set(f.keys()) == {'transforms'}
t0, _ = f['transforms']
assert t0['type'] == 'Crop'
assert t0['cropType'] == 'randomside'
assert t0['cropSize'] == '512:424'
assert t0['sideRatio'] == '0.2:0.5'
assert t0['aspectRatio'] == '0.3:0.8'
assert t0['areaRatio'] == '0.1:0.75'
assert t0['jitterType'] == 'uniratio'
config = to_dictionary(MinibatchSourceConfig([image, image]))
assert len(config['deserializers']) == 2
config = to_dictionary(MinibatchSourceConfig([image, image, image]))
assert len(config['deserializers']) == 3
def test_image_with_crop_range():
map_file = "input.txt"
feature_name = "f"
image_width = 100
image_height = 200
num_channels = 3
label_name = "l"
num_classes = 7
transforms = [
xforms.crop(crop_type='randomside',
crop_size=(512,424), side_ratio=(0.2, 0.5), area_ratio=(0.1, 0.75), aspect_ratio=(0.3, 0.8),
jitter_type='uniratio')
]
defs = StreamDefs(f=StreamDef(field='image', transforms=transforms),
l=StreamDef(field='label', shape=num_classes))
image = ImageDeserializer(map_file, defs)
config = to_dictionary(MinibatchSourceConfig([image], randomize=False))
assert len(config['deserializers']) == 1
d = config['deserializers'][0]
assert d['type'] == 'ImageDeserializer'
assert d['file'] == map_file
assert set(d['input'].keys()) == {label_name, feature_name}
l = d['input'][label_name]
assert l['labelDim'] == num_classes
f = d['input'][feature_name]
assert set(f.keys()) == {'transforms'}
t0, _ = f['transforms']
assert t0['type'] == 'Crop'
assert t0['cropType'] == 'randomside'
assert t0['cropSize'] == '512:424'
assert t0['sideRatio'] == '0.2:0.5'
assert t0['aspectRatio'] == '0.3:0.8'
assert t0['areaRatio'] == '0.1:0.75'
assert t0['jitterType'] == 'uniratio'
config = to_dictionary(MinibatchSourceConfig([image, image]))
assert len(config['deserializers']) == 2
config = to_dictionary(MinibatchSourceConfig([image, image, image]))
assert len(config['deserializers']) == 3
def test_minibatch_source_config_sweeps_and_samples(tmpdir):
ctf = create_ctf_deserializer(tmpdir)
config = MinibatchSourceConfig([ctf])
assert INFINITELY_REPEAT == config.max_samples
assert INFINITELY_REPEAT == config.max_sweeps
config.max_samples = 100
config.max_sweeps = 3
assert 100 == config.max_samples
assert 3 == config.max_sweeps
with pytest.raises(Exception):
# to_dictionary will validate the config
dictionary = to_dictionary(config)
config.max_samples = INFINITELY_REPEAT
dictionary = to_dictionary(config)
check_default_config_keys(dictionary)
def test_minibatch_source_config_sweeps_and_samples(tmpdir):
ctf = create_ctf_deserializer(tmpdir)
config = MinibatchSourceConfig([ctf])
assert INFINITELY_REPEAT == config.max_samples
assert INFINITELY_REPEAT == config.max_sweeps
config.max_samples = 100
config.max_sweeps = 3
assert 100 == config.max_samples
assert 3 == config.max_sweeps
with pytest.raises(Exception):
# to_dictionary will validate the config
dictionary = to_dictionary(config)
config.max_samples = INFINITELY_REPEAT
dictionary = to_dictionary(config)
check_default_config_keys(dictionary)
def test_image(tmpdir):
map_file = "input.txt"
mean_file = "mean.txt"
feature_name = "f"
image_width = 100
image_height = 200
num_channels = 3
label_name = "l"
num_classes = 7
transforms = [
xforms.crop(crop_type='randomside', side_ratio=0.5,
jitter_type='uniratio'),
xforms.scale(width=image_width, height=image_height,
channels=num_channels, interpolations='linear'),
xforms.mean(mean_file)]
defs = StreamDefs(f=StreamDef(field='image', transforms=transforms),
l=StreamDef(field='label', shape=num_classes))
image = ImageDeserializer(map_file, defs)
config = to_dictionary(MinibatchSourceConfig([image], randomize=False))
# Multithreading should be on by default for the ImageDeserializer.
assert config['multiThreadedDeserialization'] is True
assert len(config['deserializers']) == 1
d = config['deserializers'][0]
assert d['type'] == 'ImageDeserializer'
assert d['file'] == map_file
assert set(d['input'].keys()) == {label_name, feature_name}
l = d['input'][label_name]
assert l['labelDim'] == num_classes
f = d['input'][feature_name]
assert set(f.keys()) == {'transforms'}
t0, t1, t2, _ = f['transforms']
assert t0['type'] == 'Crop'
assert t1['type'] == 'Scale'
assert t2['type'] == 'Mean'
assert t0['cropType'] == 'randomside'
assert t0['cropSize'] == '0:0'
assert t0['sideRatio'] == '0.5:0.5'
assert t0['aspectRatio'] == '1:1'
assert t0['areaRatio'] == '0:0'
assert t0['jitterType'] == 'uniratio'
assert t1['width'] == image_width
assert t1['height'] == image_height
assert t1['channels'] == num_channels
assert t1['interpolations'] == 'linear'
assert t2['meanFile'] == mean_file
config = to_dictionary(MinibatchSourceConfig([image, image]))
assert len(config['deserializers']) == 2
ctf = create_ctf_deserializer(tmpdir)
config = to_dictionary(MinibatchSourceConfig([image, ctf, image]))
# Multithreading should still be enabled.
assert config['multiThreadedDeserialization'] is True
assert len(config['deserializers']) == 3
# TODO depends on ImageReader.dll
'''
