def test_minibatch_source_config_constructor(tmpdir): ctf = create_ctf_deserializer(tmpdir) config = MinibatchSourceConfig([ctf], False) dictionary = to_dictionary(config) check_default_config_keys(dictionary) assert 5 == len(dictionary.keys()) assert dictionary['randomize'] is False config = MinibatchSourceConfig([ctf], True) dictionary = to_dictionary(config) check_default_config_keys(dictionary) assert 8 == len(dictionary.keys()) assert dictionary['randomize'] is True assert DEFAULT_RANDOMIZATION_WINDOW_IN_CHUNKS == dictionary[ 'randomizationWindow'] assert False == dictionary['sampleBasedRandomizationWindow'] config = MinibatchSourceConfig([ctf]) # 'randomize' is omitted dictionary = to_dictionary(config) check_default_config_keys(dictionary) assert 8 == len(dictionary.keys()) assert dictionary['randomize'] is True assert DEFAULT_RANDOMIZATION_WINDOW_IN_CHUNKS == dictionary[ 'randomizationWindow'] assert False == dictionary['sampleBasedRandomizationWindow']
def test_minibatch_source_config_randomization(tmpdir): ctf = create_ctf_deserializer(tmpdir) config = MinibatchSourceConfig([ctf]) dictionary = to_dictionary(config) check_default_config_keys(dictionary) assert dictionary['randomize'] is True config.randomization_window_in_chunks = 0 dictionary = to_dictionary(config) check_default_config_keys(dictionary) assert dictionary['randomize'] is False config.randomization_window_in_chunks = 10 dictionary = to_dictionary(config) check_default_config_keys(dictionary) assert dictionary['randomize'] is True assert 10 == dictionary['randomizationWindow'] assert dictionary['sampleBasedRandomizationWindow'] is False config.randomization_window_in_samples = 100 with pytest.raises(Exception): # to_dictionary will validate the config dictionary = to_dictionary(config) config.randomization_window_in_chunks = 0 dictionary = to_dictionary(config) check_default_config_keys(dictionary) assert dictionary['randomize'] is True assert 100 == dictionary['randomizationWindow'] assert dictionary['sampleBasedRandomizationWindow'] is True
def test_image(): map_file = "input.txt" mean_file = "mean.txt" feature_name = "f" image_width = 100 image_height = 200 num_channels = 3 label_name = "l" num_classes = 7 transforms = [ xforms.crop(crop_type='randomside', side_ratio=0.5, jitter_type='uniratio'), xforms.scale(width=image_width, height=image_height, channels=num_channels, interpolations='linear'), xforms.mean(mean_file)] defs = StreamDefs(f=StreamDef(field='image', transforms=transforms), l=StreamDef(field='label', shape=num_classes)) image = ImageDeserializer(map_file, defs) config = to_dictionary(MinibatchSourceConfig([image], randomize=False)) assert len(config['deserializers']) == 1 d = config['deserializers'][0] assert d['type'] == 'ImageDeserializer' assert d['file'] == map_file assert set(d['input'].keys()) == {label_name, feature_name} l = d['input'][label_name] assert l['labelDim'] == num_classes f = d['input'][feature_name] assert set(f.keys()) == {'transforms'} t0, t1, t2, _ = f['transforms'] assert t0['type'] == 'Crop' assert t1['type'] == 'Scale' assert t2['type'] == 'Mean' assert t0['cropType'] == 'randomside' assert t0['sideRatio'] == 0.5 assert t0['aspectRatio'] == 1.0 assert t0['jitterType'] == 'uniratio' assert t1['width'] == image_width assert t1['height'] == image_height assert t1['channels'] == num_channels assert t1['interpolations'] == 'linear' assert t2['meanFile'] == mean_file config = to_dictionary(MinibatchSourceConfig([image, image])) assert len(config['deserializers']) == 2 config = to_dictionary(MinibatchSourceConfig([image, image, image])) assert len(config['deserializers']) == 3 # TODO depends on ImageReader.dll '''
def test_minibatch_source_config_other_properties(tmpdir): ctf = create_ctf_deserializer(tmpdir) config = MinibatchSourceConfig([ctf]) config.is_multithreaded = True config.trace_level = TraceLevel.Info.value config.is_frame_mode_enabled = True dictionary = to_dictionary(config) assert 8 == len(dictionary.keys()) assert TraceLevel.Info == dictionary['traceLevel'] assert dictionary['frameMode'] is True assert dictionary['multiThreadedDeserialization'] is True config.is_multithreaded = False config.trace_level = 0 config.truncation_length = 123 with pytest.raises(Exception): # to_dictionary will validate the config dictionary = to_dictionary(config) config.is_frame_mode_enabled = False dictionary = to_dictionary(config) assert 10 == len(dictionary.keys()) assert 0 == dictionary['traceLevel'] assert dictionary['frameMode'] is False assert dictionary['multiThreadedDeserialization'] is False assert dictionary['truncated'] is True assert 123 == dictionary['truncationLength']
def test_image_with_crop_range(): map_file = "input.txt" feature_name = "f" image_width = 100 image_height = 200 num_channels = 3 label_name = "l" num_classes = 7 transforms = [ xforms.crop(crop_type='randomside', crop_size=(512, 424), side_ratio=(0.2, 0.5), area_ratio=(0.1, 0.75), aspect_ratio=(0.3, 0.8), jitter_type='uniratio') ] defs = StreamDefs(f=StreamDef(field='image', transforms=transforms), l=StreamDef(field='label', shape=num_classes)) image = ImageDeserializer(map_file, defs) config = to_dictionary(MinibatchSourceConfig([image], randomize=False)) assert len(config['deserializers']) == 1 d = config['deserializers'][0] assert d['type'] == 'ImageDeserializer' assert d['file'] == map_file assert set(d['input'].keys()) == {label_name, feature_name} l = d['input'][label_name] assert l['labelDim'] == num_classes f = d['input'][feature_name] assert set(f.keys()) == {'transforms'} t0, _ = f['transforms'] assert t0['type'] == 'Crop' assert t0['cropType'] == 'randomside' assert t0['cropSize'] == '512:424' assert t0['sideRatio'] == '0.2:0.5' assert t0['aspectRatio'] == '0.3:0.8' assert t0['areaRatio'] == '0.1:0.75' assert t0['jitterType'] == 'uniratio' config = to_dictionary(MinibatchSourceConfig([image, image])) assert len(config['deserializers']) == 2 config = to_dictionary(MinibatchSourceConfig([image, image, image])) assert len(config['deserializers']) == 3
def test_minibatch_source_config_sweeps_and_samples(tmpdir): ctf = create_ctf_deserializer(tmpdir) config = MinibatchSourceConfig([ctf]) assert INFINITELY_REPEAT == config.max_samples assert INFINITELY_REPEAT == config.max_sweeps config.max_samples = 100 config.max_sweeps = 3 assert 100 == config.max_samples assert 3 == config.max_sweeps with pytest.raises(Exception): # to_dictionary will validate the config dictionary = to_dictionary(config) config.max_samples = INFINITELY_REPEAT dictionary = to_dictionary(config) check_default_config_keys(dictionary)
def test_minibatch_source_config_sweeps_and_samples(tmpdir): ctf = create_ctf_deserializer(tmpdir) config = MinibatchSourceConfig([ctf]) assert INFINITELY_REPEAT == config.max_samples assert INFINITELY_REPEAT == config.max_sweeps config.max_samples = 100 config.max_sweeps = 3 assert 100 == config.max_samples assert 3 == config.max_sweeps with pytest.raises(Exception): # to_dictionary will validate the config dictionary = to_dictionary(config) config.max_samples = INFINITELY_REPEAT dictionary = to_dictionary(config) check_default_config_keys(dictionary)
def test_minibatch_source_config_randomization(tmpdir): ctf = create_ctf_deserializer(tmpdir) config = MinibatchSourceConfig([ctf]) dictionary = to_dictionary(config) check_default_config_keys(dictionary) assert dictionary['randomize'] is True config.randomization_window_in_chunks = 0 dictionary = to_dictionary(config) check_default_config_keys(dictionary) assert dictionary['randomize'] is False config.randomization_window_in_chunks = 10 dictionary = to_dictionary(config) check_default_config_keys(dictionary) assert dictionary['randomize'] is True assert 10 == dictionary['randomizationWindow'] assert dictionary['sampleBasedRandomizationWindow'] is False config.randomization_window_in_samples = 100 with pytest.raises(Exception): # to_dictionary will validate the config dictionary = to_dictionary(config) config.randomization_window_in_chunks = 0 dictionary = to_dictionary(config) check_default_config_keys(dictionary) assert dictionary['randomize'] is True assert 100 == dictionary['randomizationWindow'] assert dictionary['sampleBasedRandomizationWindow'] is True
def test_minibatch_source_config_other_properties(tmpdir): ctf = create_ctf_deserializer(tmpdir) config = MinibatchSourceConfig([ctf]) config.is_multithreaded.set(True) config.trace_level = TraceLevel.Info.value config.is_frame_mode_enabled = True dictionary = to_dictionary(config) assert 8 == len(dictionary.keys()) assert TraceLevel.Info == dictionary['traceLevel'] assert dictionary['frameMode'] is True assert dictionary['multiThreadedDeserialization'] is True config.is_multithreaded.set(False) config.trace_level = 0 config.truncation_length = 123 with pytest.raises(Exception): # to_dictionary will validate the config dictionary = to_dictionary(config) config.is_frame_mode_enabled = False dictionary = to_dictionary(config) assert 10 == len(dictionary.keys()) assert 0 == dictionary['traceLevel'] assert dictionary['frameMode'] is False assert dictionary['multiThreadedDeserialization'] is False assert dictionary['truncated'] is True assert 123 == dictionary['truncationLength']
def create_config(tmpdir): tmpfile = create_temp_file(tmpdir) return MinibatchSourceConfig() \ .add_deserializer( CTFDeserializer(tmpfile, StreamDefs(features=StreamDef(field='S0', shape=1))))