def create_image_mb_source(map_file, mean_file, is_training, total_number_of_samples):
if not os.path.exists(map_file) or not os.path.exists(mean_file):
raise RuntimeError("File '%s' or '%s' does not exist." %
(map_file, mean_file))
# transformation pipeline for the features has jitter/crop only when training
transforms = []
if is_training:
transforms += [
xforms.crop(crop_type='randomside', side_ratio=0.8, jitter_type='uniratio') # train uses jitter
]
else:
transforms += [
xforms.crop(crop_type='center', crop_size=IMAGE_WIDTH)
]
transforms += [
xforms.scale(width=IMAGE_WIDTH, height=IMAGE_HEIGHT, channels=NUM_CHANNELS, interpolations='linear'),
xforms.mean(mean_file)
]
# deserializer
return MinibatchSource(
ImageDeserializer(map_file, StreamDefs(
features=StreamDef(field='image', transforms=transforms), # first column in map file is referred to as 'image'
labels=StreamDef(field='label', shape=NUM_CLASSES))), # and second as 'label'
randomize=is_training,
max_samples=total_number_of_samples,
multithreaded_deserializer = True)
def create_image_mb_source(map_file, mean_file, train, total_number_of_samples):
if not os.path.exists(map_file) or not os.path.exists(mean_file):
raise RuntimeError("File '%s' or '%s' does not exist." % (map_file, mean_file))
# transformation pipeline for the features has jitter/crop only when training
transforms = []
if train:
transforms += [
xforms.crop(crop_type='randomarea', area_ratio=(0.08, 1.0), aspect_ratio=(0.75, 1.3333), jitter_type='uniratio'), # train uses jitter
xforms.color(brightness_radius=0.4, contrast_radius=0.4, saturation_radius=0.4)
]
else:
transforms += [
C.io.transforms.crop(crop_type='center', side_ratio=0.875) # test has no jitter
]
transforms += [
xforms.scale(width=image_width, height=image_height, channels=num_channels, interpolations='cubic'),
xforms.mean(mean_file)
]
# deserializer
return C.io.MinibatchSource(
C.io.ImageDeserializer(map_file, C.io.StreamDefs(
features=C.io.StreamDef(field='image', transforms=transforms), # 1st col in mapfile referred to as 'image'
labels=C.io.StreamDef(field='label', shape=num_classes))), # and second as 'label'
randomize=train,
max_samples=total_number_of_samples,
multithreaded_deserializer=True)
def create_reader(map_file, is_train):
transforms = [xforms.crop(crop_type="center", side_ratio=0.9),
xforms.scale(width=img_width, height=img_height, channels=img_channel, interpolations="linear")]
return C.io.MinibatchSource(C.io.ImageDeserializer(map_file, C.io.StreamDefs(
image=C.io.StreamDef(field="image", transforms=transforms),
dummy=C.io.StreamDef(field="label", shape=num_classes))),
randomize=is_train, max_sweeps=C.io.INFINITELY_REPEAT if is_train else 1)
def test_base64_image_deserializer(tmpdir):
import io, base64, uuid
from PIL import Image
images, b64_images = [], []
np.random.seed(1)
for i in range(10):
data = np.random.randint(0, 2**8, (5, 7, 3))
image = Image.fromarray(data.astype('uint8'), "RGB")
buf = io.BytesIO()
image.save(buf, format='PNG')
assert image.width == 7 and image.height == 5
b64_images.append(base64.b64encode(buf.getvalue()))
images.append(np.array(image))
image_data = str(tmpdir / 'mbdata1.txt')
seq_ids = []
uid = uuid.uuid1().int >> 64
with open(image_data, 'wb') as f:
for i, data in enumerate(b64_images):
seq_id = uid ^ i
seq_id = str(seq_id).encode('ascii')
seq_ids.append(seq_id)
line = seq_id + b'\t'
label = str(i).encode('ascii')
line += label + b'\t' + data + b'\n'
f.write(line)
ctf_data = str(tmpdir / 'mbdata2.txt')
with open(ctf_data, 'wb') as f:
for i, sid in enumerate(seq_ids):
line = sid + b'\t' + b'|index ' + str(i).encode('ascii') + b'\n'
f.write(line)
transforms = [xforms.scale(width=7, height=5, channels=3)]
b64_deserializer = Base64ImageDeserializer(
image_data,
StreamDefs(images=StreamDef(field='image', transforms=transforms),
labels=StreamDef(field='label', shape=10)))
ctf_deserializer = CTFDeserializer(
ctf_data, StreamDefs(index=StreamDef(field='index', shape=1)))
mb_source = MinibatchSource([ctf_deserializer, b64_deserializer])
assert isinstance(mb_source, MinibatchSource)
for j in range(100):
mb = mb_source.next_minibatch(10)
index_stream = mb_source.streams['index']
index = mb[index_stream].asarray().flatten()
image_stream = mb_source.streams['images']
results = mb[image_stream].asarray()
for i in range(10):
# original images are RBG, openCV produces BGR images,
# reverse the last dimension of the original images
bgrImage = images[int(index[i])][:, :, ::-1]
assert (bgrImage == results[i][0]).all()
def create_mb_source(image_height, image_width, num_channels, map_file, mean_file, is_training):
if not os.path.exists(map_file):
raise RuntimeError("File '%s' does not exist." % (map_file))
# transformation pipeline for the features has jitter/crop only when training
transforms = []
if is_training:
transforms += [
xforms.crop(crop_type='randomside', side_ratio=0.875, jitter_type='uniratio') # train uses jitter
]
else:
transforms += [
xforms.crop(crop_type='center', side_ratio=0.875) # test has no jitter
]
transforms += [
xforms.scale(width=image_width, height=image_height, channels=num_channels, interpolations='linear'),
]
if mean_file != '':
transforms += [
xforms.mean(mean_file),
]
# deserializer
return MinibatchSource(
ImageDeserializer(map_file, StreamDefs(
features = StreamDef(field='image', transforms=transforms) # first column in map file is referred to as 'image'
)),
randomize = is_training,
multithreaded_deserializer = True,
max_sweeps = 1)
def create_reader(map_file, mean_file, is_training):
if not os.path.exists(map_file) or not os.path.exists(mean_file):
raise RuntimeError(
"File '%s' or '%s' does not exist. Please run install_cifar10.py from DataSets/CIFAR-10 to fetch them"
% (map_file, mean_file))
# transformation pipeline for the features has jitter/crop only when training
transforms = []
if is_training:
transforms += [
xforms.crop(crop_type='randomside',
side_ratio=0.8,
jitter_type='uniratio') # train uses jitter
]
transforms += [
xforms.scale(width=image_width,
height=image_height,
channels=num_channels,
interpolations='linear'),
xforms.mean(mean_file)
]
# deserializer
return MinibatchSource(
ImageDeserializer(
map_file,
StreamDefs(
features=StreamDef(
field='image', transforms=transforms
), # first column in map file is referred to as 'image'
labels=StreamDef(field='label',
shape=num_classes))), # and second as 'label'
randomize=is_training,
max_sweeps=INFINITELY_REPEAT if is_training else 1)
def create_image_mb_source(map_file, is_training, total_number_of_samples):
if not os.path.exists(map_file):
raise RuntimeError("File '%s' does not exist." %map_file)
# transformation pipeline for the features has jitter/crop only when training
transforms = []
if is_training:
transforms += [
xforms.crop(crop_type='randomside', side_ratio=0.4375:0.875, jitter_type='uniratio') # train uses jitter
]
else:
transforms += [
xforms.crop(crop_type='center', side_ratio=0.5833333) # test has no jitter
]
transforms += [
xforms.scale(width=image_width, height=image_height, channels=num_channels, interpolations='linear'),
]
# deserializer
return MinibatchSource(
ImageDeserializer(map_file, StreamDefs(
features = StreamDef(field='image', transforms=transforms), # first column in map file is referred to as 'image'
labels = StreamDef(field='label', shape=num_classes))), # and second as 'label'
randomize = is_training,
epoch_size=total_number_of_samples,
multithreaded_deserializer = True)
def create_reader(map_file, mean_file, train):
# transformation pipeline for the features has jitter/crop only when training
trs = []
# if train:
# transforms += [
# ImageDeserializer.crop(crop_type='Random', ratio=0.8, jitter_type='uniRatio') # train uses jitter
# ]
trs += [
transforms.scale(width=image_width,
height=image_height,
channels=num_channels,
interpolations='linear'),
transforms.mean(mean_file)
]
# deserializer
return MinibatchSource(
ImageDeserializer(
map_file,
StreamDefs(
features=StreamDef(
field='image', transforms=trs
), # first column in map file is referred to as 'image'
labels=StreamDef(field='label',
shape=num_classes) # and second as 'label'
)))
def create_image_mb_source(map_file, mean_file, train,
total_number_of_samples):
if not os.path.exists(map_file) or not os.path.exists(mean_file):
raise RuntimeError(
"File '%s' or '%s' does not exist. Please run install_cifar10.py from DataSets/CIFAR-10 to fetch them"
% (map_file, mean_file))
# transformation pipeline for the features has jitter/crop only when training
transforms = []
# if train:
# transforms += [
# xforms.crop(crop_type='randomside', side_ratio=(0.8, 1.0), jitter_type='uniratio') # train uses jitter
# ]
transforms += [
xforms.scale(width=image_width,
height=image_height,
channels=num_channels,
interpolations='linear'),
xforms.mean(mean_file)
]
# deserializer
return MinibatchSource(
ImageDeserializer(
map_file,
StreamDefs(
features=StreamDef(
field='image', transforms=transforms
), # first column in map file is referred to as 'image'
labels=StreamDef(field='label',
shape=num_classes))), # and second as 'label'
randomize=train,
max_samples=total_number_of_samples,
multithreaded_deserializer=True)
def create_mb_source(image_height, image_width, num_channels, map_file):
transforms = [xforms.scale(width=image_width, height=image_height, channels=num_channels, interpolations='linear')]
return C.io.MinibatchSource(
C.io.ImageDeserializer(map_file, C.io.StreamDefs(
features=C.io.StreamDef(field='image', transforms=transforms),
labels=C.io.StreamDef(field='label', shape=1000))),
randomize=False)
def create_mb_source(image_height, image_width, num_channels, map_file):
transforms = [xforms.scale(width=image_width, height=image_height, channels=num_channels, interpolations='linear')]
return C.io.MinibatchSource(
C.io.ImageDeserializer(map_file, C.io.StreamDefs(
features=C.io.StreamDef(field='image', transforms=transforms),
labels=C.io.StreamDef(field='label', shape=1000))),
randomize=False)
def create_image_mb_source(map_file, mean_file, train, total_number_of_samples):
if not os.path.exists(map_file) or not os.path.exists(mean_file):
raise RuntimeError("File '%s' or '%s' does not exist." % (map_file, mean_file))
# transformation pipeline for the features has jitter/crop only when training
transforms = []
if train:
transforms += [
xforms.crop(crop_type='randomarea', area_ratio=(0.08, 1.0), aspect_ratio=(0.75, 1.3333), jitter_type='uniratio'), # train uses jitter
xforms.color(brightness_radius=0.4, contrast_radius=0.4, saturation_radius=0.4)
]
else:
transforms += [
C.io.transforms.crop(crop_type='center', side_ratio=0.875) # test has no jitter
]
transforms += [
xforms.scale(width=image_width, height=image_height, channels=num_channels, interpolations='cubic'),
xforms.mean(mean_file)
]
# deserializer
return C.io.MinibatchSource(
C.io.ImageDeserializer(map_file, C.io.StreamDefs(
features=C.io.StreamDef(field='image', transforms=transforms), # 1st col in mapfile referred to as 'image'
labels=C.io.StreamDef(field='label', shape=num_classes))), # and second as 'label'
randomize=train,
max_samples=total_number_of_samples,
multithreaded_deserializer=True)
def create_reader(map_file, train, dimensions, classes,
total_number_of_samples):
print(
f"Reading map file: {map_file} with number of samples {total_number_of_samples}"
)
# transformation pipeline for the features has jitter/crop only when training
transforms = []
# finalize_network uses data augmentation (translation only)
if train:
transforms += [
xforms.crop(crop_type='randomside',
area_ratio=(0.08, 1.0),
aspect_ratio=(0.75, 1.3333),
jitter_type='uniratio'),
xforms.color(brightness_radius=0.4,
contrast_radius=0.4,
saturation_radius=0.4)
]
transforms += [
xforms.scale(width=dimensions['width'],
height=dimensions['height'],
channels=dimensions['depth'],
interpolations='linear')
]
source = MinibatchSource(ImageDeserializer(
map_file,
StreamDefs(features=StreamDef(field='image', transforms=transforms),
labels=StreamDef(field='label', shape=len(classes)))),
randomize=train,
max_samples=total_number_of_samples,
multithreaded_deserializer=True)
return source
def create_video_mb_source(map_files, num_channels, image_height, image_width,
num_classes):
transforms = [
xforms.crop(crop_type='center', crop_size=224),
xforms.scale(width=image_width,
height=image_height,
channels=num_channels,
interpolations='linear')
]
map_files = sorted(map_files,
key=lambda x: int(x.split('Map_')[1].split('.')[0]))
print(map_files)
# Create multiple image sources
sources = []
for i, map_file in enumerate(map_files):
streams = {
"feature" + str(i): StreamDef(field='image',
transforms=transforms),
"label" + str(i): StreamDef(field='label', shape=num_classes)
}
sources.append(ImageDeserializer(map_file, StreamDefs(**streams)))
return MinibatchSource(sources, max_sweeps=1, randomize=False)
def create_image_mb_source(map_file, train, total_number_of_samples):
'''
Input: map_file, train:bool, total_num_of_samples
Function:
- checks if it is training or testing phase
- for training: It applies Image Augmentation techniques like cropping, width_shift, height_shift,
horizontal_flip, color_contrast to prevent overfitting of the model.
Return: MinibatchSource to be fed into the CNTK Model
'''
print('Creating source for {}.'.format(map_file))
transforms = []
if train:
# Apply translational and color transformations only for the Image set
transforms += [
xforms.crop(crop_type='randomarea', area_ratio=(0.08, 1.0), aspect_ratio=(0.75, 1), jitter_type='uniratio'), # train uses jitter
xforms.color(brightness_radius=0.4, contrast_radius=0.4, saturation_radius=0.4)
]
# Scale the images to a specified size (224 x 224) as expected by the model
transforms += [
xforms.scale(width=image_width, height=image_height, channels=num_channels, interpolations='cubic')
]
return C.io.MinibatchSource(
C.io.ImageDeserializer(
map_file,
C.io.StreamDefs(features=C.io.StreamDef(field='image', transforms=transforms), # 1st col in mapfile referred to as 'image'
labels=C.io.StreamDef(field='label', shape=num_classes))), # and second as 'label'
randomize=train,
max_samples=total_number_of_samples,
multithreaded_deserializer=False)
def create_mb_source(data_set, img_height, img_width, n_classes, n_rois, data_path, randomize):
# set paths
map_file = join(data_path, data_set + '.txt')
roi_file = join(data_path, data_set + '.rois.txt')
label_file = join(data_path, data_set + '.roilabels.txt')
if not os.path.exists(map_file) or not os.path.exists(roi_file) or not os.path.exists(label_file):
raise RuntimeError("File '%s', '%s' or '%s' does not exist. " % (map_file, roi_file, label_file))
# read images
nrImages = len(readTable(map_file))
transforms = [scale(width=img_width, height=img_height, channels=3,
scale_mode="pad", pad_value=114, interpolations='linear')]
image_source = ImageDeserializer(map_file, StreamDefs(features = StreamDef(field='image', transforms=transforms)))
# read rois and labels
rois_dim = 4 * n_rois
label_dim = n_classes * n_rois
roi_source = CTFDeserializer(roi_file, StreamDefs(
rois = StreamDef(field='rois', shape=rois_dim, is_sparse=False)))
label_source = CTFDeserializer(label_file, StreamDefs(
roiLabels = StreamDef(field='roiLabels', shape=label_dim, is_sparse=False)))
# define a composite reader
mb = MinibatchSource([image_source, roi_source, label_source], epoch_size=sys.maxsize, randomize=randomize)
return (mb, nrImages)
def create_reader(map_file, mean_file, is_training):
print("Reading map file:", map_file)
print("Reading mean file:", mean_file)
if not os.path.exists(map_file) or not os.path.exists(mean_file):
raise RuntimeError(
"This tutorials depends 201A tutorials, please run 201A first.")
# transformation pipeline for the features has jitter/crop only when training
transforms = []
# train uses data augmentation (translation only)
if is_training:
transforms += [xforms.crop(crop_type='randomside', side_ratio=0.8)]
transforms += [
xforms.scale(width=image_width,
height=image_height,
channels=num_channels,
interpolations='linear'),
xforms.mean(mean_file)
]
# deserializer
return C.io.MinibatchSource(
C.io.ImageDeserializer(
map_file,
C.io.StreamDefs(
features=C.io.StreamDef(
field='image', transforms=transforms
), # first column in map file is referred to as 'image'
labels=C.io.StreamDef(
field='label', shape=num_classes) # and second as 'label'
)))
def create_image_mb_source(map_file, mean_file, train, total_number_of_samples):
if not os.path.exists(map_file) or not os.path.exists(mean_file):
raise RuntimeError("File '%s' or '%s' does not exist. Please run install_cifar10.py from DataSets/CIFAR-10 to fetch them" %
(map_file, mean_file))
# transformation pipeline for the features has jitter/crop only when training
transforms = []
if train:
transforms += [
xforms.crop(crop_type='randomside', side_ratio=0.8, jitter_type='uniratio') # train uses jitter
]
transforms += [
xforms.scale(width=image_width, height=image_height, channels=num_channels, interpolations='linear'),
xforms.mean(mean_file)
]
# deserializer
return cntk.io.MinibatchSource(
cntk.io.ImageDeserializer(map_file, cntk.io.StreamDefs(
features = cntk.io.StreamDef(field='image', transforms=transforms), # first column in map file is referred to as 'image'
labels = cntk.io.StreamDef(field='label', shape=num_classes))), # and second as 'label'
randomize=train,
epoch_size=total_number_of_samples,
multithreaded_deserializer = True)
def create_reader(map_file, mean_file, train):
"""
Define the reader for both training and evaluation action.
"""
if not os.path.exists(map_file) or not os.path.exists(mean_file):
raise RuntimeError("This example require prepared dataset. \
Please run cifar_prepare.py example.")
transforms = []
if train:
transforms += [xforms.crop(crop_type='randomside', side_ratio=0.8)]
transforms += [
xforms.scale(width=image_width,
height=image_height,
channels=num_channels,
interpolations='linear'),
xforms.mean(mean_file)
]
return C.io.MinibatchSource(
C.io.ImageDeserializer(
map_file,
C.io.StreamDefs(features=C.io.StreamDef(field='image',
transforms=transforms),
labels=C.io.StreamDef(field='label',
shape=num_classes))))
def create_mb_source(img_height, img_width, img_channels, n_classes, n_rois, data_path, data_set):
rois_dim = 4 * n_rois
label_dim = n_classes * n_rois
path = os.path.normpath(os.path.join(abs_path, data_path))
if data_set == 'test':
map_file = os.path.join(path, test_map_filename)
else:
map_file = os.path.join(path, train_map_filename)
roi_file = os.path.join(path, data_set + rois_filename_postfix)
label_file = os.path.join(path, data_set + roilabels_filename_postfix)
if not os.path.exists(map_file) or not os.path.exists(roi_file) or not os.path.exists(label_file):
raise RuntimeError("File '%s', '%s' or '%s' does not exist. "
"Please run install_fastrcnn.py from Examples/Image/Detection/FastRCNN to fetch them" %
(map_file, roi_file, label_file))
# read images
transforms = [scale(width=img_width, height=img_height, channels=img_channels,
scale_mode="pad", pad_value=114, interpolations='linear')]
image_source = ImageDeserializer(map_file, StreamDefs(
features = StreamDef(field='image', transforms=transforms)))
# read rois and labels
roi_source = CTFDeserializer(roi_file, StreamDefs(
rois = StreamDef(field=roi_stream_name, shape=rois_dim, is_sparse=False)))
label_source = CTFDeserializer(label_file, StreamDefs(
roiLabels = StreamDef(field=label_stream_name, shape=label_dim, is_sparse=False)))
# define a composite reader
return MinibatchSource([image_source, roi_source, label_source], epoch_size=sys.maxsize, randomize=data_set == "train")
def create_image_mb_source(map_file, is_training, total_number_of_samples):
if not os.path.exists(map_file):
raise RuntimeError("File '%s' does not exist." %map_file)
# transformation pipeline for the features has jitter/crop only when training
transforms = []
if is_training:
transforms += [
xforms.crop(crop_type='randomside', side_ratio=0.88671875, jitter_type='uniratio') # train uses jitter
]
else:
transforms += [
xforms.crop(crop_type='center', side_ratio=0.88671875) # test has no jitter
]
transforms += [
xforms.scale(width=image_width, height=image_height, channels=num_channels, interpolations='linear'),
]
# deserializer
return MinibatchSource(
ImageDeserializer(map_file, StreamDefs(
features=StreamDef(field='image', transforms=transforms), # first column in map file is referred to as 'image'
labels=StreamDef(field='label', shape=num_classes))), # and second as 'label'
randomize=is_training,
max_samples=total_number_of_samples,
multithreaded_deserializer=True)
def create_reader(map_file, mean_file, is_training):
if not os.path.exists(map_file) or not os.path.exists(mean_file):
raise RuntimeError(
"File '%s' or '%s' does not exist. Please run install_cifar10.py from DataSets/CIFAR-10 to fetch them"
% (map_file, mean_file))
# crop transform 40 * 0.8 = 32
transforms = []
if is_training:
transforms += [
xforms.crop(crop_type='RandomSide',
side_ratio=0.8,
jitter_type='uniRatio') # train uses jitter
]
transforms += [
xforms.scale(width=image_width,
height=image_height,
channels=num_channels,
interpolations='linear'),
xforms.mean(mean_file)
]
# deserializer
return C.io.MinibatchSource(C.io.ImageDeserializer(
map_file,
C.io.StreamDefs(features=C.io.StreamDef(field='image',
transforms=transforms),
labels=C.io.StreamDef(field='label',
shape=num_classes))),
randomize=is_training)
def create_mb_source(img_height, img_width, img_channels, n_classes, n_rois, data_path, data_set):
rois_dim = 4 * n_rois
label_dim = n_classes * n_rois
path = os.path.normpath(os.path.join(abs_path, data_path))
if data_set == 'test':
map_file = os.path.join(path, test_map_filename)
else:
map_file = os.path.join(path, train_map_filename)
roi_file = os.path.join(path, data_set + rois_filename_postfix)
label_file = os.path.join(path, data_set + roilabels_filename_postfix)
if not os.path.exists(map_file) or not os.path.exists(roi_file) or not os.path.exists(label_file):
raise RuntimeError("File '%s', '%s' or '%s' does not exist. "
"Please run install_data_and_model.py from Examples/Image/Detection/FastRCNN to fetch them" %
(map_file, roi_file, label_file))
# read images
transforms = [scale(width=img_width, height=img_height, channels=img_channels,
scale_mode="pad", pad_value=114, interpolations='linear')]
image_source = ImageDeserializer(map_file, StreamDefs(
features = StreamDef(field='image', transforms=transforms)))
# read rois and labels
roi_source = CTFDeserializer(roi_file, StreamDefs(
rois = StreamDef(field=roi_stream_name, shape=rois_dim, is_sparse=False)))
label_source = CTFDeserializer(label_file, StreamDefs(
roiLabels = StreamDef(field=label_stream_name, shape=label_dim, is_sparse=False)))
# define a composite reader
return MinibatchSource([image_source, roi_source, label_source], max_samples=sys.maxsize, randomize=data_set == "train")
def test_create_two_image_deserializers(tmpdir):
mbdata = r'''filename 0
filename2 0
'''
map_file = str(tmpdir / 'mbdata.txt')
with open(map_file, 'w') as f:
f.write(mbdata)
image_width = 100
image_height = 200
num_channels = 3
transforms = [xforms.crop(crop_type='randomside', side_ratio=0.5,
jitter_type='uniratio'),
xforms.scale(width=image_width, height=image_height,
channels=num_channels, interpolations='linear')]
image1 = ImageDeserializer(
map_file, StreamDefs(f1=StreamDef(field='image',
transforms=transforms)))
image2 = ImageDeserializer(
map_file, StreamDefs(f2=StreamDef(field='image',
transforms=transforms)))
mb_source = MinibatchSource([image1, image2])
assert isinstance(mb_source, MinibatchSource)
def create_reader(map_file,
mean_file,
train,
image_height=64,
image_width=64,
num_channels=3,
num_classes=32):
# transformation pipeline for the features has jitter/crop only when training
# https://docs.microsoft.com/en-us/python/api/cntk.io.transforms?view=cntk-py-2.2
trs = []
if train:
trs += [
transforms.crop(crop_type='randomside',
side_ratio=0,
jitter_type='none') # Horizontal flip enabled
]
trs += [
transforms.scale(width=image_width,
height=image_height,
channels=num_channels,
interpolations='linear'),
transforms.mean(mean_file)
]
# deserializer
image_source = ImageDeserializer(
map_file,
StreamDefs(
features=StreamDef(
field='image', transforms=trs
), # first column in map file is referred to as 'image'
labels=StreamDef(field='label',
shape=num_classes) # and second as 'label'
))
return MinibatchSource(image_source)
def test_create_two_image_deserializers(tmpdir):
mbdata = r'''filename 0
filename2 0
'''
map_file = str(tmpdir / 'mbdata.txt')
with open(map_file, 'w') as f:
f.write(mbdata)
image_width = 100
image_height = 200
num_channels = 3
transforms = [xforms.crop(crop_type='randomside', side_ratio=0.5,
jitter_type='uniratio'),
xforms.scale(width=image_width, height=image_height,
channels=num_channels, interpolations='linear')]
image1 = ImageDeserializer(
map_file, StreamDefs(f1=StreamDef(field='image',
transforms=transforms)))
image2 = ImageDeserializer(
map_file, StreamDefs(f2=StreamDef(field='image',
transforms=transforms)))
mb_source = MinibatchSource([image1, image2])
assert isinstance(mb_source, MinibatchSource)
def create_mb_source(map_file, image_width, image_height, num_channels, num_classes, randomize=True):
transforms = []
transforms += [xforms.crop(crop_type='randomside', side_ratio=0.8)]
transforms += [xforms.scale(width=image_width, height=image_height, channels=num_channels, interpolations='linear')]
return MinibatchSource(ImageDeserializer(map_file, StreamDefs(
features =StreamDef(field='image', transforms=transforms),
labels =StreamDef(field='label', shape=num_classes))),
randomize=randomize)
def create_mb(map_file, params, training_set):
transforms = []
image_dimensions = params['image_dimensions']
num_classes = params['num_classes']
if training_set:
# Scale to square-sized image. without this the cropping transform would chop the larger dimension of an
# image to make it squared, and then take 0.9 crops from within the squared image.
transforms += [
xforms.scale(width=2 * image_dimensions[0],
height=2 * image_dimensions[1],
channels=image_dimensions[2],
scale_mode='pad',
pad_value=114)
]
transforms += [
xforms.crop(crop_type='randomside',
side_ratio=0.9,
jitter_type='uniratio')
] # Randomly crop square area
#randomside enables Horizontal flipping
#new_dim = side_ratio * min(old_w,old_h) , 0.9 * 224 = 201.6
#transforms += [xforms.crop(crop_type='center')]
transforms += [
xforms.color(brightness_radius=0.2,
contrast_radius=0.2,
saturation_radius=0.2)
]
transforms += [xforms.crop(crop_type='center',
side_ratio=0.875)] # test has no jitter]
# Scale down and pad
transforms += [
xforms.scale(width=image_dimensions[0],
height=image_dimensions[1],
channels=image_dimensions[2],
scale_mode='pad',
pad_value=114)
]
return MinibatchSource(ImageDeserializer(
map_file,
StreamDefs(features=StreamDef(field='image', transforms=transforms),
labels=StreamDef(field='label', shape=num_classes))),
randomize=training_set,
multithreaded_deserializer=True)
def test_image():
map_file = "input.txt"
mean_file = "mean.txt"
feature_name = "f"
image_width = 100
image_height = 200
num_channels = 3
label_name = "l"
num_classes = 7
transforms = [
xforms.crop(crop_type='randomside', side_ratio=0.5,
jitter_type='uniratio'),
xforms.scale(width=image_width, height=image_height,
channels=num_channels, interpolations='linear'),
xforms.mean(mean_file)]
defs = StreamDefs(f=StreamDef(field='image', transforms=transforms),
l=StreamDef(field='label', shape=num_classes))
image = ImageDeserializer(map_file, defs)
config = to_dictionary(MinibatchSourceConfig([image], randomize=False))
assert len(config['deserializers']) == 1
d = config['deserializers'][0]
assert d['type'] == 'ImageDeserializer'
assert d['file'] == map_file
assert set(d['input'].keys()) == {label_name, feature_name}
l = d['input'][label_name]
assert l['labelDim'] == num_classes
f = d['input'][feature_name]
assert set(f.keys()) == {'transforms'}
t0, t1, t2, _ = f['transforms']
assert t0['type'] == 'Crop'
assert t1['type'] == 'Scale'
assert t2['type'] == 'Mean'
assert t0['cropType'] == 'randomside'
assert t0['sideRatio'] == 0.5
assert t0['aspectRatio'] == 1.0
assert t0['jitterType'] == 'uniratio'
assert t1['width'] == image_width
assert t1['height'] == image_height
assert t1['channels'] == num_channels
assert t1['interpolations'] == 'linear'
assert t2['meanFile'] == mean_file
config = to_dictionary(MinibatchSourceConfig([image, image]))
assert len(config['deserializers']) == 2
config = to_dictionary(MinibatchSourceConfig([image, image, image]))
assert len(config['deserializers']) == 3
# TODO depends on ImageReader.dll
'''
def create_mb_source(map_file, image_width, image_height, num_channels, num_classes, boTrain):
transforms = []
if boTrain:
# Scale to square-sized image. without this the cropping transform would chop the larger dimension of an
# image to make it squared, and then take 0.9 crops from within the squared image.
transforms += [xforms.scale(width=2*image_width, height=2*image_height, channels=num_channels,
interpolations='linear', scale_mode='pad', pad_value=114)]
transforms += [xforms.crop(crop_type='randomside', side_ratio=0.9, jitter_type='uniratio')] # Randomly crop square area
transforms += [xforms.scale(width=image_width, height=image_height, channels=num_channels, # Scale down and pad
interpolations='linear', scale_mode='pad', pad_value=114)]
if boTrain:
transforms += [xforms.color(brightness_radius=0.2, contrast_radius=0.2, saturation_radius=0.2)]
return MinibatchSource(ImageDeserializer(map_file, StreamDefs(
features = StreamDef(field='image', transforms=transforms),
labels = StreamDef(field='label', shape=num_classes))),
randomize = boTrain,
multithreaded_deserializer=True)
def create_feature_deserializer(path): transforms = [xforms.scale(width = ImageW, height = ImageH, channels = ImageC, interpolations = "linear")] deserializer = ImageDeserializer( path, StreamDefs( features = StreamDef(field = 'image', transforms = transforms), ignored = StreamDef(field = 'label', shape = 1) ) ) deserializer['grayscale'] = Grayscale return deserializer
def create_minibatch_source(map_file, is_training, num_outputs):
'''Create a minibatch source'''
transforms = []
# train uses data augmentation (translation only)
if is_training:
transforms += [xforms.crop(crop_type='randomside', side_ratio=0.8) ]
transforms += [xforms.scale(width=IMAGE_WIDTH, height=IMAGE_HEIGHT, channels=3, interpolations='linear')]
data_source = C.io.ImageDeserializer(MAP_FILE_PATH, C.io.StreamDefs(
image = C.io.StreamDef(field='image', transforms=transforms),
label = C.io.StreamDef(field='label', shape=num_outputs)
))
return C.io.MinibatchSource([data_source], randomize=is_training)
def create_label_deserializer_list(path):
transforms = [xforms.scale(width = ImageW, height = ImageH, channels = 1, interpolations = "linear")]
list = []
for x in range(1, numLabels + 1):
deserializer = ImageDeserializer(
path % x,
eval("StreamDefs(label%d = StreamDef(field = 'image', transforms = transforms), ignored%d = StreamDef(field = 'label', shape = 1))" % (x, x))
)
deserializer['grayscale'] = True
list.append(deserializer)
return list
def create_mb_source(image_height, image_width, num_channels, map_file):
transforms = [
xforms.scale(width=image_width,
height=image_height,
channels=num_channels,
interpolations='linear')
]
return MinibatchSource(
ImageDeserializer(
map_file,
StreamDefs(
features=StreamDef(field='image', transforms=transforms))
), # first column in map file is referred to as 'image'
randomize=False) # second column is labels and is ignored
def create_mb_source(map_file, image_dims, num_classes, randomize=True):
transforms = [
xforms.scale(width=image_dims[2],
height=image_dims[1],
channels=image_dims[0],
interpolations='linear')
]
return C.io.MinibatchSource(C.io.ImageDeserializer(
map_file,
C.io.StreamDefs(features=C.io.StreamDef(field='image',
transforms=transforms),
labels=C.io.StreamDef(field='label',
shape=num_classes))),
randomize=randomize)
def test_base64_is_equal_image(tmpdir):
import io, base64
from PIL import Image
np.random.seed(1)
file_mapping_path = str(tmpdir / 'file_mapping.txt')
base64_mapping_path = str(tmpdir / 'base64_mapping.txt')
with open(file_mapping_path, 'w') as file_mapping:
with open(base64_mapping_path, 'w') as base64_mapping:
for i in range(10):
data = np.random.randint(0, 2**8, (5, 7, 3))
image = Image.fromarray(data.astype('uint8'), "RGB")
buf = io.BytesIO()
image.save(buf, format='PNG')
assert image.width == 7 and image.height == 5
label = str(i)
# save to base 64 mapping file
encoded = base64.b64encode(buf.getvalue()).decode('ascii')
base64_mapping.write('%s\t%s\n' % (label, encoded))
# save to mapping + png file
file_name = label + '.png'
with open(str(tmpdir / file_name), 'wb') as f:
f.write(buf.getvalue())
file_mapping.write('.../%s\t%s\n' % (file_name, label))
transforms = [xforms.scale(width=7, height=5, channels=3)]
b64_deserializer = Base64ImageDeserializer(
base64_mapping_path,
StreamDefs(images1=StreamDef(field='image', transforms=transforms),
labels1=StreamDef(field='label', shape=10)))
file_image_deserializer = ImageDeserializer(
file_mapping_path,
StreamDefs(images2=StreamDef(field='image', transforms=transforms),
labels2=StreamDef(field='label', shape=10)))
mb_source = MinibatchSource([b64_deserializer, file_image_deserializer])
for j in range(20):
mb = mb_source.next_minibatch(1)
images1_stream = mb_source.streams['images1']
images1 = mb[images1_stream].asarray()
images2_stream = mb_source.streams['images2']
images2 = mb[images2_stream].asarray()
assert (images1 == images2).all()
def test_crop_dimensionality(tmpdir):
import io; from PIL import Image
np.random.seed(1)
file_mapping_path = str(tmpdir / 'file_mapping.txt')
with open(file_mapping_path, 'w') as file_mapping:
for i in range(5):
data = np.random.randint(0, 2**8, (20, 40, 3))
image = Image.fromarray(data.astype('uint8'), "RGB")
buf = io.BytesIO()
image.save(buf, format='PNG')
assert image.width == 40 and image.height == 20
label = str(i)
# save to mapping + png file
file_name = label + '.png'
with open(str(tmpdir/file_name), 'wb') as f:
f.write(buf.getvalue())
file_mapping.write('.../%s\t%s\n' % (file_name, label))
transforms1 = [
xforms.scale(width=40, height=20, channels=3),
xforms.crop(crop_type='randomside',
crop_size=(20, 10), side_ratio=(0.2, 0.5),
jitter_type='uniratio')]
transforms2 = [
xforms.crop(crop_type='randomside',
crop_size=(20, 10), side_ratio=(0.2, 0.5),
jitter_type='uniratio')]
d1 = ImageDeserializer(file_mapping_path,
StreamDefs(
images1=StreamDef(field='image', transforms=transforms1),
labels1=StreamDef(field='label', shape=10)))
d2 = ImageDeserializer(file_mapping_path,
StreamDefs(
images2=StreamDef(field='image', transforms=transforms2),
labels2=StreamDef(field='label', shape=10)))
mbs = MinibatchSource([d1, d2])
for j in range(5):
mb = mbs.next_minibatch(1)
images1 = mb[mbs.streams.images1].asarray()
images2 = mb[mbs.streams.images2].asarray()
assert images1.shape == (1, 1, 3, 10, 20)
assert (images1 == images2).all()
def test_crop_dimensionality(tmpdir):
import io; from PIL import Image
np.random.seed(1)
file_mapping_path = str(tmpdir / 'file_mapping.txt')
with open(file_mapping_path, 'w') as file_mapping:
for i in range(5):
data = np.random.randint(0, 2**8, (20, 40, 3))
image = Image.fromarray(data.astype('uint8'), "RGB")
buf = io.BytesIO()
image.save(buf, format='PNG')
assert image.width == 40 and image.height == 20
label = str(i)
# save to mapping + png file
file_name = label + '.png'
with open(str(tmpdir/file_name), 'wb') as f:
f.write(buf.getvalue())
file_mapping.write('.../%s\t%s\n' % (file_name, label))
transforms1 = [
xforms.scale(width=40, height=20, channels=3),
xforms.crop(crop_type='randomside',
crop_size=(20, 10), side_ratio=(0.2, 0.5),
jitter_type='uniratio')]
transforms2 = [
xforms.crop(crop_type='randomside',
crop_size=(20, 10), side_ratio=(0.2, 0.5),
jitter_type='uniratio')]
d1 = ImageDeserializer(file_mapping_path,
StreamDefs(
images1=StreamDef(field='image', transforms=transforms1),
labels1=StreamDef(field='label', shape=10)))
d2 = ImageDeserializer(file_mapping_path,
StreamDefs(
images2=StreamDef(field='image', transforms=transforms2),
labels2=StreamDef(field='label', shape=10)))
mbs = MinibatchSource([d1, d2])
for j in range(5):
mb = mbs.next_minibatch(1)
images1 = mb[mbs.streams.images1].asarray()
images2 = mb[mbs.streams.images2].asarray()
assert images1.shape == (1, 1, 3, 10, 20)
assert (images1 == images2).all()
def test_base64_is_equal_image(tmpdir):
import io, base64; from PIL import Image
np.random.seed(1)
file_mapping_path = str(tmpdir / 'file_mapping.txt')
base64_mapping_path = str(tmpdir / 'base64_mapping.txt')
with open(file_mapping_path, 'w') as file_mapping:
with open(base64_mapping_path, 'w') as base64_mapping:
for i in range(10):
data = np.random.randint(0, 2**8, (5,7,3))
image = Image.fromarray(data.astype('uint8'), "RGB")
buf = io.BytesIO()
image.save(buf, format='PNG')
assert image.width == 7 and image.height == 5
label = str(i)
# save to base 64 mapping file
encoded = base64.b64encode(buf.getvalue()).decode('ascii')
base64_mapping.write('%s\t%s\n' % (label, encoded))
# save to mapping + png file
file_name = label + '.png'
with open(str(tmpdir/file_name), 'wb') as f:
f.write(buf.getvalue())
file_mapping.write('.../%s\t%s\n' % (file_name, label))
transforms = [xforms.scale(width=7, height=5, channels=3)]
b64_deserializer = Base64ImageDeserializer(base64_mapping_path,
StreamDefs(
images1=StreamDef(field='image', transforms=transforms),
labels1=StreamDef(field='label', shape=10)))
file_image_deserializer = ImageDeserializer(file_mapping_path,
StreamDefs(
images2=StreamDef(field='image', transforms=transforms),
labels2=StreamDef(field='label', shape=10)))
mb_source = MinibatchSource([b64_deserializer, file_image_deserializer])
for j in range(20):
mb = mb_source.next_minibatch(1)
images1_stream = mb_source.streams['images1']
images1 = mb[images1_stream].asarray()
images2_stream = mb_source.streams['images2']
images2 = mb[images2_stream].asarray()
assert(images1 == images2).all()
def create_reader(map_file, mean_file, is_training):
if not os.path.exists(map_file) or not os.path.exists(mean_file):
raise RuntimeError("File '%s' or '%s' does not exist. Please run install_cifar10.py from DataSets/CIFAR-10 to fetch them" %
(map_file, mean_file))
# transformation pipeline for the features has jitter/crop only when training
transforms = []
if is_training:
transforms += [
xforms.crop(crop_type='randomside', side_ratio=0.8, jitter_type='uniratio') # train uses jitter
]
transforms += [
xforms.scale(width=image_width, height=image_height, channels=num_channels, interpolations='linear'),
xforms.mean(mean_file)
]
# deserializer
return MinibatchSource(ImageDeserializer(map_file, StreamDefs(
features=StreamDef(field='image', transforms=transforms), # first column in map file is referred to as 'image'
labels=StreamDef(field='label', shape=num_classes))), # and second as 'label'
randomize=is_training, max_sweeps = INFINITELY_REPEAT if is_training else 1)
def create_mb_source(image_height, image_width, num_channels, map_file):
transforms = [xforms.scale(width=image_width, height=image_height, channels=num_channels, interpolations='linear')]
return MinibatchSource(ImageDeserializer(map_file,
StreamDefs(features=StreamDef(field='image', transforms=transforms))), # first column in map file is referred to as 'image'
randomize=False) # second column is labels and is ignored
def create_reader(map_file1, map_file2):
transforms = [xforms.crop(crop_type='randomside', side_ratio=0.8, jitter_type='uniratio'), xforms.scale(width=224, height=224, channels=3, interpolations='linear')]
source1 = C.io.ImageDeserializer(map_file1, C.io.StreamDefs(
source_image = C.io.StreamDef(field='image', transforms=transforms)))
source2 = C.io.ImageDeserializer(map_file2, C.io.StreamDefs(
target_image = C.io.StreamDef(field='image', transforms=transforms)))
return C.io.MinibatchSource([source1, source2], max_samples=sys.maxsize, randomize=True, multithreaded_deserializer=False)
def create_mb_source(map_file, image_width, image_height, num_channels, num_classes, randomize=True):
transforms = [xforms.scale(width=image_width, height=image_height, channels=num_channels, interpolations='linear')]
return MinibatchSource(ImageDeserializer(map_file, StreamDefs(
features =StreamDef(field='image', transforms=transforms),
labels =StreamDef(field='label', shape=num_classes))),
randomize=randomize)
def test_image():
map_file = "input.txt"
mean_file = "mean.txt"
epoch_size = 150
feature_name = "f"
image_width = 100
image_height = 200
num_channels = 3
label_name = "l"
num_classes = 7
transforms = [xforms.crop(crop_type='randomside', side_ratio=0.5, jitter_type='uniratio'),
xforms.scale(width=image_width, height=image_height, channels=num_channels, interpolations='linear'),
xforms.mean(mean_file)]
image = ImageDeserializer(map_file, StreamDefs(f = StreamDef(field='image', transforms=transforms), l = StreamDef(field='label', shape=num_classes)))
rc = _ReaderConfig(image, randomize=False, epoch_size=epoch_size)
assert rc['epochSize'].value == epoch_size
assert rc['randomize'] == False
assert rc['sampleBasedRandomizationWindow'] == False
assert len(rc['deserializers']) == 1
d = rc['deserializers'][0]
assert d['type'] == 'ImageDeserializer'
assert d['file'] == map_file
assert set(d['input'].keys()) == {label_name, feature_name}
l = d['input'][label_name]
assert l['labelDim'] == num_classes
f = d['input'][feature_name]
assert set(f.keys()) == { 'transforms' }
t0, t1, t2 = f['transforms']
assert t0['type'] == 'Crop'
assert t1['type'] == 'Scale'
assert t2['type'] == 'Mean'
assert t0['cropType'] == 'randomside'
assert t0['sideRatio'] == 0.5
assert t0['aspectRatio'] == 1.0
assert t0['jitterType'] == 'uniratio'
assert t1['width'] == image_width
assert t1['height'] == image_height
assert t1['channels'] == num_channels
assert t1['interpolations'] == 'linear'
assert t2['meanFile'] == mean_file
rc = _ReaderConfig(image, randomize=False, randomization_window = 100,
sample_based_randomization_window = True, epoch_size=epoch_size)
assert rc['epochSize'].value == epoch_size
assert rc['randomize'] == False
assert rc['sampleBasedRandomizationWindow'] == True
assert len(rc['deserializers']) == 1
d = rc['deserializers'][0]
assert d['type'] == 'ImageDeserializer'
assert d['file'] == map_file
assert set(d['input'].keys()) == {label_name, feature_name}
l = d['input'][label_name]
assert l['labelDim'] == num_classes
rc = _ReaderConfig(image, randomize=True, randomization_window = 100,
sample_based_randomization_window = True, epoch_size=epoch_size)
assert rc['epochSize'].value == epoch_size
assert rc['randomize'] == True
assert rc['sampleBasedRandomizationWindow'] == True
assert len(rc['deserializers']) == 1
d = rc['deserializers'][0]
assert d['type'] == 'ImageDeserializer'
assert d['file'] == map_file
assert set(d['input'].keys()) == {label_name, feature_name}
l = d['input'][label_name]
assert l['labelDim'] == num_classes
# TODO depends on ImageReader.dll
'''
def test_base64_image_deserializer(tmpdir):
import io, base64, uuid; from PIL import Image
images, b64_images = [], []
np.random.seed(1)
for i in range(10):
data = np.random.randint(0, 2**8, (5,7,3))
image = Image.fromarray(data.astype('uint8'), "RGB")
buf = io.BytesIO()
image.save(buf, format='PNG')
assert image.width == 7 and image.height == 5
b64_images.append(base64.b64encode(buf.getvalue()))
images.append(np.array(image))
image_data = str(tmpdir / 'mbdata1.txt')
seq_ids = []
uid = uuid.uuid1().int >> 64
with open(image_data, 'wb') as f:
for i,data in enumerate(b64_images):
seq_id = uid ^ i
seq_id = str(seq_id).encode('ascii')
seq_ids.append(seq_id)
line = seq_id + b'\t'
label = str(i).encode('ascii')
line += label + b'\t' + data + b'\n'
f.write(line)
ctf_data = str(tmpdir / 'mbdata2.txt')
with open(ctf_data, 'wb') as f:
for i, sid in enumerate(seq_ids):
line = sid + b'\t' + b'|index '+str(i).encode('ascii') + b'\n'
f.write(line)
transforms = [xforms.scale(width=7, height=5, channels=3)]
b64_deserializer = Base64ImageDeserializer(image_data,
StreamDefs(
images=StreamDef(field='image', transforms=transforms),
labels=StreamDef(field='label', shape=10)))
ctf_deserializer = CTFDeserializer(ctf_data,
StreamDefs(index=StreamDef(field='index', shape=1)))
mb_source = MinibatchSource([ctf_deserializer, b64_deserializer])
assert isinstance(mb_source, MinibatchSource)
for j in range(100):
mb = mb_source.next_minibatch(10)
index_stream = mb_source.streams['index']
index = mb[index_stream].asarray().flatten()
image_stream = mb_source.streams['images']
results = mb[image_stream].asarray()
for i in range(10):
# original images are RBG, openCV produces BGR images,
# reverse the last dimension of the original images
bgrImage = images[int(index[i])][:,:,::-1]
# transposing to get CHW representation
bgrImage = np.transpose(bgrImage, (2, 0, 1))
assert (bgrImage == results[i][0]).all()
def test_image(tmpdir):
map_file = "input.txt"
mean_file = "mean.txt"
feature_name = "f"
image_width = 100
image_height = 200
num_channels = 3
label_name = "l"
num_classes = 7
transforms = [
xforms.crop(crop_type='randomside', side_ratio=0.5,
jitter_type='uniratio'),
xforms.scale(width=image_width, height=image_height,
channels=num_channels, interpolations='linear'),
xforms.mean(mean_file)]
defs = StreamDefs(f=StreamDef(field='image', transforms=transforms),
l=StreamDef(field='label', shape=num_classes))
image = ImageDeserializer(map_file, defs)
config = to_dictionary(MinibatchSourceConfig([image], randomize=False))
# Multithreading should be on by default for the ImageDeserializer.
assert config['multiThreadedDeserialization'] is True
assert len(config['deserializers']) == 1
d = config['deserializers'][0]
assert d['type'] == 'ImageDeserializer'
assert d['file'] == map_file
assert set(d['input'].keys()) == {label_name, feature_name}
l = d['input'][label_name]
assert l['labelDim'] == num_classes
f = d['input'][feature_name]
assert set(f.keys()) == {'transforms'}
t0, t1, t2, _ = f['transforms']
assert t0['type'] == 'Crop'
assert t1['type'] == 'Scale'
assert t2['type'] == 'Mean'
assert t0['cropType'] == 'randomside'
assert t0['cropSize'] == '0:0'
assert t0['sideRatio'] == '0.5:0.5'
assert t0['aspectRatio'] == '1:1'
assert t0['areaRatio'] == '0:0'
assert t0['jitterType'] == 'uniratio'
assert t1['width'] == image_width
assert t1['height'] == image_height
assert t1['channels'] == num_channels
assert t1['interpolations'] == 'linear'
assert t2['meanFile'] == mean_file
config = to_dictionary(MinibatchSourceConfig([image, image]))
assert len(config['deserializers']) == 2
ctf = create_ctf_deserializer(tmpdir)
config = to_dictionary(MinibatchSourceConfig([image, ctf, image]))
# Multithreading should still be enabled.
assert config['multiThreadedDeserialization'] is True
assert len(config['deserializers']) == 3
# TODO depends on ImageReader.dll
'''
