def __init__(self, directory, image_data_generator,
resize=None, target_size=None, color_mode='rgb',
dim_ordering='default',
classes=None, class_mode='categorical',
batch_size=32, shuffle=True, seed=None, gt_directory=None,
save_to_dir=None, save_prefix='', save_format='jpeg', model_name=None):
# Check dim order
if dim_ordering == 'default':
dim_ordering = K.image_dim_ordering()
self.dim_ordering = dim_ordering
self.directory = directory
self.gt_directory = gt_directory
self.image_data_generator = image_data_generator
self.resize = resize
self.save_to_dir = save_to_dir
self.save_prefix = save_prefix
self.save_format = save_format
self.model_name = model_name
# Check target size
if target_size is None and batch_size > 1:
raise ValueError('Target_size None works only with batch_size=1')
self.target_size = (None, None) if target_size is None else tuple(target_size)
# Check color mode
if color_mode not in {'rgb', 'grayscale', 'bgr'}:
raise ValueError('Invalid color mode:', color_mode,
'; expected "rgb" or "grayscale".')
self.color_mode = color_mode
if self.color_mode == 'rgb' or self.color_mode == 'bgr':
self.grayscale = False
if self.dim_ordering == 'tf':
self.image_shape = self.target_size + (3,)
self.gt_image_shape = self.target_size + (1,)
else:
self.image_shape = (3,) + self.target_size
self.gt_image_shape = (1,) + self.target_size
else:
self.grayscale = True
if self.dim_ordering == 'tf':
self.image_shape = self.target_size + (1,)
self.gt_image_shape = self.image_shape
else:
self.image_shape = (1,) + self.target_size
self.gt_image_shape = self.image_shape
# Check class mode
if class_mode not in {'categorical', 'binary', 'sparse',
'segmentation', 'detection', None}:
raise ValueError('Invalid class_mode:', class_mode,
'; expected one of "categorical", '
'"binary", "sparse", "segmentation", "detection" or None.')
self.class_mode = class_mode
self.has_gt_image = True if self.class_mode == 'segmentation' else False
# Check class names
if not classes:
if self.class_mode == 'segmentation' or self.class_mode == 'detection':
raise ValueError('You should input the class names')
else:
classes = list_subdirs(directory)
else:
classes = classes.values()
self.nb_class = len(classes)
self.class_indices = dict(zip(classes, range(len(classes))))
self.nb_sample = 0
self.filenames = []
self.classes = []
# Get filenames
if self.class_mode == 'detection':
for fname in os.listdir(directory):
if has_valid_extension(fname):
self.filenames.append(fname)
# Look for the GT filename
gt_fname = os.path.join(directory,fname.replace('jpg','txt'))
if not os.path.isfile(gt_fname):
raise ValueError('GT file not found: ' + gt_fname)
self.filenames = np.sort(self.filenames)
if self.model_name == 'ssd':
priors = pickle.load(open('weights/prior_boxes_ssd300.pkl', 'rb'))
self.bbox_util = BBoxUtility(self.nb_class+1, priors)
elif not self.class_mode == 'segmentation':
for subdir in classes:
subpath = os.path.join(directory, subdir)
for fname in os.listdir(subpath):
if has_valid_extension(fname):
self.classes.append(self.class_indices[subdir])
self.filenames.append(os.path.join(subdir, fname))
self.classes = np.array(self.classes)
else:
for fname in os.listdir(gt_directory):
if has_valid_extension(fname):
self.filenames.append(fname)
# Look for the GT filename
gt_fname = os.path.join(gt_directory,
os.path.split(fname)[1])
if not os.path.isfile(gt_fname):
raise ValueError('GT file not found: ' + gt_fname)
self.filenames = np.sort(self.filenames)
self.nb_sample = len(self.filenames)
print(' Found %d images belonging to %d classes' % (self.nb_sample,
self.nb_class))
super(DirectoryIterator, self).__init__(self.nb_sample, batch_size,
shuffle, seed)
class DirectoryIterator(Iterator):
def __init__(self, directory, image_data_generator,
resize=None, target_size=None, color_mode='rgb',
dim_ordering='default',
classes=None, class_mode='categorical',
batch_size=32, shuffle=True, seed=None, gt_directory=None,
save_to_dir=None, save_prefix='', save_format='jpeg', model_name=None):
# Check dim order
if dim_ordering == 'default':
dim_ordering = K.image_dim_ordering()
self.dim_ordering = dim_ordering
self.directory = directory
self.gt_directory = gt_directory
self.image_data_generator = image_data_generator
self.resize = resize
self.save_to_dir = save_to_dir
self.save_prefix = save_prefix
self.save_format = save_format
self.model_name = model_name
# Check target size
if target_size is None and batch_size > 1:
raise ValueError('Target_size None works only with batch_size=1')
self.target_size = (None, None) if target_size is None else tuple(target_size)
# Check color mode
if color_mode not in {'rgb', 'grayscale', 'bgr'}:
raise ValueError('Invalid color mode:', color_mode,
'; expected "rgb" or "grayscale".')
self.color_mode = color_mode
if self.color_mode == 'rgb' or self.color_mode == 'bgr':
self.grayscale = False
if self.dim_ordering == 'tf':
self.image_shape = self.target_size + (3,)
self.gt_image_shape = self.target_size + (1,)
else:
self.image_shape = (3,) + self.target_size
self.gt_image_shape = (1,) + self.target_size
else:
self.grayscale = True
if self.dim_ordering == 'tf':
self.image_shape = self.target_size + (1,)
self.gt_image_shape = self.image_shape
else:
self.image_shape = (1,) + self.target_size
self.gt_image_shape = self.image_shape
# Check class mode
if class_mode not in {'categorical', 'binary', 'sparse',
'segmentation', 'detection', None}:
raise ValueError('Invalid class_mode:', class_mode,
'; expected one of "categorical", '
'"binary", "sparse", "segmentation", "detection" or None.')
self.class_mode = class_mode
self.has_gt_image = True if self.class_mode == 'segmentation' else False
# Check class names
if not classes:
if self.class_mode == 'segmentation' or self.class_mode == 'detection':
raise ValueError('You should input the class names')
else:
classes = list_subdirs(directory)
else:
classes = classes.values()
self.nb_class = len(classes)
self.class_indices = dict(zip(classes, range(len(classes))))
self.nb_sample = 0
self.filenames = []
self.classes = []
# Get filenames
if self.class_mode == 'detection':
for fname in os.listdir(directory):
if has_valid_extension(fname):
self.filenames.append(fname)
# Look for the GT filename
gt_fname = os.path.join(directory,fname.replace('jpg','txt'))
if not os.path.isfile(gt_fname):
raise ValueError('GT file not found: ' + gt_fname)
self.filenames = np.sort(self.filenames)
if self.model_name == 'ssd':
priors = pickle.load(open('weights/prior_boxes_ssd300.pkl', 'rb'))
self.bbox_util = BBoxUtility(self.nb_class+1, priors)
elif not self.class_mode == 'segmentation':
for subdir in classes:
subpath = os.path.join(directory, subdir)
for fname in os.listdir(subpath):
if has_valid_extension(fname):
self.classes.append(self.class_indices[subdir])
self.filenames.append(os.path.join(subdir, fname))
self.classes = np.array(self.classes)
else:
for fname in os.listdir(gt_directory):
if has_valid_extension(fname):
self.filenames.append(fname)
# Look for the GT filename
gt_fname = os.path.join(gt_directory,
os.path.split(fname)[1])
if not os.path.isfile(gt_fname):
raise ValueError('GT file not found: ' + gt_fname)
self.filenames = np.sort(self.filenames)
self.nb_sample = len(self.filenames)
print(' Found %d images belonging to %d classes' % (self.nb_sample,
self.nb_class))
super(DirectoryIterator, self).__init__(self.nb_sample, batch_size,
shuffle, seed)
def next(self):
# Lock the generation of index only. The rest is not under thread
# lock so it can be done in parallel
with self.lock:
index_array, current_index, current_batch_size = next(self.index_generator)
# Create the batch_x and batch_y
if current_batch_size > 1:
batch_x = np.zeros((current_batch_size,) + self.image_shape)
if self.has_gt_image:
batch_y = np.zeros((current_batch_size,) + self.gt_image_shape)
if self.class_mode == 'detection':
batch_y = []
# Build batch of image data
for i, j in enumerate(index_array):
# Load image
fname = self.filenames[j]
# print(fname)
img = load_img(os.path.join(self.directory, fname),
grayscale=self.grayscale,
resize=self.resize, order=1)
x = img_to_array(img, dim_ordering=self.dim_ordering)
# Load GT image if segmentation
if self.has_gt_image:
# Load GT image
gt_img = load_img(os.path.join(self.gt_directory, fname),
grayscale=True,
resize=self.resize, order=0)
y = img_to_array(gt_img, dim_ordering=self.dim_ordering)
else:
y = None
# Load GT image if detection
if self.class_mode == 'detection':
label_path = os.path.join(self.directory, fname).replace('jpg','txt')
gt = np.loadtxt(label_path)
if len(gt.shape) == 1:
gt = gt[np.newaxis,]
y = gt.copy()
y = y[((y[:,1] > 0.) & (y[:,1] < 1.))]
y = y[((y[:,2] > 0.) & (y[:,2] < 1.))]
y = y[((y[:,3] > 0.) & (y[:,3] < 1.))]
y = y[((y[:,4] > 0.) & (y[:,4] < 1.))]
if (y.shape != gt.shape) or (y.shape[0] == 0):
warnings.warn('DirectoryIterator: found an invalid annotation '
'on GT file '+label_path)
# shuffle gt boxes order
np.random.shuffle(y)
# Standarize image
x = self.image_data_generator.standardize(x, y)
# Data augmentation
x, y = self.image_data_generator.random_transform(x, y)
# Add images to batches
if current_batch_size > 1:
batch_x[i] = x
if self.has_gt_image:
batch_y[i] = y
elif self.class_mode == 'detection':
batch_y.append(y)
else:
batch_x = np.expand_dims(x, axis=0)
if self.has_gt_image:
batch_y = np.expand_dims(y, axis=0)
elif self.class_mode == 'detection':
batch_y = [y]
# optionally save augmented images to disk for debugging purposes
if self.save_to_dir:
for i in range(current_batch_size):
fname = '{prefix}_{index}_{hash}.{format}'.format(prefix=self.save_prefix,
index=current_index + i,
hash=np.random.randint(1e4),
format=self.save_format)
if self.class_mode == 'segmentation':
nclasses = self.classes # TODO: Change
color_map = sns.hls_palette(nclasses+1)
void_label = nclasses
save_img2(batch_x[i], batch_y[i],
os.path.join(self.save_to_dir, fname), color_map,
void_label)
else:
img = array_to_img(batch_x[i], self.dim_ordering,
scale=True)
img.save(os.path.join(self.save_to_dir, fname))
# Build batch of labels
if self.class_mode == 'sparse':
batch_y = self.classes[index_array]
elif self.class_mode == 'binary':
batch_y = self.classes[index_array].astype('float32')
elif self.class_mode == 'categorical':
batch_y = np.zeros((len(batch_x), self.nb_class), dtype='float32')
for i, label in enumerate(self.classes[index_array]):
batch_y[i, label] = 1.
elif self.class_mode == 'detection':
if 'yolo' in self.model_name:
batch_y = yolo_build_gt_batch(batch_y, self.image_shape, self.nb_class)
elif self.model_name == 'ssd':
batch_y = self.bbox_util.ssd_build_gt_batch_v2(batch_y)
# targets = []
# for boxes in batch_y:
# boxes_corrected = np.zeros((boxes.shape[0], 4+self.nb_class))
# for b,box in enumerate(boxes):
# boxes_corrected[b,0] = box[1] - box[3]/2
# boxes_corrected[b,1] = box[2] - box[4]/2
# boxes_corrected[b,2] = box[1] + box[3]/2
# boxes_corrected[b,3] = box[2] + box[4]/2
# c = 4+int(box[0])
# boxes_corrected[b,c] = 1.
# boxes_corrected = self.bbox_util.assign_boxes(boxes_corrected)
# targets.append(boxes_corrected)
# batch_y = np.array(targets)
elif self.class_mode == None:
return batch_x
return batch_x, batch_y
def load(self):
config_path = self.config_path
exp_name = self.exp_name
dataset_path = self.dataset_path
shared_dataset_path = self.shared_dataset_path
experiments_path = self.experiments_path
shared_experiments_path = self.shared_experiments_path
# Load configuration file
print config_path
cf = imp.load_source('config', config_path)
# Save extra parameter
cf.config_path = config_path
cf.exp_name = exp_name
# Create output folders
cf.savepath = os.path.join(experiments_path, cf.dataset_name,
cf.exp_name)
cf.final_savepath = os.path.join(shared_experiments_path,
cf.dataset_name, cf.exp_name)
cf.log_file = os.path.join(cf.savepath, "logfile.log")
if not os.path.exists(cf.savepath):
os.makedirs(cf.savepath)
cf.usr_path = self.usr_path
# Copy config file
shutil.copyfile(config_path, os.path.join(cf.savepath, "config.py"))
# Load dataset configuration
cf.dataset = self.load_config_dataset(cf.dataset_name, dataset_path,
shared_dataset_path,
cf.problem_type,
'config_dataset')
if cf.dataset_name2:
cf.dataset2 = self.load_config_dataset(cf.dataset_name2,
dataset_path,
shared_dataset_path,
cf.problem_type,
'config_dataset2')
# If in Debug mode use few images
if cf.debug and cf.debug_images_train > 0:
cf.dataset.n_images_train = cf.debug_images_train
if cf.debug and cf.debug_images_valid > 0:
cf.dataset.n_images_valid = cf.debug_images_valid
if cf.debug and cf.debug_images_test > 0:
cf.dataset.n_images_test = cf.debug_images_test
if cf.debug and cf.debug_n_epochs > 0:
cf.n_epochs = cf.debug_n_epochs
# Define target sizes
if cf.crop_size_train is not None:
cf.target_size_train = cf.crop_size_train
elif cf.resize_train is not None:
cf.target_size_train = cf.resize_train
else:
cf.target_size_train = cf.dataset.img_shape
if cf.crop_size_valid is not None:
cf.target_size_valid = cf.crop_size_valid
elif cf.resize_valid is not None:
cf.target_size_valid = cf.resize_valid
else:
cf.target_size_valid = cf.dataset.img_shape
if cf.crop_size_test is not None:
cf.target_size_test = cf.crop_size_test
elif cf.resize_test is not None:
cf.target_size_test = cf.resize_test
else:
cf.target_size_test = cf.dataset.img_shape
# Get training weights file name
path, _ = os.path.split(cf.weights_file)
if path == '':
cf.weights_file = os.path.join(cf.savepath, cf.weights_file)
# Get testing weights file name
try:
path_test, _ = os.path.split(cf.weights_test_file)
if path_test == '':
cf.weights_test_file = os.path.join(cf.savepath,
cf.weights_test_file)
except:
cf.weights_test_file = os.path.join(cf.savepath, 'weights.hdf5')
# Plot metrics
if cf.dataset.class_mode == 'segmentation':
#cf.train_metrics = ['loss', 'acc', 'jaccard']
#cf.valid_metrics = ['val_loss', 'val_acc', 'val_jaccard']
#cf.best_metric = 'val_jaccard'
class_weights_path = os.path.join(self.usr_path, 'extra',
cf.dataset_name + '.pkl')
if not os.path.exists(class_weights_path):
cf.dataset.cb_weights2 = None
print('No weights are available for weighted cross-entropy')
else:
cf.dataset.cb_weights2 = pickle.load(
open(class_weights_path, 'rb'))
cf.train_metrics = ['loss', 'acc', 'jaccard_coef']
cf.valid_metrics = ['val_loss', 'val_acc', 'val_jaccard_coef']
cf.best_metric = 'val_jaccard_coef'
cf.best_type = 'max'
elif cf.dataset.class_mode == 'detection':
# TODO detection : different nets may have other metrics
cf.train_metrics = ['loss', 'avg_recall', 'avg_iou']
cf.valid_metrics = ['val_loss', 'val_avg_recall', 'val_avg_iou']
cf.best_metric = 'val_avg_recall'
cf.best_type = 'max'
if cf.model_name == 'ssd':
cf.dataset.priors = pickle.load(
open(
os.path.join(self.usr_path, 'extra',
'prior_boxes_ssd300.pkl'), 'rb'))
cf.bbox_util = BBoxUtility(cf.dataset.n_classes,
cf.dataset.priors)
else:
cf.train_metrics = ['loss', 'acc']
cf.valid_metrics = ['val_loss', 'val_acc']
cf.best_metric = 'val_acc'
cf.best_type = 'max'
self.configuration = cf
return cf
])
num_images_chunk = images.shape[0]
net_out = model.predict(images, batch_size=8, verbose=1)
# Store the predictions
for ind in range(num_images_chunk):
if model_name == 'yolo' or model_name == 'tiny-yolo':
boxes_pred = yolo_postprocess_net_out(net_out[ind], priors,
classes,
detection_threshold,
nms_threshold)
else:
priors = pickle.load(open('prior_boxes_ssd300.pkl', 'rb'))
real_num_classes = num_classes - 1 # Background is not included
bbox_util = BBoxUtility(real_num_classes,
priors=priors,
nms_thresh=nms_threshold)
boxes_pred = bbox_util.detection_out(
net_out[ind],
background_label_id=0,
confidence_threshold=detection_threshold)
current_img = images[ind]
if 'yolo' in model_name:
current_img = np.transpose(current_img, (1, 2, 0))
plt.imshow(current_img)
currentAxis = plt.gca()
# Compute number of predictions that match with GT with a minimum of 50% IoU
for b in boxes_pred:
pred_idx = np.argmax(b.probs)
print "ERR: path_to_images do not contain any jpg file"
quit()
inputs = []
img_paths = []
chunk_size = 128 # we are going to process all image files in chunks
ok = 0.
total_true = 0.
total_pred = 0.
total_p = 0.
total_r = 0.
iters = 0
total_fps = 0.
bbox_util = BBoxUtility(NUM_CLASSES-1, nms_thresh=nms_threshold) # -1 because the void class is added inside
for i,img_path in enumerate(imfiles):
img = image.load_img(img_path, target_size=(input_shape[0], input_shape[1]))
img = image.img_to_array(img)
img = img / 255.
inputs.append(img.copy())
img_paths.append(img_path)
if len(img_paths)%chunk_size == 0 or i+1 == len(imfiles):
inputs = np.array(inputs)
start_time = time.time()
net_out = model.predict(inputs, batch_size=16, verbose=1)
print ('{} images predicted in {:.5f} seconds. {:.5f} fps').format(len(inputs),time.time() - start_time,(len(inputs)/(time.time() - start_time)))
total_fps = total_fps + len(inputs)/(time.time() - start_time)
# predicted boxes
def next(self):
# Lock the generation of index only. The rest is not under thread
# lock so it can be done in parallel
with self.lock:
index_array, current_index, current_batch_size = next(self.index_generator)
# Create the batch_x and batch_y
if current_batch_size > 1:
batch_x = np.zeros((current_batch_size,) + self.image_shape)
if self.has_gt_image:
batch_y = np.zeros((current_batch_size,) + self.gt_image_shape)
if self.class_mode == 'detection':
batch_y = []
# Build batch of image data
for i, j in enumerate(index_array):
# Load image
fname = self.filenames[j]
# print(fname)
img = load_img(os.path.join(self.directory, fname),
grayscale=self.grayscale,
resize=self.resize, order=1)
x = img_to_array(img, dim_ordering=self.dim_ordering)
# Load GT image if segmentation
if self.has_gt_image:
# Load GT image
gt_img = load_img(os.path.join(self.gt_directory, fname),
grayscale=True,
resize=self.resize, order=0)
y = img_to_array(gt_img, dim_ordering=self.dim_ordering)
else:
y = None
# Load GT image if detection
if self.class_mode == 'detection':
label_path = os.path.join(self.directory, fname).replace('jpg','txt')
gt = np.loadtxt(label_path)
if len(gt.shape) == 1:
gt = gt[np.newaxis,]
y = gt.copy()
y = y[((y[:,1] > 0.) & (y[:,1] < 1.))]
y = y[((y[:,2] > 0.) & (y[:,2] < 1.))]
y = y[((y[:,3] > 0.) & (y[:,3] < 1.))]
y = y[((y[:,4] > 0.) & (y[:,4] < 1.))]
if (y.shape != gt.shape) or (y.shape[0] == 0):
warnings.warn('DirectoryIterator: found an invalid annotation '
'on GT file '+label_path)
# shuffle gt boxes order
np.random.shuffle(y)
# Standarize image
x = self.image_data_generator.standardize(x, y)
# Data augmentation
x, y = self.image_data_generator.random_transform(x, y)
# Add images to batches
if current_batch_size > 1:
batch_x[i] = x
if self.has_gt_image:
batch_y[i] = y
elif self.class_mode == 'detection':
batch_y.append(y)
else:
batch_x = np.expand_dims(x, axis=0)
if self.has_gt_image:
batch_y = np.expand_dims(y, axis=0)
elif self.class_mode == 'detection':
batch_y = [y]
# optionally save augmented images to disk for debugging purposes
if self.save_to_dir:
for i in range(current_batch_size):
fname = '{prefix}_{index}_{hash}.{format}'.format(prefix=self.save_prefix,
index=current_index + i,
hash=np.random.randint(1e4),
format=self.save_format)
if self.class_mode == 'segmentation':
nclasses = self.classes # TODO: Change
color_map = sns.hls_palette(nclasses+1)
void_label = nclasses
save_img2(batch_x[i], batch_y[i],
os.path.join(self.save_to_dir, fname), color_map,
void_label)
else:
img = array_to_img(batch_x[i], self.dim_ordering,
scale=True)
img.save(os.path.join(self.save_to_dir, fname))
# Build batch of labels
if self.class_mode == 'sparse':
batch_y = self.classes[index_array]
elif self.class_mode == 'binary':
batch_y = self.classes[index_array].astype('float32')
elif self.class_mode == 'categorical':
batch_y = np.zeros((len(batch_x), self.nb_class), dtype='float32')
for i, label in enumerate(self.classes[index_array]):
batch_y[i, label] = 1.
elif self.class_mode == 'detection':
if self.model_name == 'ssd':
bbox_util = BBoxUtility(self.nb_class)
batch_y = bbox_util.ssd_build_gt_batch(batch_y)
elif self.model_name == 'yolo' or self.model_name == 'tiny-yolo':
# TODO detection: check model, other networks may expect a different batch_y format and shape
# YOLOLoss expects a particular batch_y format and shape
batch_y = yolo_build_gt_batch(batch_y, self.image_shape, self.nb_class)
else:
print('Unknown detection model')
elif self.class_mode == None:
return batch_x
return batch_x, batch_y