def batch_predict_loop(model, inputs, batch_size, verbose=0):
"""Predict function for eager execution when input is arrays or tensors.
Arguments:
model: Instance of `Model`.
inputs: List of input arrays.
batch_size: Integer batch size.
verbose: Verbosity mode.
Returns:
Array of predictions (if the model has a single output)
or list of arrays of predictions (if the model has multiple outputs).
"""
outs = []
num_samples = training_utils.check_num_samples(inputs, batch_size)
if verbose == 1:
progbar = generic_utils.Progbar(target=num_samples)
batches = generic_utils.make_batches(num_samples, batch_size)
index_array = np.arange(num_samples)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
inputs_batch = slice_arrays(inputs, batch_ids)
inputs_batch = [
ops.convert_to_tensor(val, dtype=backend.floatx())
for val in inputs_batch
]
if len(inputs_batch) == 1:
if model._expects_training_arg:
batch_outs = model.call(inputs_batch[0], training=False)
else:
batch_outs = model.call(inputs_batch[0])
else:
if model._expects_training_arg:
batch_outs = model.call(inputs_batch, training=False)
else:
batch_outs = model.call(inputs_batch)
if not isinstance(batch_outs, list):
batch_outs = [batch_outs]
if batch_index == 0:
# Pre-allocate the results arrays.
for batch_out in batch_outs:
dims = batch_out.shape[1:].dims
dims_list = [d.value for d in dims]
shape = (num_samples, ) + tuple(dims_list)
outs.append(
np.zeros(shape, dtype=batch_out.dtype.as_numpy_dtype))
for i, batch_out in enumerate(batch_outs):
outs[i][batch_start:batch_end] = batch_out
if verbose == 1:
progbar.update(batch_end)
if len(outs) == 1:
return outs[0]
return outs
def train_model(result_dir="result", data=data_train):
train_datagen = Data(data_train, input_shape[:2], local_shape[:2])
for n in range(n_epoch):
progbar = generic_utils.Progbar(len(train_datagen))
for inputs, points, masks in train_datagen.flow(batch_size):
completion_image = completion_model.predict([inputs, masks])
valid = np.ones((batch_size, 1))
fake = np.zeros((batch_size, 1))
g_loss = 0.0
d_loss = 0.0
if n < tc:
g_loss = completion_model.train_on_batch([inputs, masks],
inputs)
else:
d_loss_real = d_model.train_on_batch([inputs, points], valid)
d_loss_fake = d_model.train_on_batch(
[completion_image, points], fake)
d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)
if n >= tc + td:
g_loss = all_model.train_on_batch([inputs, masks, points],
[inputs, valid])
g_loss = g_loss[0] + alpha * g_loss[1]
progbar.add(inputs.shape[0],
values=[("D loss", d_loss), ("G mse", g_loss)])
num_img = min(5, batch_size)
fig, axs = plt.subplots(num_img, 3)
for i in range(num_img):
axs[i, 0].imshow(inputs[i] * (1 - masks[i]))
axs[i, 0].axis('off')
axs[i, 0].set_title('Input')
axs[i, 1].imshow(completion_image[i])
axs[i, 1].axis('off')
axs[i, 1].set_title('Output')
axs[i, 2].imshow(inputs[i])
axs[i, 2].axis('off')
axs[i, 2].set_title('Ground Truth')
fig.savefig(os.path.join(result_dir, "result_%d.png" % n))
plt.close()
# Save the checkpoints every 10 epochs
if (n + 1) % 10 == 0:
checkpoint.save(file_prefix=checkpoint_prefix)
# save model
generator.save(os.path.join("model", "generator.h5"))
completion_model.save(os.path.join("model", "completion.h5"))
discriminator.save(os.path.join("model", "discriminator.h5"))
def fit_generator(self,
batch_gen,
noise_gen,
steps_per_epoch=1,
num_epochs=1,
training_ratio=1):
disc_out_shape = (batch_gen.batch_size, self.disc.output_shape[1])
real_target = np.ones(disc_out_shape, dtype=np.float32)
fake_target = -real_target
gp_target = np.zeros_like(real_target)
for epoch in range(num_epochs):
print("Epoch {}/{}".format(epoch + 1, num_epochs))
# Initialize progbar and batch counter
progbar = generic_utils.Progbar(steps_per_epoch *
batch_gen.batch_size)
for step in range(steps_per_epoch):
# Train discriminator
with Nontrainable(self.gen):
for repeat in range(training_ratio):
image_batch = next(batch_gen)
noise_batch = next(noise_gen)
disc_loss = self.disc_trainer.train_on_batch(
[image_batch] + noise_batch,
[real_target, fake_target, gp_target])
# Train generator
with Nontrainable(self.disc):
noise_batch = next(noise_gen)
gen_loss = self.gen_trainer.train_on_batch(
noise_batch, real_target)
losses = []
for (i, dl) in enumerate(disc_loss):
losses.append(("D{}".format(i), dl))
losses.append(("G0", gen_loss))
progbar.add(batch_gen.batch_size, values=losses)
def main_video_process(curr_config, class_to_color, model_rpn, model_classifier_only, video_file_name):
"""
Extracting the test video into image frame by calling convert_video_frame method.
Detection process is called using Faster R-CNN with VGG 16 network.
Display the detected results as a video output.
:param curr_config: get config class.
:param class_to_color: get colour for the bounding boxes as each class(label) represent different colour.
:param model_rpn: get optimal rpn model.
:param model_classifier_only: get optimal classifier model.
:param video_file_name: get test image frame.
:return test_data: get the dict of detected results.
:return new_images: get the list of detected result image.
"""
test_data = {}
new_images = []
print('Test video : {}'.format(video_file_name))
video_frames = convert_video_frame(video_file_name)
print('Start Image Testing')
progbar = generic_utils.Progbar(len(video_frames))
for idx, frame in enumerate(video_frames):
all_data, new_img = image_testing(frame, curr_config, model_rpn, model_classifier_only, class_to_color,
online=True)
progbar.update(idx + 1, [('Image Testing Continue.', idx)])
if all_data:
new_images.append(new_img)
for data in all_data:
class_name = data['class']
if class_name not in test_data:
test_data[class_name] = []
test_data[class_name].append(data)
print('End Image Testing')
return test_data, new_images
def iterator_predict_loop(model, inputs, steps, verbose=0):
"""Predict function for eager execution when input is dataset iterator.
Arguments:
model: Instance of `Model`.
inputs: Input dataset iterator.
steps: Total number of steps (batches of samples) before declaring
`_predict_loop` finished.
verbose: Verbosity mode.
Returns:
Array of predictions (if the model has a single output)
or list of arrays of predictions (if the model has multiple outputs).
Raises:
ValueError: In case of mismatch between given number of inputs and
expectations of the model.
"""
assert isinstance(inputs, iterator_ops.EagerIterator)
if not isinstance(inputs.output_shapes,
collections.Sequence) or len(inputs.output_shapes) > 3:
raise ValueError(
'Please provide data as a list or tuple of 1, 2, or 3 elements '
' - `(input)`, or `(input, target)`, or `(input, target,'
'sample_weights)`. Received %s. We do not use the `target` or'
'`sample_weights` value here.' % inputs.output_shapes)
outs = []
if verbose == 1:
progbar = generic_utils.Progbar(target=steps)
for step_index in range(steps):
# Get data from the iterator.
try:
next_element = inputs.get_next()
except errors.OutOfRangeError:
logging.warning(
'Your dataset iterator ran out of data; interrupting prediction. '
'Make sure that your dataset can generate at least `steps` batches '
'(in this case, %d batches). You may need to use the repeat() '
'function when building your dataset.', steps)
break
# expects a tuple, where first element of tuple represents inputs
x = next_element[0]
# Validate and standardize data.
x, _, _ = model._standardize_user_data(x)
x = training_utils.cast_if_floating_dtype(x)
if isinstance(x, list) and len(x) == 1:
x = x[0]
if model._expects_training_arg:
batch_outs = model.call(x, training=False)
else:
batch_outs = model.call(x)
if not isinstance(batch_outs, list):
batch_outs = [batch_outs]
# We collect the results from every step and then concatenate them once
# in the end. This is an expensive process. We are doing this because we
# do not know the number of samples beforehand.
if step_index == 0:
for _ in batch_outs:
outs.append([])
for i, batch_out in enumerate(batch_outs):
outs[i].append(backend.get_value(batch_out))
if verbose == 1:
progbar.update(step_index + 1)
for i, out in enumerate(outs):
outs[i] = np.concatenate(tuple(out), axis=0)
if len(outs) == 1:
return outs[0]
return outs
def iterator_test_loop(model, inputs, steps, verbose=0):
"""Test function for eager execution when input is given as dataset iterator.
Arguments:
model: Model instance that is being evaluated in Eager mode.
inputs: Input dataset iterator.
steps: Total number of steps (batches of samples) before declaring
predictions finished.
verbose: Verbosity mode.
Returns:
Scalar loss (if the model has a single output and no metrics)
or list of scalars (if the model has multiple outputs
and/or metrics). The attribute `model.metrics_names` will give you
the display labels for the scalar outputs.
Raises:
ValueError: In case of mismatch between given number of inputs and
expectations of the model.
"""
assert isinstance(inputs, iterator_ops.EagerIterator)
# make sure either x,y or x,y,sample_weights is provided
if (not isinstance(inputs.output_shapes, collections.Sequence)
or len(inputs.output_shapes) < 2 or len(inputs.output_shapes) > 3):
raise ValueError('Please provide either inputs and targets'
'or inputs, targets, and sample_weights')
outs = []
# Create metric wrapper for the losses.
output_loss_metrics = []
for i in range(len(model.outputs)):
loss_fn = model.loss_functions[i]
loss_name = loss_fn.name if isinstance(
loss_fn, losses_module.Loss) else loss_fn.__name__
mean_wrapped_loss = metrics_module.MeanMetricWrapper(loss_fn,
name=loss_name)
output_loss_metrics.append(mean_wrapped_loss)
num_samples = 0
if verbose == 1:
progbar = generic_utils.Progbar(target=steps)
for step_index in range(steps):
# Get data from the iterator.
try:
next_element = inputs.get_next()
except errors.OutOfRangeError:
logging.warning(
'Your dataset iterator ran out of data interrupting testing. '
'Make sure that your dataset can generate at least `steps` batches '
'(in this case, %d batches). You may need to use the repeat() '
'function when building your dataset.', steps)
break
if len(inputs.output_shapes) == 2:
x, y = next_element
sample_weights = None
else:
x, y, sample_weights = next_element
# Validate and standardize data.
x, y, sample_weights = model._standardize_user_data(
x, y, sample_weight=sample_weights)
x = training_utils.cast_if_floating_dtype(x)
y = training_utils.cast_if_floating_dtype(y)
if sample_weights:
sample_weights = [
training_utils.cast_if_floating_dtype(
ops.convert_to_tensor(val, dtype=backend.floatx()))
if val is not None else None for val in sample_weights
]
if step_index == 0:
# Get stateful metrics indices. We do not do this before the `steps` loop
# because model will be compiled only in the first iteration of this loop
# in the deferred build scenario.
if hasattr(model, '_compile_metrics'):
for m in model.metrics:
m.reset_states()
for m in output_loss_metrics:
m.reset_states()
# Calculate model output, loss values.
loss_outs, loss, _, aggregated_loss_metrics, masks = _model_loss(
model,
x,
y,
output_loss_metrics=output_loss_metrics,
sample_weights=sample_weights,
training=False)
metrics_results = _eager_metrics_fn(model,
loss_outs,
y,
sample_weights=sample_weights,
masks=masks)
batch_outs = []
for _, v in zip(model.metrics_names, [backend.mean(loss)] +
aggregated_loss_metrics + metrics_results):
batch_outs.append(tensor_util.constant_value(v))
# Get current step size.
if isinstance(x, list):
step_size = x[0].get_shape().as_list()[0]
elif isinstance(x, dict):
step_size = list(x.values())[0].get_shape().as_list()[0]
else:
step_size = x.get_shape().as_list()[0]
# Accumulate results in output array.
if not isinstance(batch_outs, list):
batch_outs = [batch_outs]
if step_index == 0:
for _ in enumerate(batch_outs):
outs.append(0.)
outs[0] += batch_outs[0] * step_size # index 0 = 'loss'
outs[1:] = batch_outs[1:]
# Calculate sample size.
num_samples += step_size
if verbose == 1:
progbar.update(step_index + 1)
outs[0] /= num_samples # index 0 = 'loss'
if len(outs) == 1:
return outs[0]
return outs
def iterator_test_loop(model, inputs, steps, verbose=0):
"""Test function for eager execution when input is given as dataset iterator.
Arguments:
model: Model instance that is being evaluated in Eager mode.
inputs: Input dataset iterator.
steps: Total number of steps (batches of samples) before declaring
predictions finished.
verbose: Verbosity mode.
Returns:
Scalar loss (if the model has a single output and no metrics)
or list of scalars (if the model has multiple outputs
and/or metrics). The attribute `model.metrics_names` will give you
the display labels for the scalar outputs.
Raises:
ValueError: In case of mismatch between given number of inputs and
expectations of the model.
"""
assert isinstance(inputs, iterator_ops.EagerIterator)
# make sure either x,y or x,y,sample_weights is provided
if (not isinstance(inputs.output_shapes, (list, tuple))
or len(inputs.output_shapes) < 2 or len(inputs.output_shapes) > 3):
raise ValueError('Please provide either inputs and targets'
'or inputs, targets, and sample_weights')
outs = []
num_samples = 0
if verbose == 1:
progbar = generic_utils.Progbar(target=steps)
for step_index in range(steps):
# Get data from the iterator.
try:
next_element = inputs.get_next()
except errors.OutOfRangeError:
logging.warning(
'Your dataset iterator ran out of data interrupting testing. '
'Make sure that your dataset can generate at least `steps` batches '
'(in this case, %d batches).', steps)
break
if len(inputs.output_shapes) == 2:
x, y = next_element
sample_weights = None
else:
x, y, sample_weights = next_element
# Validate and standardize data.
x, y, sample_weights = model._standardize_user_data(x, y)
x = training_utils.cast_if_floating_dtype(x)
y = training_utils.cast_if_floating_dtype(y)
# Calculate model output, loss values.
loss_outs, loss, loss_metrics = _model_loss(
model, x, y, sample_weights=sample_weights, training=False)
metrics_results = _eager_metrics_fn(model, loss_outs, y)
batch_outs = []
for _, v in zip(model.metrics_names,
[backend.mean(loss)] + loss_metrics + metrics_results):
batch_outs.append(tensor_util.constant_value(v))
# Get current step size.
if isinstance(x, list):
step_size = x[0].get_shape().as_list()[0]
else:
step_size = x.get_shape().as_list()[0]
# Accumulate results in output array.
if not isinstance(batch_outs, list):
batch_outs = [batch_outs]
if step_index == 0:
for _ in enumerate(batch_outs):
outs.append(0.)
for i, batch_out in enumerate(batch_outs):
outs[i] += batch_out * step_size
# Calculate sample size.
num_samples += step_size
if verbose == 1:
progbar.update(step_index + 1)
for i in range(len(outs)):
outs[i] /= num_samples
if len(outs) == 1:
return outs[0]
return outs
def train_frcnn(options):
if options.parser == 'pascal_voc':
from utils import voc_parser as get_data
elif options.parser == 'simple':
from utils import simple_parser as get_data
else:
raise ValueError(
"Command line option parser must be one of 'pascal_voc' or 'simple'"
)
# pass the settings from the command line, and persist them in the config object
C = Config()
C.use_horizontal_flips = bool(options.horizontal_flips)
C.use_vertical_flips = bool(options.vertical_flips)
C.rot_90 = bool(options.rot_90)
C.model_path = options.output_weight_path.format(options.network)
C.num_rois = int(options.num_rois)
if options.network == 'resnet50':
C.network = 'resnet50'
from utils import rpn_res as rpn
from utils import classifier_res as classifier_func
from utils import get_img_output_length_res as get_img_output_length
from utils import nn_base_res as nn_base
elif options.network == 'vgg':
C.network = 'vgg'
from utils import rpn_vgg as rpn
from utils import classifier_vgg as classifier_func
from utils import get_img_output_length_vgg as get_img_output_length
from utils import nn_base_vgg as nn_base
else:
print('Not a valid model')
raise ValueError
# check if weight path was passed via command line
if options.input_weight_path:
C.base_net_weights = options.input_weight_path
else:
# set the path to weights based on backend and model
C.base_net_weights = get_weight_path(options.network)
all_imgs, classes_count, class_mapping = get_data(options.path)
if 'bg' not in classes_count:
classes_count['bg'] = 0
class_mapping['bg'] = len(class_mapping)
C.class_mapping = class_mapping
inv_map = {v: k for k, v in class_mapping.items()}
print('Training images per class:')
pprint.pprint(classes_count)
print('Num classes (including bg) = {}'.format(len(classes_count)))
config_output_filename = options.config_filename
with open(config_output_filename, 'wb') as config_f:
pickle.dump(C, config_f)
print(
'Config has been written to {}, and can be loaded when testing to ensure correct results'
.format(config_output_filename))
#
random.shuffle(all_imgs)
train_imgs = [s for s in all_imgs if s['imageset'] == 'trainval']
val_imgs = [s for s in all_imgs if s['imageset'] == 'test']
print('Num train samples {}'.format(len(train_imgs)))
print('Num val samples {}'.format(len(val_imgs)))
data_gen_train = get_anchor_gt(train_imgs,
classes_count,
C,
get_img_output_length,
K.backend(),
mode='train')
data_gen_val = get_anchor_gt(val_imgs,
classes_count,
C,
get_img_output_length,
K.backend(),
mode='val')
if K.backend() == "theano":
input_shape_img = (3, None, None)
else:
input_shape_img = (None, None, 3)
img_input = Input(shape=input_shape_img)
roi_input = Input(shape=(None, 4))
# define the base network (resnet here, can be VGG, Inception, etc)
shared_layers = nn_base(img_input, trainable=True)
# define the RPN, built on the base layers
num_anchors = len(C.anchor_box_scales) * len(C.anchor_box_ratios)
rpn = rpn(shared_layers, num_anchors)
classifier = classifier_func(shared_layers,
roi_input,
C.num_rois,
nb_classes=len(classes_count),
trainable=True)
model_rpn = Model(img_input, rpn[:2])
model_classifier = Model([img_input, roi_input], classifier)
# this is a model that holds both the RPN and the classifier, used to load/save weights for the models
model_all = Model([img_input, roi_input], rpn[:2] + classifier)
try:
print('loading weights from {}'.format(C.base_net_weights))
model_rpn.load_weights(C.base_net_weights + "rpn.h5", by_name=True)
model_classifier.load_weights(C.base_net_weights + "classifier.h5",
by_name=True)
except Exception as e:
model_rpn.load_weights(C.base_net_weights, by_name=True)
model_classifier.load_weights(C.base_net_weights, by_name=True)
print('Exception: {}'.format(e))
optimizer = Adam(lr=1e-5, decay=2e-7)
optimizer_classifier = Adam(lr=1e-5, decay=2e-7)
model_rpn.compile(
optimizer=optimizer,
loss=[rpn_loss_cls(num_anchors),
rpn_loss_regr(num_anchors)])
model_classifier.compile(
optimizer=optimizer_classifier,
loss=[class_loss_cls,
class_loss_regr(len(classes_count) - 1)],
metrics={'dense_class_{}'.format(len(classes_count)): 'accuracy'})
model_all.compile(optimizer='sgd', loss='mae')
epoch_length = options.epoch_length
num_epochs = int(options.num_epochs)
iter_num = 0
losses = np.zeros((epoch_length, 5))
rpn_accuracy_rpn_monitor = []
rpn_accuracy_for_epoch = []
start_time = time.time()
best_loss = np.Inf
print('Starting training')
for epoch_num in range(num_epochs):
progbar = generic_utils.Progbar(epoch_length)
print('Epoch {}/{}'.format(epoch_num + 1, num_epochs))
while True:
try:
if len(rpn_accuracy_rpn_monitor) == epoch_length and C.verbose:
mean_overlapping_bboxes = float(
sum(rpn_accuracy_rpn_monitor)) / len(
rpn_accuracy_rpn_monitor)
rpn_accuracy_rpn_monitor = []
print(
'Average number of overlapping bounding boxes from RPN = {} for {} previous iterations'
.format(mean_overlapping_bboxes, epoch_length))
if mean_overlapping_bboxes == 0:
print(
'RPN is not producing bounding boxes that overlap the ground truth boxes. '
'Check RPN settings or keep training.')
X, Y, img_data = next(data_gen_train)
loss_rpn = model_rpn.train_on_batch(X, Y)
P_rpn = model_rpn.predict_on_batch(X)
R = rpn_to_roi(P_rpn[0],
P_rpn[1],
C,
K.backend(),
use_regr=True,
overlap_thresh=0.7,
max_boxes=300)
# note: calc_iou converts from (x1,y1,x2,y2) to (x,y,w,h) format
X2, Y1, Y2, IouS = calc_iou(R, img_data, C, class_mapping)
if X2 is None:
rpn_accuracy_rpn_monitor.append(0)
rpn_accuracy_for_epoch.append(0)
continue
neg_samples = np.where(Y1[0, :, -1] == 1)
pos_samples = np.where(Y1[0, :, -1] == 0)
if len(neg_samples) > 0:
neg_samples = neg_samples[0]
else:
neg_samples = []
if len(pos_samples) > 0:
pos_samples = pos_samples[0]
else:
pos_samples = []
rpn_accuracy_rpn_monitor.append(len(pos_samples))
rpn_accuracy_for_epoch.append((len(pos_samples)))
if C.num_rois > 1:
if len(pos_samples) < C.num_rois // 2:
selected_pos_samples = pos_samples.tolist()
else:
selected_pos_samples = np.random.choice(
pos_samples, C.num_rois // 2,
replace=False).tolist()
try:
selected_neg_samples = np.random.choice(
neg_samples,
C.num_rois - len(selected_pos_samples),
replace=False).tolist()
except:
selected_neg_samples = np.random.choice(
neg_samples,
C.num_rois - len(selected_pos_samples),
replace=True).tolist()
sel_samples = selected_pos_samples + selected_neg_samples
else:
# in the extreme case where num_rois = 1, we pick a random pos or neg sample
selected_pos_samples = pos_samples.tolist()
selected_neg_samples = neg_samples.tolist()
if np.random.randint(0, 2):
sel_samples = random.choice(selected_neg_samples)
else:
sel_samples = random.choice(selected_pos_samples)
loss_class = model_classifier.train_on_batch(
[X, X2[:, sel_samples, :]],
[Y1[:, sel_samples, :], Y2[:, sel_samples, :]])
losses[iter_num, 0] = loss_rpn[1]
losses[iter_num, 1] = loss_rpn[2]
losses[iter_num, 2] = loss_class[1]
losses[iter_num, 3] = loss_class[2]
losses[iter_num, 4] = loss_class[3]
progbar.update(iter_num + 1,
[('rpn_cls', losses[iter_num, 0]),
('rpn_regr', losses[iter_num, 1]),
('detector_cls', losses[iter_num, 2]),
('detector_regr', losses[iter_num, 3])])
iter_num += 1
if iter_num == epoch_length:
loss_rpn_cls = np.mean(losses[:, 0])
loss_rpn_regr = np.mean(losses[:, 1])
loss_class_cls = np.mean(losses[:, 2])
loss_class_regr = np.mean(losses[:, 3])
class_acc = np.mean(losses[:, 4])
mean_overlapping_bboxes = float(sum(
rpn_accuracy_for_epoch)) / len(rpn_accuracy_for_epoch)
rpn_accuracy_for_epoch = []
if C.verbose:
print(
'Mean number of bounding boxes from RPN overlapping ground truth boxes: {}'
.format(mean_overlapping_bboxes))
print(
'Classifier accuracy for bounding boxes from RPN: {}'
.format(class_acc))
print('Loss RPN classifier: {}'.format(loss_rpn_cls))
print('Loss RPN regression: {}'.format(loss_rpn_regr))
print('Loss Detector classifier: {}'.format(
loss_class_cls))
print('Loss Detector regression: {}'.format(
loss_class_regr))
print('Elapsed time: {}'.format(time.time() -
start_time))
curr_loss = loss_rpn_cls + loss_rpn_regr + loss_class_cls + loss_class_regr
iter_num = 0
start_time = time.time()
if curr_loss < best_loss:
if C.verbose:
print(
f'Total loss decreased from {best_loss:.3f} to {curr_loss:.3f}, saving weights to '
f'{C.model_path}')
best_loss = curr_loss
model_classifier.save_weights(C.model_path +
"classifier.h5")
model_rpn.save_weights(C.model_path + "rpn.h5")
break
except Exception as e:
print('Exception: {}'.format(e))
continue
print('Training complete, exiting.')
def iterator_predict_loop(model, inputs, steps, verbose=0):
"""Predict function for eager execution when input is dataset iterator.
Arguments:
model: Instance of `Model`.
inputs: Input dataset iterator.
steps: Total number of steps (batches of samples) before declaring
`_predict_loop` finished.
verbose: Verbosity mode.
Returns:
Array of predictions (if the model has a single output)
or list of arrays of predictions (if the model has multiple outputs).
Raises:
ValueError: In case of mismatch between given number of inputs and
expectations of the model.
"""
assert isinstance(inputs, iterator_ops.EagerIterator)
outs = []
if verbose == 1:
progbar = generic_utils.Progbar(target=steps)
for step_index in range(steps):
# Get data from the iterator.
try:
next_element = inputs.get_next()
except errors.OutOfRangeError:
logging.warning(
'Your dataset iterator ran out of data; '
'interrupting prediction. Make sure that your '
'dataset can generate at least `steps` '
'batches (in this case, %d batches).', steps)
break
if not isinstance(next_element,
(list, tuple)) or len(next_element) != 2:
raise ValueError(
'Please provide data as a list or tuple of 2 elements '
' - input and target pair. Received %s. We do not use the '
'`target` value here.' % next_element)
x, _ = next_element
# Validate and standardize data.
x, _, _ = model._standardize_user_data(x)
if model._expects_training_arg:
batch_outs = model.call(x[0] if len(x) == 1 else x, training=False)
else:
batch_outs = model.call(x[0] if len(x) == 1 else x)
if not isinstance(batch_outs, list):
batch_outs = [batch_outs]
# We collect the results from every step and then concatenate them once
# in the end. This is an expensive process. We are doing this because we
# do not know the number of samples beforehand.
if step_index == 0:
for _ in batch_outs:
outs.append([])
for i, batch_out in enumerate(batch_outs):
outs[i].append(backend.get_value(batch_out))
if verbose == 1:
progbar.update(step_index + 1)
for i, out in enumerate(outs):
outs[i] = np.concatenate(tuple(out), axis=0)
if len(outs) == 1:
return outs[0]
return outs
def batch_test_loop(model,
inputs,
targets,
batch_size,
sample_weights=None,
verbose=0):
"""Test function for eager execution when input is given as arrays or tensors.
Arguments:
model: Model instance that is being evaluated in Eager mode.
inputs: List of input arrays.
targets: List of target arrays.
batch_size: Integer batch size.
sample_weights: Optional list of sample weight arrays.
verbose: Verbosity mode.
Returns:
Scalar loss (if the model has a single output and no metrics)
or list of scalars (if the model has multiple outputs
and/or metrics). The attribute `model.metrics_names` will give you
the display labels for the scalar outputs.
"""
outs = []
feed_data = inputs + targets
if sample_weights:
feed_data += sample_weights
num_samples = training_utils.check_num_samples(feed_data,
batch_size=batch_size)
if verbose == 1:
progbar = generic_utils.Progbar(target=num_samples)
batches = generic_utils.make_batches(num_samples, batch_size)
index_array = np.arange(num_samples)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
inputs_batch = slice_arrays(inputs, batch_ids)
targets_batch = slice_arrays(targets, batch_ids)
if sample_weights:
sample_weights_batch = slice_arrays(sample_weights, batch_ids)
else:
sample_weights_batch = None
inputs_batch = [
ops.convert_to_tensor(val, dtype=backend.floatx())
for val in inputs_batch
]
targets_batch = [
ops.convert_to_tensor(val, dtype=backend.floatx())
for val in targets_batch
]
if sample_weights:
sample_weights_batch = [
ops.convert_to_tensor(val, dtype=backend.floatx())
if val is not None else None for val in sample_weights_batch
]
loss_outs, loss, loss_metrics = _model_loss(
model,
inputs_batch,
targets_batch,
sample_weights=sample_weights_batch,
training=False)
metrics_results = _eager_metrics_fn(model, loss_outs, targets_batch)
batch_outs = []
for _, v in zip(model.metrics_names,
[backend.mean(loss)] + loss_metrics + metrics_results):
batch_outs.append(tensor_util.constant_value(v))
if isinstance(batch_outs, list):
if batch_index == 0:
for _ in enumerate(batch_outs):
outs.append(0.)
for i, batch_out in enumerate(batch_outs):
outs[i] += batch_out * len(batch_ids)
else:
if batch_index == 0:
outs.append(0.)
outs[0] += batch_outs * len(batch_ids)
if verbose == 1:
progbar.update(batch_end)
for i in range(len(outs)):
outs[i] /= num_samples
if len(outs) == 1:
return outs[0]
return outs
def image_training(curr_config, class_mapping, model_all, model_rpn,
model_classifier, data_gen_train):
"""
Starting the training process.
Generate X (resizeImg) and label Y ([y_rpn_classifier, y_rpn_regr])
Training rpn model and get loss value for loss rpn classifier and loss rpn regression.
Getting predicted rpn from rpn model for rpn classifier and loss rpn regression.
Converting rpn layer to roi bounding boxes.
Generating data such as loss rpn classifier, loss rpn regression, loss class classifier, loss class regression,
total_loss and class_accuracy for each number of Epochs. These data will be used to generate graphs.
:param curr_config: updated config class.
:param class_mapping: abnormal behaviour classes(labels).
:param model_all: optimal models for faster r-cnn and vgg16.
:param model_rpn: optimal rpn model.
:param model_classifier: optimal classifier model.
:param data_gen_train: the ground_truth anchors data.
"""
print('Starting image training')
# start time
start_time = time.time()
rpn_accuracy_rpn_monitor = []
rpn_accuracy_for_epoch = []
losses = np.zeros((curr_config.epoch_length, 5))
best_loss = np.Inf
iter_num = 0
all_data = []
# record_dataset = pd.DataFrame(columns=['mean_overlapping_bboxes', 'class_acc', 'loss_rpn_cls','loss_rpn_regr', 'loss_class_cls', 'loss_class_regr','curr_loss', 'elapsed_time'])
for epoch_num in range(curr_config.num_epochs):
progbar = generic_utils.Progbar(curr_config.epoch_length)
print('Epoch {}/{}'.format(epoch_num + 1, curr_config.num_epochs))
while True:
try:
if len(rpn_accuracy_rpn_monitor
) == curr_config.epoch_length and curr_config.verbose:
mean_overlapping_bboxes = float(
sum(rpn_accuracy_rpn_monitor)) / len(
rpn_accuracy_rpn_monitor)
rpn_accuracy_rpn_monitor = []
print(
'Average number of overlapping bounding boxes from RPN = {} for {} previous iterations'
.format(mean_overlapping_bboxes,
curr_config.epoch_length))
if mean_overlapping_bboxes == 0:
print(
'RPN is not producing bounding boxes that overlap the ground truth boxes. Check RPN settings or keep training.'
)
X, Y, img_data = next(data_gen_train)
loss_rpn = model_rpn.train_on_batch(X, Y)
P_rpn = model_rpn.predict_on_batch(X)
R = roi_helpers.rpn_to_roi(P_rpn[0],
P_rpn[1],
curr_config,
K.image_data_format(),
use_regr=True,
overlap_thresh=0.7,
max_boxes=300)
# calc_iou converts from (x1,y1,x2,y2) to (x,y,w,h) format.
# X2: bounding boxes that IntersectionOfUnion is greater than config.classifierMinOverlap.
# for all bounding boxes ground truth in 300 non_max_suppression bounding boxes.
# Y1: one hot code for bounding boxes from above => x_roi (X).
# Y2: corresponding labels and corresponding gt bounding boxes.
X2, Y1, Y2, IouS = roi_helpers.calculate_iou(
R, img_data, curr_config, class_mapping)
if X2 is None:
rpn_accuracy_rpn_monitor.append(0)
rpn_accuracy_for_epoch.append(0)
continue
neg_samples = np.where(Y1[0, :, -1] == 1)
pos_samples = np.where(Y1[0, :, -1] == 0)
if len(neg_samples) > 0:
neg_samples = neg_samples[0]
else:
neg_samples = []
if len(pos_samples) > 0:
pos_samples = pos_samples[0]
else:
pos_samples = []
rpn_accuracy_rpn_monitor.append(len(pos_samples))
rpn_accuracy_for_epoch.append((len(pos_samples)))
if curr_config.num_rois > 1:
# if number of positive anchors is larger than 4//2 --> randomly choose 2 pos samples.
if len(pos_samples) < curr_config.num_rois // 2:
selected_pos_samples = pos_samples.tolist()
else:
selected_pos_samples = np.random.choice(
pos_samples,
curr_config.num_rois // 2,
replace=False).tolist()
try:
selected_neg_samples = np.random.choice(
neg_samples,
curr_config.num_rois - len(selected_pos_samples),
replace=False).tolist()
except:
selected_neg_samples = np.random.choice(
neg_samples,
curr_config.num_rois - len(selected_pos_samples),
replace=True).tolist()
sel_samples = selected_pos_samples + selected_neg_samples
else:
# in the extreme case where num_rois = 1, we pick a random pos or neg sample
selected_pos_samples = pos_samples.tolist()
selected_neg_samples = neg_samples.tolist()
if np.random.randint(0, 2):
sel_samples = random.choice(neg_samples)
else:
sel_samples = random.choice(pos_samples)
loss_class = model_classifier.train_on_batch(
[X, X2[:, sel_samples, :]],
[Y1[:, sel_samples, :], Y2[:, sel_samples, :]])
losses[iter_num, 0] = loss_rpn[1]
losses[iter_num, 1] = loss_rpn[2]
losses[iter_num, 2] = loss_class[1]
losses[iter_num, 3] = loss_class[2]
losses[iter_num, 4] = loss_class[3]
progbar.update(iter_num + 1,
[('rpn_cls', losses[iter_num, 0]),
('rpn_regr', losses[iter_num, 1]),
('detector_cls', losses[iter_num, 2]),
('detector_regr', losses[iter_num, 3])])
iter_num += 1
if iter_num == curr_config.epoch_length:
loss_rpn_cls = np.mean(losses[:, 0])
loss_rpn_regr = np.mean(losses[:, 1])
loss_class_cls = np.mean(losses[:, 2])
loss_class_regr = np.mean(losses[:, 3])
class_acc = np.mean(losses[:, 4])
mean_overlapping_bboxes = float(sum(
rpn_accuracy_for_epoch)) / len(rpn_accuracy_for_epoch)
rpn_accuracy_for_epoch = []
if curr_config.verbose:
print(
'Mean number of bounding boxes from RPN overlapping ground truth boxes: {}'
.format(mean_overlapping_bboxes))
print(
'Classifier accuracy for bounding boxes from RPN: {}'
.format(class_acc))
print('Loss RPN classifier: {}'.format(loss_rpn_cls))
print('Loss RPN regression: {}'.format(loss_rpn_regr))
print('Loss Detector classifier: {}'.format(
loss_class_cls))
print('Loss Detector regression: {}'.format(
loss_class_regr))
print('Elapsed time: {}'.format(time.time() -
start_time))
elapsed_time = (time.time() - start_time)
curr_loss = loss_rpn_cls + loss_rpn_regr + loss_class_cls + loss_class_regr
iter_num = 0
# finish time
start_time = time.time()
if curr_loss < best_loss:
if curr_config.verbose:
print(
'Total loss decreased from {} to {}, saving weights'
.format(best_loss, curr_loss))
best_loss = curr_loss
model_all.save_weights(curr_config.model_path)
# Create record csv file to store the train process
new_row = {
'mean_overlapping_bboxes':
round(mean_overlapping_bboxes, 3),
'class_acc':
round(float(class_acc), 3),
'loss_rpn_cls':
round(float(loss_rpn_cls), 3),
'loss_rpn_regr':
round(float(loss_rpn_regr), 3),
'loss_class_cls':
round(float(loss_class_cls), 3),
'loss_class_regr':
round(float(loss_class_regr), 3),
'curr_loss':
round(curr_loss, 3),
'elapsed_time':
round(elapsed_time, 3)
}
all_data.append(new_row)
record_dataset = pd.DataFrame(all_data)
record_dataset.to_csv(curr_config.train_process_data_file,
index=0)
break
except Exception as e:
print('Exception while Training images: {}'.format(e))
continue
print('Training complete, exiting.')
def example_gan(result_dir="output", data_dir="data"):
input_shape = (256, 256, 3)
local_shape = (128, 128, 3)
batch_size = 4
n_epoch = 100
tc = int(n_epoch * 0.18)
td = int(n_epoch * 0.02)
alpha = 0.0004
train_datagen = DataGenerator(data_dir, input_shape[:2], local_shape[:2])
generator = model_generator(input_shape)
discriminator = model_discriminator(input_shape, local_shape)
optimizer = Adadelta()
# build model
org_img = Input(shape=input_shape)
mask = Input(shape=(input_shape[0], input_shape[1], 1))
in_img = Lambda(lambda x: x[0] * (1 - x[1]),
output_shape=input_shape)([org_img, mask])
imitation = generator(in_img)
completion = Lambda(lambda x: x[0] * x[2] + x[1] * (1 - x[2]),
output_shape=input_shape)([imitation, org_img, mask])
cmp_container = Network([org_img, mask], completion)
cmp_out = cmp_container([org_img, mask])
cmp_model = Model([org_img, mask], cmp_out)
cmp_model.compile(loss='mse', optimizer=optimizer)
cmp_model.summary()
in_pts = Input(shape=(4, ), dtype='int32')
d_container = Network([org_img, in_pts], discriminator([org_img, in_pts]))
d_model = Model([org_img, in_pts], d_container([org_img, in_pts]))
d_model.compile(loss='binary_crossentropy', optimizer=optimizer)
d_model.summary()
d_container.trainable = False
all_model = Model([org_img, mask, in_pts],
[cmp_out, d_container([cmp_out, in_pts])])
all_model.compile(loss=['mse', 'binary_crossentropy'],
loss_weights=[1.0, alpha],
optimizer=optimizer)
all_model.summary()
for n in range(n_epoch):
progbar = generic_utils.Progbar(len(train_datagen))
for inputs, points, masks in train_datagen.flow(batch_size):
cmp_image = cmp_model.predict([inputs, masks])
valid = np.ones((batch_size, 1))
fake = np.zeros((batch_size, 1))
g_loss = 0.0
d_loss = 0.0
if n < tc:
g_loss = cmp_model.train_on_batch([inputs, masks], inputs)
else:
d_loss_real = d_model.train_on_batch([inputs, points], valid)
d_loss_fake = d_model.train_on_batch([cmp_image, points], fake)
d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)
if n >= tc + td:
g_loss = all_model.train_on_batch([inputs, masks, points],
[inputs, valid])
g_loss = g_loss[0] + alpha * g_loss[1]
progbar.add(inputs.shape[0],
values=[("D loss", d_loss), ("G mse", g_loss)])
num_img = min(5, batch_size)
fig, axs = plt.subplots(num_img, 3)
for i in range(num_img):
axs[i, 0].imshow(inputs[i] * (1 - masks[i]))
axs[i, 0].axis('off')
axs[i, 0].set_title('Input')
axs[i, 1].imshow(cmp_image[i])
axs[i, 1].axis('off')
axs[i, 1].set_title('Output')
axs[i, 2].imshow(inputs[i])
axs[i, 2].axis('off')
axs[i, 2].set_title('Ground Truth')
fig.savefig(os.path.join(result_dir, "result_%d.png" % n))
plt.close()
# save model
generator.save(os.path.join(result_dir, "generator.h5"))
discriminator.save(os.path.join(result_dir, "discriminator.h5"))
def train(self, batch_gen, noise_gen, num_gen_batches=1,
training_ratio=1, show_progress=True):
disc_target_real = None
if show_progress:
# Initialize progbar and batch counter
progbar = generic_utils.Progbar(
num_gen_batches*batch_gen.batch_size)
disc_target_real = np.ones(
(batch_gen.batch_size, batch_gen.num_frames, 1), dtype=np.float32)
disc_target_fake = -disc_target_real
gen_target = disc_target_real
target_gp = np.zeros((batch_gen.batch_size, 1), dtype=np.float32)
disc_target = [disc_target_real, disc_target_fake, target_gp]
loss_log = []
for k in range(num_gen_batches):
# train discriminator
disc_loss = None
disc_loss_n = 0
for rep in range(training_ratio):
# generate some real samples
(sample, cond) = next(batch_gen)
noise = noise_gen()
with Nontrainable(self.gen):
dl = self.disc_trainer.train_on_batch(
[cond,sample]+noise, disc_target)
if disc_loss is None:
disc_loss = np.array(dl)
else:
disc_loss += np.array(dl)
disc_loss_n += 1
del sample, cond
disc_loss /= disc_loss_n
with Nontrainable(self.disc):
(sample, cond) = next(batch_gen)
gen_loss = self.gen_trainer.train_on_batch(
[cond]+noise_gen(), gen_target)
del sample, cond
if show_progress:
losses = []
for (i,dl) in enumerate(disc_loss):
losses.append(("D{}".format(i), dl))
for (i,gl) in enumerate([gen_loss]):
losses.append(("G{}".format(i), gl))
progbar.add(batch_gen.batch_size,
values=losses)
loss_log.append(np.hstack((disc_loss,gen_loss)))
gc.collect()
return np.array(loss_log)
def train(self,
batch_gen,
latent_gen,
num_gen_batches=1,
training_ratio=1,
show_progress=True):
disc_target_real = None
if show_progress:
# Initialize progbar and batch counter
progbar = generic_utils.Progbar(num_gen_batches *
batch_gen.batch_size)
for k in range(num_gen_batches):
# Train discriminator
try:
self.gen.trainable = False
disc_loss = None
disc_loss_n = 0
for rep in range(training_ratio):
# generate some real samples
(Y_real, cond) = next(batch_gen)
latent = latent_gen()
if disc_target_real is None: # on the first iteration
# run discriminator once just to find the shapes
disc_outputs = self.disc_trainer.predict([Y_real] +
latent)
disc_target_real = np.ones(disc_outputs[0].shape,
dtype=np.float32)
disc_target_fake = -disc_target_real
gp_target = np.zeros(disc_outputs[2].shape,
dtype=np.float32)
del disc_outputs
dl = self.disc_trainer.train_on_batch([Y_real] + latent, [
disc_target_real, disc_target_fake, gp_target,
latent[0]
])
if disc_loss is None:
disc_loss = np.array(dl)
else:
disc_loss += np.array(dl)
disc_loss_n += 1
del Y_real, cond
disc_loss /= disc_loss_n
finally:
self.gen.trainable = True
# Train generator
try:
self.disc.trainable = False
latent = latent_gen()
gen_loss = self.gen_trainer.train_on_batch(
latent, [disc_target_fake, latent[0]])
finally:
self.disc.trainable = True
if show_progress:
losses = []
for (i, dl) in enumerate(disc_loss):
losses.append(("D{}".format(i), dl))
for (i, gl) in enumerate(gen_loss):
losses.append(("G{}".format(i), gl))
progbar.add(batch_gen.batch_size, values=losses)
gc.collect()
# Compile DCGAN model
DCGAN_model.compile(loss=loss,
loss_weights=loss_weights,
optimizer=opt_dcgan)
# Compile discriminator model
discriminator_model.trainable = True
discriminator_model.compile(loss='binary_crossentropy',
optimizer=opt_discriminator)
# Training
print("Start training")
for e in range(nb_epoch):
# Initialize progbar and batch counter
progbar = generic_utils.Progbar(epoch_size)
batch_counter = 1
start = time.time()
for X_full_batch, X_sketch_batch in gen_batch(X_full_train,
X_sketch_train,
batch_size):
# Create a batch to feed the discriminator model
X_disc, y_disc = get_disc_batch(X_full_batch,
X_sketch_batch,
generator_model,
batch_counter,
patch_size,
image_data_format,
label_smoothing=label_smoothing,
losses = np.zeros((epoch_length, 5))
rpn_accuracy_rpn_monitor = []
rpn_accuracy_for_epoch = []
start_time = time.time()
best_loss = np.Inf
class_mapping_inv = {v: k for k, v in class_mapping.items()}
print('Starting training')
vis = True
for epoch_num in range(num_epochs):
progbar = generic_utils.Progbar(epoch_length)
print(f'Epoch {epoch_num + 1}/{num_epochs}')
while True:
try:
if len(rpn_accuracy_rpn_monitor) == epoch_length and C.verbose:
mean_overlapping_bboxes = float(sum(
rpn_accuracy_rpn_monitor)) / len(rpn_accuracy_rpn_monitor)
rpn_accuracy_rpn_monitor = []
print(
f'Average number of overlapping bounding boxes from RPN = {mean_overlapping_bboxes} for {epoch_length} previous iterations'
)
if mean_overlapping_bboxes == 0:
print(
'RPN is not producing bounding boxes that overlap the ground truth boxes. Check RPN settings or keep training.'
