def batch_predict_loop(model, inputs, batch_size, verbose=0): """Predict function for eager execution when input is arrays or tensors. Arguments: model: Instance of `Model`. inputs: List of input arrays. batch_size: Integer batch size. verbose: Verbosity mode. Returns: Array of predictions (if the model has a single output) or list of arrays of predictions (if the model has multiple outputs). """ outs = [] num_samples = training_utils.check_num_samples(inputs, batch_size) if verbose == 1: progbar = generic_utils.Progbar(target=num_samples) batches = generic_utils.make_batches(num_samples, batch_size) index_array = np.arange(num_samples) for batch_index, (batch_start, batch_end) in enumerate(batches): batch_ids = index_array[batch_start:batch_end] inputs_batch = slice_arrays(inputs, batch_ids) inputs_batch = [ ops.convert_to_tensor(val, dtype=backend.floatx()) for val in inputs_batch ] if len(inputs_batch) == 1: if model._expects_training_arg: batch_outs = model.call(inputs_batch[0], training=False) else: batch_outs = model.call(inputs_batch[0]) else: if model._expects_training_arg: batch_outs = model.call(inputs_batch, training=False) else: batch_outs = model.call(inputs_batch) if not isinstance(batch_outs, list): batch_outs = [batch_outs] if batch_index == 0: # Pre-allocate the results arrays. for batch_out in batch_outs: dims = batch_out.shape[1:].dims dims_list = [d.value for d in dims] shape = (num_samples, ) + tuple(dims_list) outs.append( np.zeros(shape, dtype=batch_out.dtype.as_numpy_dtype)) for i, batch_out in enumerate(batch_outs): outs[i][batch_start:batch_end] = batch_out if verbose == 1: progbar.update(batch_end) if len(outs) == 1: return outs[0] return outs
def train_model(result_dir="result", data=data_train): train_datagen = Data(data_train, input_shape[:2], local_shape[:2]) for n in range(n_epoch): progbar = generic_utils.Progbar(len(train_datagen)) for inputs, points, masks in train_datagen.flow(batch_size): completion_image = completion_model.predict([inputs, masks]) valid = np.ones((batch_size, 1)) fake = np.zeros((batch_size, 1)) g_loss = 0.0 d_loss = 0.0 if n < tc: g_loss = completion_model.train_on_batch([inputs, masks], inputs) else: d_loss_real = d_model.train_on_batch([inputs, points], valid) d_loss_fake = d_model.train_on_batch( [completion_image, points], fake) d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) if n >= tc + td: g_loss = all_model.train_on_batch([inputs, masks, points], [inputs, valid]) g_loss = g_loss[0] + alpha * g_loss[1] progbar.add(inputs.shape[0], values=[("D loss", d_loss), ("G mse", g_loss)]) num_img = min(5, batch_size) fig, axs = plt.subplots(num_img, 3) for i in range(num_img): axs[i, 0].imshow(inputs[i] * (1 - masks[i])) axs[i, 0].axis('off') axs[i, 0].set_title('Input') axs[i, 1].imshow(completion_image[i]) axs[i, 1].axis('off') axs[i, 1].set_title('Output') axs[i, 2].imshow(inputs[i]) axs[i, 2].axis('off') axs[i, 2].set_title('Ground Truth') fig.savefig(os.path.join(result_dir, "result_%d.png" % n)) plt.close() # Save the checkpoints every 10 epochs if (n + 1) % 10 == 0: checkpoint.save(file_prefix=checkpoint_prefix) # save model generator.save(os.path.join("model", "generator.h5")) completion_model.save(os.path.join("model", "completion.h5")) discriminator.save(os.path.join("model", "discriminator.h5"))
def fit_generator(self, batch_gen, noise_gen, steps_per_epoch=1, num_epochs=1, training_ratio=1): disc_out_shape = (batch_gen.batch_size, self.disc.output_shape[1]) real_target = np.ones(disc_out_shape, dtype=np.float32) fake_target = -real_target gp_target = np.zeros_like(real_target) for epoch in range(num_epochs): print("Epoch {}/{}".format(epoch + 1, num_epochs)) # Initialize progbar and batch counter progbar = generic_utils.Progbar(steps_per_epoch * batch_gen.batch_size) for step in range(steps_per_epoch): # Train discriminator with Nontrainable(self.gen): for repeat in range(training_ratio): image_batch = next(batch_gen) noise_batch = next(noise_gen) disc_loss = self.disc_trainer.train_on_batch( [image_batch] + noise_batch, [real_target, fake_target, gp_target]) # Train generator with Nontrainable(self.disc): noise_batch = next(noise_gen) gen_loss = self.gen_trainer.train_on_batch( noise_batch, real_target) losses = [] for (i, dl) in enumerate(disc_loss): losses.append(("D{}".format(i), dl)) losses.append(("G0", gen_loss)) progbar.add(batch_gen.batch_size, values=losses)
def main_video_process(curr_config, class_to_color, model_rpn, model_classifier_only, video_file_name): """ Extracting the test video into image frame by calling convert_video_frame method. Detection process is called using Faster R-CNN with VGG 16 network. Display the detected results as a video output. :param curr_config: get config class. :param class_to_color: get colour for the bounding boxes as each class(label) represent different colour. :param model_rpn: get optimal rpn model. :param model_classifier_only: get optimal classifier model. :param video_file_name: get test image frame. :return test_data: get the dict of detected results. :return new_images: get the list of detected result image. """ test_data = {} new_images = [] print('Test video : {}'.format(video_file_name)) video_frames = convert_video_frame(video_file_name) print('Start Image Testing') progbar = generic_utils.Progbar(len(video_frames)) for idx, frame in enumerate(video_frames): all_data, new_img = image_testing(frame, curr_config, model_rpn, model_classifier_only, class_to_color, online=True) progbar.update(idx + 1, [('Image Testing Continue.', idx)]) if all_data: new_images.append(new_img) for data in all_data: class_name = data['class'] if class_name not in test_data: test_data[class_name] = [] test_data[class_name].append(data) print('End Image Testing') return test_data, new_images
def iterator_predict_loop(model, inputs, steps, verbose=0): """Predict function for eager execution when input is dataset iterator. Arguments: model: Instance of `Model`. inputs: Input dataset iterator. steps: Total number of steps (batches of samples) before declaring `_predict_loop` finished. verbose: Verbosity mode. Returns: Array of predictions (if the model has a single output) or list of arrays of predictions (if the model has multiple outputs). Raises: ValueError: In case of mismatch between given number of inputs and expectations of the model. """ assert isinstance(inputs, iterator_ops.EagerIterator) if not isinstance(inputs.output_shapes, collections.Sequence) or len(inputs.output_shapes) > 3: raise ValueError( 'Please provide data as a list or tuple of 1, 2, or 3 elements ' ' - `(input)`, or `(input, target)`, or `(input, target,' 'sample_weights)`. Received %s. We do not use the `target` or' '`sample_weights` value here.' % inputs.output_shapes) outs = [] if verbose == 1: progbar = generic_utils.Progbar(target=steps) for step_index in range(steps): # Get data from the iterator. try: next_element = inputs.get_next() except errors.OutOfRangeError: logging.warning( 'Your dataset iterator ran out of data; interrupting prediction. ' 'Make sure that your dataset can generate at least `steps` batches ' '(in this case, %d batches). You may need to use the repeat() ' 'function when building your dataset.', steps) break # expects a tuple, where first element of tuple represents inputs x = next_element[0] # Validate and standardize data. x, _, _ = model._standardize_user_data(x) x = training_utils.cast_if_floating_dtype(x) if isinstance(x, list) and len(x) == 1: x = x[0] if model._expects_training_arg: batch_outs = model.call(x, training=False) else: batch_outs = model.call(x) if not isinstance(batch_outs, list): batch_outs = [batch_outs] # We collect the results from every step and then concatenate them once # in the end. This is an expensive process. We are doing this because we # do not know the number of samples beforehand. if step_index == 0: for _ in batch_outs: outs.append([]) for i, batch_out in enumerate(batch_outs): outs[i].append(backend.get_value(batch_out)) if verbose == 1: progbar.update(step_index + 1) for i, out in enumerate(outs): outs[i] = np.concatenate(tuple(out), axis=0) if len(outs) == 1: return outs[0] return outs
def iterator_test_loop(model, inputs, steps, verbose=0): """Test function for eager execution when input is given as dataset iterator. Arguments: model: Model instance that is being evaluated in Eager mode. inputs: Input dataset iterator. steps: Total number of steps (batches of samples) before declaring predictions finished. verbose: Verbosity mode. Returns: Scalar loss (if the model has a single output and no metrics) or list of scalars (if the model has multiple outputs and/or metrics). The attribute `model.metrics_names` will give you the display labels for the scalar outputs. Raises: ValueError: In case of mismatch between given number of inputs and expectations of the model. """ assert isinstance(inputs, iterator_ops.EagerIterator) # make sure either x,y or x,y,sample_weights is provided if (not isinstance(inputs.output_shapes, collections.Sequence) or len(inputs.output_shapes) < 2 or len(inputs.output_shapes) > 3): raise ValueError('Please provide either inputs and targets' 'or inputs, targets, and sample_weights') outs = [] # Create metric wrapper for the losses. output_loss_metrics = [] for i in range(len(model.outputs)): loss_fn = model.loss_functions[i] loss_name = loss_fn.name if isinstance( loss_fn, losses_module.Loss) else loss_fn.__name__ mean_wrapped_loss = metrics_module.MeanMetricWrapper(loss_fn, name=loss_name) output_loss_metrics.append(mean_wrapped_loss) num_samples = 0 if verbose == 1: progbar = generic_utils.Progbar(target=steps) for step_index in range(steps): # Get data from the iterator. try: next_element = inputs.get_next() except errors.OutOfRangeError: logging.warning( 'Your dataset iterator ran out of data interrupting testing. ' 'Make sure that your dataset can generate at least `steps` batches ' '(in this case, %d batches). You may need to use the repeat() ' 'function when building your dataset.', steps) break if len(inputs.output_shapes) == 2: x, y = next_element sample_weights = None else: x, y, sample_weights = next_element # Validate and standardize data. x, y, sample_weights = model._standardize_user_data( x, y, sample_weight=sample_weights) x = training_utils.cast_if_floating_dtype(x) y = training_utils.cast_if_floating_dtype(y) if sample_weights: sample_weights = [ training_utils.cast_if_floating_dtype( ops.convert_to_tensor(val, dtype=backend.floatx())) if val is not None else None for val in sample_weights ] if step_index == 0: # Get stateful metrics indices. We do not do this before the `steps` loop # because model will be compiled only in the first iteration of this loop # in the deferred build scenario. if hasattr(model, '_compile_metrics'): for m in model.metrics: m.reset_states() for m in output_loss_metrics: m.reset_states() # Calculate model output, loss values. loss_outs, loss, _, aggregated_loss_metrics, masks = _model_loss( model, x, y, output_loss_metrics=output_loss_metrics, sample_weights=sample_weights, training=False) metrics_results = _eager_metrics_fn(model, loss_outs, y, sample_weights=sample_weights, masks=masks) batch_outs = [] for _, v in zip(model.metrics_names, [backend.mean(loss)] + aggregated_loss_metrics + metrics_results): batch_outs.append(tensor_util.constant_value(v)) # Get current step size. if isinstance(x, list): step_size = x[0].get_shape().as_list()[0] elif isinstance(x, dict): step_size = list(x.values())[0].get_shape().as_list()[0] else: step_size = x.get_shape().as_list()[0] # Accumulate results in output array. if not isinstance(batch_outs, list): batch_outs = [batch_outs] if step_index == 0: for _ in enumerate(batch_outs): outs.append(0.) outs[0] += batch_outs[0] * step_size # index 0 = 'loss' outs[1:] = batch_outs[1:] # Calculate sample size. num_samples += step_size if verbose == 1: progbar.update(step_index + 1) outs[0] /= num_samples # index 0 = 'loss' if len(outs) == 1: return outs[0] return outs
def iterator_test_loop(model, inputs, steps, verbose=0): """Test function for eager execution when input is given as dataset iterator. Arguments: model: Model instance that is being evaluated in Eager mode. inputs: Input dataset iterator. steps: Total number of steps (batches of samples) before declaring predictions finished. verbose: Verbosity mode. Returns: Scalar loss (if the model has a single output and no metrics) or list of scalars (if the model has multiple outputs and/or metrics). The attribute `model.metrics_names` will give you the display labels for the scalar outputs. Raises: ValueError: In case of mismatch between given number of inputs and expectations of the model. """ assert isinstance(inputs, iterator_ops.EagerIterator) # make sure either x,y or x,y,sample_weights is provided if (not isinstance(inputs.output_shapes, (list, tuple)) or len(inputs.output_shapes) < 2 or len(inputs.output_shapes) > 3): raise ValueError('Please provide either inputs and targets' 'or inputs, targets, and sample_weights') outs = [] num_samples = 0 if verbose == 1: progbar = generic_utils.Progbar(target=steps) for step_index in range(steps): # Get data from the iterator. try: next_element = inputs.get_next() except errors.OutOfRangeError: logging.warning( 'Your dataset iterator ran out of data interrupting testing. ' 'Make sure that your dataset can generate at least `steps` batches ' '(in this case, %d batches).', steps) break if len(inputs.output_shapes) == 2: x, y = next_element sample_weights = None else: x, y, sample_weights = next_element # Validate and standardize data. x, y, sample_weights = model._standardize_user_data(x, y) x = training_utils.cast_if_floating_dtype(x) y = training_utils.cast_if_floating_dtype(y) # Calculate model output, loss values. loss_outs, loss, loss_metrics = _model_loss( model, x, y, sample_weights=sample_weights, training=False) metrics_results = _eager_metrics_fn(model, loss_outs, y) batch_outs = [] for _, v in zip(model.metrics_names, [backend.mean(loss)] + loss_metrics + metrics_results): batch_outs.append(tensor_util.constant_value(v)) # Get current step size. if isinstance(x, list): step_size = x[0].get_shape().as_list()[0] else: step_size = x.get_shape().as_list()[0] # Accumulate results in output array. if not isinstance(batch_outs, list): batch_outs = [batch_outs] if step_index == 0: for _ in enumerate(batch_outs): outs.append(0.) for i, batch_out in enumerate(batch_outs): outs[i] += batch_out * step_size # Calculate sample size. num_samples += step_size if verbose == 1: progbar.update(step_index + 1) for i in range(len(outs)): outs[i] /= num_samples if len(outs) == 1: return outs[0] return outs
def train_frcnn(options): if options.parser == 'pascal_voc': from utils import voc_parser as get_data elif options.parser == 'simple': from utils import simple_parser as get_data else: raise ValueError( "Command line option parser must be one of 'pascal_voc' or 'simple'" ) # pass the settings from the command line, and persist them in the config object C = Config() C.use_horizontal_flips = bool(options.horizontal_flips) C.use_vertical_flips = bool(options.vertical_flips) C.rot_90 = bool(options.rot_90) C.model_path = options.output_weight_path.format(options.network) C.num_rois = int(options.num_rois) if options.network == 'resnet50': C.network = 'resnet50' from utils import rpn_res as rpn from utils import classifier_res as classifier_func from utils import get_img_output_length_res as get_img_output_length from utils import nn_base_res as nn_base elif options.network == 'vgg': C.network = 'vgg' from utils import rpn_vgg as rpn from utils import classifier_vgg as classifier_func from utils import get_img_output_length_vgg as get_img_output_length from utils import nn_base_vgg as nn_base else: print('Not a valid model') raise ValueError # check if weight path was passed via command line if options.input_weight_path: C.base_net_weights = options.input_weight_path else: # set the path to weights based on backend and model C.base_net_weights = get_weight_path(options.network) all_imgs, classes_count, class_mapping = get_data(options.path) if 'bg' not in classes_count: classes_count['bg'] = 0 class_mapping['bg'] = len(class_mapping) C.class_mapping = class_mapping inv_map = {v: k for k, v in class_mapping.items()} print('Training images per class:') pprint.pprint(classes_count) print('Num classes (including bg) = {}'.format(len(classes_count))) config_output_filename = options.config_filename with open(config_output_filename, 'wb') as config_f: pickle.dump(C, config_f) print( 'Config has been written to {}, and can be loaded when testing to ensure correct results' .format(config_output_filename)) # random.shuffle(all_imgs) train_imgs = [s for s in all_imgs if s['imageset'] == 'trainval'] val_imgs = [s for s in all_imgs if s['imageset'] == 'test'] print('Num train samples {}'.format(len(train_imgs))) print('Num val samples {}'.format(len(val_imgs))) data_gen_train = get_anchor_gt(train_imgs, classes_count, C, get_img_output_length, K.backend(), mode='train') data_gen_val = get_anchor_gt(val_imgs, classes_count, C, get_img_output_length, K.backend(), mode='val') if K.backend() == "theano": input_shape_img = (3, None, None) else: input_shape_img = (None, None, 3) img_input = Input(shape=input_shape_img) roi_input = Input(shape=(None, 4)) # define the base network (resnet here, can be VGG, Inception, etc) shared_layers = nn_base(img_input, trainable=True) # define the RPN, built on the base layers num_anchors = len(C.anchor_box_scales) * len(C.anchor_box_ratios) rpn = rpn(shared_layers, num_anchors) classifier = classifier_func(shared_layers, roi_input, C.num_rois, nb_classes=len(classes_count), trainable=True) model_rpn = Model(img_input, rpn[:2]) model_classifier = Model([img_input, roi_input], classifier) # this is a model that holds both the RPN and the classifier, used to load/save weights for the models model_all = Model([img_input, roi_input], rpn[:2] + classifier) try: print('loading weights from {}'.format(C.base_net_weights)) model_rpn.load_weights(C.base_net_weights + "rpn.h5", by_name=True) model_classifier.load_weights(C.base_net_weights + "classifier.h5", by_name=True) except Exception as e: model_rpn.load_weights(C.base_net_weights, by_name=True) model_classifier.load_weights(C.base_net_weights, by_name=True) print('Exception: {}'.format(e)) optimizer = Adam(lr=1e-5, decay=2e-7) optimizer_classifier = Adam(lr=1e-5, decay=2e-7) model_rpn.compile( optimizer=optimizer, loss=[rpn_loss_cls(num_anchors), rpn_loss_regr(num_anchors)]) model_classifier.compile( optimizer=optimizer_classifier, loss=[class_loss_cls, class_loss_regr(len(classes_count) - 1)], metrics={'dense_class_{}'.format(len(classes_count)): 'accuracy'}) model_all.compile(optimizer='sgd', loss='mae') epoch_length = options.epoch_length num_epochs = int(options.num_epochs) iter_num = 0 losses = np.zeros((epoch_length, 5)) rpn_accuracy_rpn_monitor = [] rpn_accuracy_for_epoch = [] start_time = time.time() best_loss = np.Inf print('Starting training') for epoch_num in range(num_epochs): progbar = generic_utils.Progbar(epoch_length) print('Epoch {}/{}'.format(epoch_num + 1, num_epochs)) while True: try: if len(rpn_accuracy_rpn_monitor) == epoch_length and C.verbose: mean_overlapping_bboxes = float( sum(rpn_accuracy_rpn_monitor)) / len( rpn_accuracy_rpn_monitor) rpn_accuracy_rpn_monitor = [] print( 'Average number of overlapping bounding boxes from RPN = {} for {} previous iterations' .format(mean_overlapping_bboxes, epoch_length)) if mean_overlapping_bboxes == 0: print( 'RPN is not producing bounding boxes that overlap the ground truth boxes. ' 'Check RPN settings or keep training.') X, Y, img_data = next(data_gen_train) loss_rpn = model_rpn.train_on_batch(X, Y) P_rpn = model_rpn.predict_on_batch(X) R = rpn_to_roi(P_rpn[0], P_rpn[1], C, K.backend(), use_regr=True, overlap_thresh=0.7, max_boxes=300) # note: calc_iou converts from (x1,y1,x2,y2) to (x,y,w,h) format X2, Y1, Y2, IouS = calc_iou(R, img_data, C, class_mapping) if X2 is None: rpn_accuracy_rpn_monitor.append(0) rpn_accuracy_for_epoch.append(0) continue neg_samples = np.where(Y1[0, :, -1] == 1) pos_samples = np.where(Y1[0, :, -1] == 0) if len(neg_samples) > 0: neg_samples = neg_samples[0] else: neg_samples = [] if len(pos_samples) > 0: pos_samples = pos_samples[0] else: pos_samples = [] rpn_accuracy_rpn_monitor.append(len(pos_samples)) rpn_accuracy_for_epoch.append((len(pos_samples))) if C.num_rois > 1: if len(pos_samples) < C.num_rois // 2: selected_pos_samples = pos_samples.tolist() else: selected_pos_samples = np.random.choice( pos_samples, C.num_rois // 2, replace=False).tolist() try: selected_neg_samples = np.random.choice( neg_samples, C.num_rois - len(selected_pos_samples), replace=False).tolist() except: selected_neg_samples = np.random.choice( neg_samples, C.num_rois - len(selected_pos_samples), replace=True).tolist() sel_samples = selected_pos_samples + selected_neg_samples else: # in the extreme case where num_rois = 1, we pick a random pos or neg sample selected_pos_samples = pos_samples.tolist() selected_neg_samples = neg_samples.tolist() if np.random.randint(0, 2): sel_samples = random.choice(selected_neg_samples) else: sel_samples = random.choice(selected_pos_samples) loss_class = model_classifier.train_on_batch( [X, X2[:, sel_samples, :]], [Y1[:, sel_samples, :], Y2[:, sel_samples, :]]) losses[iter_num, 0] = loss_rpn[1] losses[iter_num, 1] = loss_rpn[2] losses[iter_num, 2] = loss_class[1] losses[iter_num, 3] = loss_class[2] losses[iter_num, 4] = loss_class[3] progbar.update(iter_num + 1, [('rpn_cls', losses[iter_num, 0]), ('rpn_regr', losses[iter_num, 1]), ('detector_cls', losses[iter_num, 2]), ('detector_regr', losses[iter_num, 3])]) iter_num += 1 if iter_num == epoch_length: loss_rpn_cls = np.mean(losses[:, 0]) loss_rpn_regr = np.mean(losses[:, 1]) loss_class_cls = np.mean(losses[:, 2]) loss_class_regr = np.mean(losses[:, 3]) class_acc = np.mean(losses[:, 4]) mean_overlapping_bboxes = float(sum( rpn_accuracy_for_epoch)) / len(rpn_accuracy_for_epoch) rpn_accuracy_for_epoch = [] if C.verbose: print( 'Mean number of bounding boxes from RPN overlapping ground truth boxes: {}' .format(mean_overlapping_bboxes)) print( 'Classifier accuracy for bounding boxes from RPN: {}' .format(class_acc)) print('Loss RPN classifier: {}'.format(loss_rpn_cls)) print('Loss RPN regression: {}'.format(loss_rpn_regr)) print('Loss Detector classifier: {}'.format( loss_class_cls)) print('Loss Detector regression: {}'.format( loss_class_regr)) print('Elapsed time: {}'.format(time.time() - start_time)) curr_loss = loss_rpn_cls + loss_rpn_regr + loss_class_cls + loss_class_regr iter_num = 0 start_time = time.time() if curr_loss < best_loss: if C.verbose: print( f'Total loss decreased from {best_loss:.3f} to {curr_loss:.3f}, saving weights to ' f'{C.model_path}') best_loss = curr_loss model_classifier.save_weights(C.model_path + "classifier.h5") model_rpn.save_weights(C.model_path + "rpn.h5") break except Exception as e: print('Exception: {}'.format(e)) continue print('Training complete, exiting.')
def iterator_predict_loop(model, inputs, steps, verbose=0): """Predict function for eager execution when input is dataset iterator. Arguments: model: Instance of `Model`. inputs: Input dataset iterator. steps: Total number of steps (batches of samples) before declaring `_predict_loop` finished. verbose: Verbosity mode. Returns: Array of predictions (if the model has a single output) or list of arrays of predictions (if the model has multiple outputs). Raises: ValueError: In case of mismatch between given number of inputs and expectations of the model. """ assert isinstance(inputs, iterator_ops.EagerIterator) outs = [] if verbose == 1: progbar = generic_utils.Progbar(target=steps) for step_index in range(steps): # Get data from the iterator. try: next_element = inputs.get_next() except errors.OutOfRangeError: logging.warning( 'Your dataset iterator ran out of data; ' 'interrupting prediction. Make sure that your ' 'dataset can generate at least `steps` ' 'batches (in this case, %d batches).', steps) break if not isinstance(next_element, (list, tuple)) or len(next_element) != 2: raise ValueError( 'Please provide data as a list or tuple of 2 elements ' ' - input and target pair. Received %s. We do not use the ' '`target` value here.' % next_element) x, _ = next_element # Validate and standardize data. x, _, _ = model._standardize_user_data(x) if model._expects_training_arg: batch_outs = model.call(x[0] if len(x) == 1 else x, training=False) else: batch_outs = model.call(x[0] if len(x) == 1 else x) if not isinstance(batch_outs, list): batch_outs = [batch_outs] # We collect the results from every step and then concatenate them once # in the end. This is an expensive process. We are doing this because we # do not know the number of samples beforehand. if step_index == 0: for _ in batch_outs: outs.append([]) for i, batch_out in enumerate(batch_outs): outs[i].append(backend.get_value(batch_out)) if verbose == 1: progbar.update(step_index + 1) for i, out in enumerate(outs): outs[i] = np.concatenate(tuple(out), axis=0) if len(outs) == 1: return outs[0] return outs
def batch_test_loop(model, inputs, targets, batch_size, sample_weights=None, verbose=0): """Test function for eager execution when input is given as arrays or tensors. Arguments: model: Model instance that is being evaluated in Eager mode. inputs: List of input arrays. targets: List of target arrays. batch_size: Integer batch size. sample_weights: Optional list of sample weight arrays. verbose: Verbosity mode. Returns: Scalar loss (if the model has a single output and no metrics) or list of scalars (if the model has multiple outputs and/or metrics). The attribute `model.metrics_names` will give you the display labels for the scalar outputs. """ outs = [] feed_data = inputs + targets if sample_weights: feed_data += sample_weights num_samples = training_utils.check_num_samples(feed_data, batch_size=batch_size) if verbose == 1: progbar = generic_utils.Progbar(target=num_samples) batches = generic_utils.make_batches(num_samples, batch_size) index_array = np.arange(num_samples) for batch_index, (batch_start, batch_end) in enumerate(batches): batch_ids = index_array[batch_start:batch_end] inputs_batch = slice_arrays(inputs, batch_ids) targets_batch = slice_arrays(targets, batch_ids) if sample_weights: sample_weights_batch = slice_arrays(sample_weights, batch_ids) else: sample_weights_batch = None inputs_batch = [ ops.convert_to_tensor(val, dtype=backend.floatx()) for val in inputs_batch ] targets_batch = [ ops.convert_to_tensor(val, dtype=backend.floatx()) for val in targets_batch ] if sample_weights: sample_weights_batch = [ ops.convert_to_tensor(val, dtype=backend.floatx()) if val is not None else None for val in sample_weights_batch ] loss_outs, loss, loss_metrics = _model_loss( model, inputs_batch, targets_batch, sample_weights=sample_weights_batch, training=False) metrics_results = _eager_metrics_fn(model, loss_outs, targets_batch) batch_outs = [] for _, v in zip(model.metrics_names, [backend.mean(loss)] + loss_metrics + metrics_results): batch_outs.append(tensor_util.constant_value(v)) if isinstance(batch_outs, list): if batch_index == 0: for _ in enumerate(batch_outs): outs.append(0.) for i, batch_out in enumerate(batch_outs): outs[i] += batch_out * len(batch_ids) else: if batch_index == 0: outs.append(0.) outs[0] += batch_outs * len(batch_ids) if verbose == 1: progbar.update(batch_end) for i in range(len(outs)): outs[i] /= num_samples if len(outs) == 1: return outs[0] return outs
def image_training(curr_config, class_mapping, model_all, model_rpn, model_classifier, data_gen_train): """ Starting the training process. Generate X (resizeImg) and label Y ([y_rpn_classifier, y_rpn_regr]) Training rpn model and get loss value for loss rpn classifier and loss rpn regression. Getting predicted rpn from rpn model for rpn classifier and loss rpn regression. Converting rpn layer to roi bounding boxes. Generating data such as loss rpn classifier, loss rpn regression, loss class classifier, loss class regression, total_loss and class_accuracy for each number of Epochs. These data will be used to generate graphs. :param curr_config: updated config class. :param class_mapping: abnormal behaviour classes(labels). :param model_all: optimal models for faster r-cnn and vgg16. :param model_rpn: optimal rpn model. :param model_classifier: optimal classifier model. :param data_gen_train: the ground_truth anchors data. """ print('Starting image training') # start time start_time = time.time() rpn_accuracy_rpn_monitor = [] rpn_accuracy_for_epoch = [] losses = np.zeros((curr_config.epoch_length, 5)) best_loss = np.Inf iter_num = 0 all_data = [] # record_dataset = pd.DataFrame(columns=['mean_overlapping_bboxes', 'class_acc', 'loss_rpn_cls','loss_rpn_regr', 'loss_class_cls', 'loss_class_regr','curr_loss', 'elapsed_time']) for epoch_num in range(curr_config.num_epochs): progbar = generic_utils.Progbar(curr_config.epoch_length) print('Epoch {}/{}'.format(epoch_num + 1, curr_config.num_epochs)) while True: try: if len(rpn_accuracy_rpn_monitor ) == curr_config.epoch_length and curr_config.verbose: mean_overlapping_bboxes = float( sum(rpn_accuracy_rpn_monitor)) / len( rpn_accuracy_rpn_monitor) rpn_accuracy_rpn_monitor = [] print( 'Average number of overlapping bounding boxes from RPN = {} for {} previous iterations' .format(mean_overlapping_bboxes, curr_config.epoch_length)) if mean_overlapping_bboxes == 0: print( 'RPN is not producing bounding boxes that overlap the ground truth boxes. Check RPN settings or keep training.' ) X, Y, img_data = next(data_gen_train) loss_rpn = model_rpn.train_on_batch(X, Y) P_rpn = model_rpn.predict_on_batch(X) R = roi_helpers.rpn_to_roi(P_rpn[0], P_rpn[1], curr_config, K.image_data_format(), use_regr=True, overlap_thresh=0.7, max_boxes=300) # calc_iou converts from (x1,y1,x2,y2) to (x,y,w,h) format. # X2: bounding boxes that IntersectionOfUnion is greater than config.classifierMinOverlap. # for all bounding boxes ground truth in 300 non_max_suppression bounding boxes. # Y1: one hot code for bounding boxes from above => x_roi (X). # Y2: corresponding labels and corresponding gt bounding boxes. X2, Y1, Y2, IouS = roi_helpers.calculate_iou( R, img_data, curr_config, class_mapping) if X2 is None: rpn_accuracy_rpn_monitor.append(0) rpn_accuracy_for_epoch.append(0) continue neg_samples = np.where(Y1[0, :, -1] == 1) pos_samples = np.where(Y1[0, :, -1] == 0) if len(neg_samples) > 0: neg_samples = neg_samples[0] else: neg_samples = [] if len(pos_samples) > 0: pos_samples = pos_samples[0] else: pos_samples = [] rpn_accuracy_rpn_monitor.append(len(pos_samples)) rpn_accuracy_for_epoch.append((len(pos_samples))) if curr_config.num_rois > 1: # if number of positive anchors is larger than 4//2 --> randomly choose 2 pos samples. if len(pos_samples) < curr_config.num_rois // 2: selected_pos_samples = pos_samples.tolist() else: selected_pos_samples = np.random.choice( pos_samples, curr_config.num_rois // 2, replace=False).tolist() try: selected_neg_samples = np.random.choice( neg_samples, curr_config.num_rois - len(selected_pos_samples), replace=False).tolist() except: selected_neg_samples = np.random.choice( neg_samples, curr_config.num_rois - len(selected_pos_samples), replace=True).tolist() sel_samples = selected_pos_samples + selected_neg_samples else: # in the extreme case where num_rois = 1, we pick a random pos or neg sample selected_pos_samples = pos_samples.tolist() selected_neg_samples = neg_samples.tolist() if np.random.randint(0, 2): sel_samples = random.choice(neg_samples) else: sel_samples = random.choice(pos_samples) loss_class = model_classifier.train_on_batch( [X, X2[:, sel_samples, :]], [Y1[:, sel_samples, :], Y2[:, sel_samples, :]]) losses[iter_num, 0] = loss_rpn[1] losses[iter_num, 1] = loss_rpn[2] losses[iter_num, 2] = loss_class[1] losses[iter_num, 3] = loss_class[2] losses[iter_num, 4] = loss_class[3] progbar.update(iter_num + 1, [('rpn_cls', losses[iter_num, 0]), ('rpn_regr', losses[iter_num, 1]), ('detector_cls', losses[iter_num, 2]), ('detector_regr', losses[iter_num, 3])]) iter_num += 1 if iter_num == curr_config.epoch_length: loss_rpn_cls = np.mean(losses[:, 0]) loss_rpn_regr = np.mean(losses[:, 1]) loss_class_cls = np.mean(losses[:, 2]) loss_class_regr = np.mean(losses[:, 3]) class_acc = np.mean(losses[:, 4]) mean_overlapping_bboxes = float(sum( rpn_accuracy_for_epoch)) / len(rpn_accuracy_for_epoch) rpn_accuracy_for_epoch = [] if curr_config.verbose: print( 'Mean number of bounding boxes from RPN overlapping ground truth boxes: {}' .format(mean_overlapping_bboxes)) print( 'Classifier accuracy for bounding boxes from RPN: {}' .format(class_acc)) print('Loss RPN classifier: {}'.format(loss_rpn_cls)) print('Loss RPN regression: {}'.format(loss_rpn_regr)) print('Loss Detector classifier: {}'.format( loss_class_cls)) print('Loss Detector regression: {}'.format( loss_class_regr)) print('Elapsed time: {}'.format(time.time() - start_time)) elapsed_time = (time.time() - start_time) curr_loss = loss_rpn_cls + loss_rpn_regr + loss_class_cls + loss_class_regr iter_num = 0 # finish time start_time = time.time() if curr_loss < best_loss: if curr_config.verbose: print( 'Total loss decreased from {} to {}, saving weights' .format(best_loss, curr_loss)) best_loss = curr_loss model_all.save_weights(curr_config.model_path) # Create record csv file to store the train process new_row = { 'mean_overlapping_bboxes': round(mean_overlapping_bboxes, 3), 'class_acc': round(float(class_acc), 3), 'loss_rpn_cls': round(float(loss_rpn_cls), 3), 'loss_rpn_regr': round(float(loss_rpn_regr), 3), 'loss_class_cls': round(float(loss_class_cls), 3), 'loss_class_regr': round(float(loss_class_regr), 3), 'curr_loss': round(curr_loss, 3), 'elapsed_time': round(elapsed_time, 3) } all_data.append(new_row) record_dataset = pd.DataFrame(all_data) record_dataset.to_csv(curr_config.train_process_data_file, index=0) break except Exception as e: print('Exception while Training images: {}'.format(e)) continue print('Training complete, exiting.')
def example_gan(result_dir="output", data_dir="data"): input_shape = (256, 256, 3) local_shape = (128, 128, 3) batch_size = 4 n_epoch = 100 tc = int(n_epoch * 0.18) td = int(n_epoch * 0.02) alpha = 0.0004 train_datagen = DataGenerator(data_dir, input_shape[:2], local_shape[:2]) generator = model_generator(input_shape) discriminator = model_discriminator(input_shape, local_shape) optimizer = Adadelta() # build model org_img = Input(shape=input_shape) mask = Input(shape=(input_shape[0], input_shape[1], 1)) in_img = Lambda(lambda x: x[0] * (1 - x[1]), output_shape=input_shape)([org_img, mask]) imitation = generator(in_img) completion = Lambda(lambda x: x[0] * x[2] + x[1] * (1 - x[2]), output_shape=input_shape)([imitation, org_img, mask]) cmp_container = Network([org_img, mask], completion) cmp_out = cmp_container([org_img, mask]) cmp_model = Model([org_img, mask], cmp_out) cmp_model.compile(loss='mse', optimizer=optimizer) cmp_model.summary() in_pts = Input(shape=(4, ), dtype='int32') d_container = Network([org_img, in_pts], discriminator([org_img, in_pts])) d_model = Model([org_img, in_pts], d_container([org_img, in_pts])) d_model.compile(loss='binary_crossentropy', optimizer=optimizer) d_model.summary() d_container.trainable = False all_model = Model([org_img, mask, in_pts], [cmp_out, d_container([cmp_out, in_pts])]) all_model.compile(loss=['mse', 'binary_crossentropy'], loss_weights=[1.0, alpha], optimizer=optimizer) all_model.summary() for n in range(n_epoch): progbar = generic_utils.Progbar(len(train_datagen)) for inputs, points, masks in train_datagen.flow(batch_size): cmp_image = cmp_model.predict([inputs, masks]) valid = np.ones((batch_size, 1)) fake = np.zeros((batch_size, 1)) g_loss = 0.0 d_loss = 0.0 if n < tc: g_loss = cmp_model.train_on_batch([inputs, masks], inputs) else: d_loss_real = d_model.train_on_batch([inputs, points], valid) d_loss_fake = d_model.train_on_batch([cmp_image, points], fake) d_loss = 0.5 * np.add(d_loss_real, d_loss_fake) if n >= tc + td: g_loss = all_model.train_on_batch([inputs, masks, points], [inputs, valid]) g_loss = g_loss[0] + alpha * g_loss[1] progbar.add(inputs.shape[0], values=[("D loss", d_loss), ("G mse", g_loss)]) num_img = min(5, batch_size) fig, axs = plt.subplots(num_img, 3) for i in range(num_img): axs[i, 0].imshow(inputs[i] * (1 - masks[i])) axs[i, 0].axis('off') axs[i, 0].set_title('Input') axs[i, 1].imshow(cmp_image[i]) axs[i, 1].axis('off') axs[i, 1].set_title('Output') axs[i, 2].imshow(inputs[i]) axs[i, 2].axis('off') axs[i, 2].set_title('Ground Truth') fig.savefig(os.path.join(result_dir, "result_%d.png" % n)) plt.close() # save model generator.save(os.path.join(result_dir, "generator.h5")) discriminator.save(os.path.join(result_dir, "discriminator.h5"))
def train(self, batch_gen, noise_gen, num_gen_batches=1, training_ratio=1, show_progress=True): disc_target_real = None if show_progress: # Initialize progbar and batch counter progbar = generic_utils.Progbar( num_gen_batches*batch_gen.batch_size) disc_target_real = np.ones( (batch_gen.batch_size, batch_gen.num_frames, 1), dtype=np.float32) disc_target_fake = -disc_target_real gen_target = disc_target_real target_gp = np.zeros((batch_gen.batch_size, 1), dtype=np.float32) disc_target = [disc_target_real, disc_target_fake, target_gp] loss_log = [] for k in range(num_gen_batches): # train discriminator disc_loss = None disc_loss_n = 0 for rep in range(training_ratio): # generate some real samples (sample, cond) = next(batch_gen) noise = noise_gen() with Nontrainable(self.gen): dl = self.disc_trainer.train_on_batch( [cond,sample]+noise, disc_target) if disc_loss is None: disc_loss = np.array(dl) else: disc_loss += np.array(dl) disc_loss_n += 1 del sample, cond disc_loss /= disc_loss_n with Nontrainable(self.disc): (sample, cond) = next(batch_gen) gen_loss = self.gen_trainer.train_on_batch( [cond]+noise_gen(), gen_target) del sample, cond if show_progress: losses = [] for (i,dl) in enumerate(disc_loss): losses.append(("D{}".format(i), dl)) for (i,gl) in enumerate([gen_loss]): losses.append(("G{}".format(i), gl)) progbar.add(batch_gen.batch_size, values=losses) loss_log.append(np.hstack((disc_loss,gen_loss))) gc.collect() return np.array(loss_log)
def train(self, batch_gen, latent_gen, num_gen_batches=1, training_ratio=1, show_progress=True): disc_target_real = None if show_progress: # Initialize progbar and batch counter progbar = generic_utils.Progbar(num_gen_batches * batch_gen.batch_size) for k in range(num_gen_batches): # Train discriminator try: self.gen.trainable = False disc_loss = None disc_loss_n = 0 for rep in range(training_ratio): # generate some real samples (Y_real, cond) = next(batch_gen) latent = latent_gen() if disc_target_real is None: # on the first iteration # run discriminator once just to find the shapes disc_outputs = self.disc_trainer.predict([Y_real] + latent) disc_target_real = np.ones(disc_outputs[0].shape, dtype=np.float32) disc_target_fake = -disc_target_real gp_target = np.zeros(disc_outputs[2].shape, dtype=np.float32) del disc_outputs dl = self.disc_trainer.train_on_batch([Y_real] + latent, [ disc_target_real, disc_target_fake, gp_target, latent[0] ]) if disc_loss is None: disc_loss = np.array(dl) else: disc_loss += np.array(dl) disc_loss_n += 1 del Y_real, cond disc_loss /= disc_loss_n finally: self.gen.trainable = True # Train generator try: self.disc.trainable = False latent = latent_gen() gen_loss = self.gen_trainer.train_on_batch( latent, [disc_target_fake, latent[0]]) finally: self.disc.trainable = True if show_progress: losses = [] for (i, dl) in enumerate(disc_loss): losses.append(("D{}".format(i), dl)) for (i, gl) in enumerate(gen_loss): losses.append(("G{}".format(i), gl)) progbar.add(batch_gen.batch_size, values=losses) gc.collect()
# Compile DCGAN model DCGAN_model.compile(loss=loss, loss_weights=loss_weights, optimizer=opt_dcgan) # Compile discriminator model discriminator_model.trainable = True discriminator_model.compile(loss='binary_crossentropy', optimizer=opt_discriminator) # Training print("Start training") for e in range(nb_epoch): # Initialize progbar and batch counter progbar = generic_utils.Progbar(epoch_size) batch_counter = 1 start = time.time() for X_full_batch, X_sketch_batch in gen_batch(X_full_train, X_sketch_train, batch_size): # Create a batch to feed the discriminator model X_disc, y_disc = get_disc_batch(X_full_batch, X_sketch_batch, generator_model, batch_counter, patch_size, image_data_format, label_smoothing=label_smoothing,
losses = np.zeros((epoch_length, 5)) rpn_accuracy_rpn_monitor = [] rpn_accuracy_for_epoch = [] start_time = time.time() best_loss = np.Inf class_mapping_inv = {v: k for k, v in class_mapping.items()} print('Starting training') vis = True for epoch_num in range(num_epochs): progbar = generic_utils.Progbar(epoch_length) print(f'Epoch {epoch_num + 1}/{num_epochs}') while True: try: if len(rpn_accuracy_rpn_monitor) == epoch_length and C.verbose: mean_overlapping_bboxes = float(sum( rpn_accuracy_rpn_monitor)) / len(rpn_accuracy_rpn_monitor) rpn_accuracy_rpn_monitor = [] print( f'Average number of overlapping bounding boxes from RPN = {mean_overlapping_bboxes} for {epoch_length} previous iterations' ) if mean_overlapping_bboxes == 0: print( 'RPN is not producing bounding boxes that overlap the ground truth boxes. Check RPN settings or keep training.'