def train(images_tr, labels_tr, images_val, labels_val, model, model_cat, callbacks, train_config, loggers): # Create batch generators image_gen_tr = get_generator(images_tr, **train_config.daug.daug_params_tr) batch_gen_tr = generate_batches( image_gen_tr, images_tr, labels_tr, train_config.train.batch_size.gen_tr, aug_per_im=train_config.daug.aug_per_img_tr, shuffle=True, seed=train_config.seeds.batch_shuffle, n_inv_layers=train_config.optimizer.n_inv_layers) image_gen_val = get_generator(images_val, **train_config.daug.daug_params_val) batch_gen_val = generate_batches( image_gen_val, images_val, labels_val, train_config.train.batch_size.gen_val, aug_per_im=train_config.daug.aug_per_img_val, shuffle=False, n_inv_layers=train_config.optimizer.n_inv_layers) if FLAGS.no_val: batch_gen_val = None # Train model if FLAGS.no_fit_generator: metrics_names_val = [ 'val_{}'.format(metric_name) for metric_name in model.metrics_names ] no_mean_metrics_progbar = True # no_mean_metrics_progbar = False for callback in callbacks.values(): callback.set_model(model) callback.on_train_begin() for epoch in range(train_config.train.epochs): print('Epoch {}/{}'.format(epoch + 1, train_config.train.epochs)) # Progress bar # progbar = Progbar(target=train_config.train.batches_per_epoch_tr, # stateful_metrics=None) progbar = Progbar(target=train_config.train.batches_per_epoch_tr) for callback in callbacks.values(): callback.on_epoch_begin(epoch) for batch_idx in range(train_config.train.batches_per_epoch_tr): for callback in callbacks.values(): callback.on_batch_begin(batch_idx) # Train batch = next(batch_gen_tr) debug = False # Log if loggers: for logger in loggers: logger.get_activations() # debug if debug: preds = model.predict_on_batch(batch[0]) metrics = model.test_on_batch(batch[0], batch[1]) metrics_daug = metrics[model.metrics_names.index( 'daug_inv5_loss')] metrics_class = metrics[model.metrics_names.index( 'class_inv5_loss')] daug_true = batch[1][1] daug_true_rel = daug_true[:, :, 0] daug_true_all = daug_true[:, :, 1] class_true = batch[1][2] class_true_rel = class_true[:, :, 0] class_true_all = class_true[:, :, 1] pred_daug = preds[model.output_names.index( 'daug_inv5')][:, :, 0] pred_class = preds[model.output_names.index( 'class_inv5')][:, :, 0] num_daug = np.sum(daug_true_rel * pred_daug) / \ np.sum(daug_true_rel) den_daug = np.sum(daug_true_all * pred_daug) / \ np.sum(daug_true_all) loss_daug = num_daug / den_daug num_class = np.sum(class_true_rel * pred_class) / \ np.sum(class_true_rel) den_class = np.sum(class_true_all * pred_class) / \ np.sum(class_true_all) loss_class = num_class / den_class # debug metrics = model.train_on_batch(batch[0], batch[1]) if model_cat: output_inv = model.predict_on_batch(batch[0])[0] metrics_cat = model_cat.train_on_batch( output_inv, batch[1][0]) metrics_names_cat = model_cat.metrics_names[:] else: metrics_cat = [] metrics_names_cat = [] # Progress bar if batch_idx + 1 < progbar.target: metrics_progbar = sel_metrics( model.metrics_names, metrics, no_mean_metrics_progbar, metrics_cat=metrics_names_cat) metrics_progbar.extend(zip(metrics_names_cat, metrics_cat)) progbar.update(current=batch_idx + 1, values=metrics_progbar) # Log if loggers: metrics_log = sel_metrics(model.metrics_names, metrics, no_mean=False, metrics_cat=metrics_names_cat) metrics_log.extend(zip(metrics_names_cat, metrics_cat)) for logger in loggers: logger.log(metrics_log) for callback in callbacks.values(): callback.on_batch_end(batch_idx) # Validation metrics_val = np.zeros(len(metrics)) for batch_idx in range(train_config.train.batches_per_epoch_val): batch = next(batch_gen_val) metrics_val_batch = model.test_on_batch(batch[0], batch[1]) for idx, metric in enumerate(metrics_val_batch): metrics_val[idx] += metric metrics_val /= train_config.train.batches_per_epoch_val metrics_val = metrics_val.tolist() # Progress bar metrics_progbar = sel_metrics( model.metrics_names + metrics_names_val, metrics + metrics_val, no_mean_metrics_progbar, no_val_daug=train_config.daug.aug_per_img_val == 1) progbar.add(1, values=metrics_progbar) # Tensorboard metrics_names_tensorboard = list(progbar.sum_values.keys()) metrics_tensorboard = [ metric[0] / float(metric[1]) for metric in progbar.sum_values.values() ] for metric_name, metric in zip( model.metrics_names + metrics_names_val, metrics + metrics_val): if metric_name not in metrics_names_tensorboard: metrics_names_tensorboard.append(metric_name) metrics_tensorboard.append(metric) metrics_tensorboard = sel_metrics( metrics_names_tensorboard, metrics_tensorboard, no_mean=False, no_val_daug=train_config.daug.aug_per_img_val > 1, metrics_cat=[]) metrics_tensorboard = [ list(item) for item in zip(*metrics_tensorboard) ] write_tensorboard(callbacks['tensorboard'], metrics_tensorboard[0], metrics_tensorboard[1], epoch) for callback in callbacks.values(): callback.on_epoch_end(epoch) history = None else: history = model.fit_generator( generator=batch_gen_tr, steps_per_epoch=train_config.train.batches_per_epoch_tr, epochs=train_config.train.epochs, validation_data=batch_gen_val, validation_steps=train_config.train.batches_per_epoch_val, initial_epoch=train_config.train.initial_epoch, max_queue_size=train_config.data.queue_size, callbacks=list(callbacks.values())) if loggers: for logger in loggers: logger.close() return history, model
def test(images, labels, batch_size, model, model_adv, image_params_dict, attack_params_dict, output_file=None, do_print=True): """ Tests the performance of a model on adversarial images. The adversarial images are computed according to the attack specified in the arguments. Parameters ---------- images : dask array The set of images labels : dask array The ground truth labels batch_size : int Batch size model : Keras Model The model model_adv : Keras Model The model used to generate adversarial examples image_params_dict : dict Dictionary of data augmentation parameters attack_params_dict : dict Dictionary of the attack parameters output_file : str or None The outfile to write the results do_print : bool Whether to print the adversarial accuracy and MSE Returns ------- results_dict : dict Dictionary containing some performance metrics """ # Set test phase and get session sess = K.get_session() if isinstance(K.learning_phase(), int): learning_phase = K.learning_phase() K.set_learning_phase(0) # Initialize adversarial attack attack, attack_params, bs = init_attack(model_adv, attack_params_dict, sess) if bs: batch_size = bs # Create batch generator image_gen = data_input.get_generator(images, **image_params_dict) batch_gen = batch_generator(image_gen, images, labels, batch_size, aug_per_im=1, shuffle=False) n_batches_per_epoch = int(np.ceil(float(images.shape[0]) / batch_size)) # Define input TF placeholder if image_params_dict['crop_size']: image_shape = image_params_dict['crop_size'] else: image_shape = images.shape[1:] x = tf.placeholder(K.floatx(), shape=(bs,) + tuple(image_shape)) y = tf.placeholder(K.floatx(), shape=(bs,) + (labels.shape[-1],)) # Define adversarial predictions symbolically x_adv = attack.generate(x, **attack_params) x_adv = tf.stop_gradient(x_adv) predictions_adv = model(x_adv) # Define accuracy symbolically correct_preds = tf.equal(tf.argmax(y, axis=-1), tf.argmax(predictions_adv, axis=-1)) acc_value = tf.reduce_mean(tf.to_float(correct_preds)) # Define mean squared error symbolically mse_value = tf.reduce_mean(tf.square(tf.subtract(x, x_adv))) # Init results variables accuracy = 0.0 mse = 0.0 # Initialize matrix to store the predictions. # predictions = np.zeros([images.shape[0], labels.shape[-1]]) with sess.as_default(): init = 0 for _ in tqdm(range(n_batches_per_epoch)): batch = next(batch_gen()) this_batch_size = batch[0].shape[0] # Evaluate accuracy if isinstance(batch[1], (list, )): yy = batch[1][0] else: yy = batch[1] batch_acc = acc_value.eval(feed_dict={x: batch[0], y: yy}) # Evaluate MSE batch_mse = mse_value.eval(feed_dict={x: batch[0]}) # Adversarial predictions # predictions[init:init+this_batch_size, :] = \ # predictions_adv.eval(feed_dict={x: batch[0]}) # Update accuracy and MSE accuracy += (this_batch_size * batch_acc) mse += (this_batch_size * batch_mse) init += this_batch_size accuracy /= images.shape[0] mse /= images.shape[0] # Compute accuracy # acc_post = np.divide(np.sum(np.argmax(predictions, axis=1) == \ # np.argmax(labels, axis=1)), # float(images.shape[0])) if do_print: print('Aversarial accuracy against %s: %.4f' % (attack_params_dict['attack'], accuracy)) # print('Aversarial accuracy against %s: %.4f' % # (attack_params_dict['attack'], acc_post)) print('MSE between %s adversaries and originals: %.4f\n' % (attack_params_dict['attack'], mse)) if output_file is not None: _write_results(accuracy, output_file) # Set original phase if isinstance(K.learning_phase(), int): K.set_learning_phase(learning_phase) return accuracy, mse
def activations(images, labels, batch_size, model, layer_regex, nodaug_params, daug_params, include_input=False, class_invariance=False, n_daug_rep=0, norms=['fro']): """ Computes metrics from the activations, such as the norm of the feature maps, data augmentation invariance, class invariance, etc. Parameters ---------- images : h5py Dataset The set of images labels : h5py Dataset The ground truth labels batch_size : int Batch size model : Keras Model The model nodaug_params : dict Dictionary of data augmentation parameters for the baseline daug_params : dict Dictionary of data augmentation parameters include_input : bool If True, the input layer is considered for the analysis class_invariance : bool If True, the class invariance score is computed n_daug_rep : int If larger than 0, the data augentation invariance score is computed, performing n_daug_rep repetitions of random augmentations norms : list List of keywords to specify the types of norms to compute on the activations Returns ------- results_dict : dict Dictionary containing some performance metrics """ def _update_stats(mean_norm, std_norm, norm): mean_norm_batch = np.mean(norm, axis=0) std_norm_batch = np.std(norm, axis=0) mean_norm = init / float(end) * mean_norm + \ batch_size / float(end) * mean_norm_batch std_norm = init / float(end) * std_norm ** 2 + \ batch_size / float(end) * std_norm_batch ** 2 + \ (init * batch_size) / float(end ** 2) * \ (mean_norm - mean_norm_batch) ** 2 std_norm = np.sqrt(std_norm) return mean_norm, std_norm def _frobenius_norm(activations): norm = np.linalg.norm( activations, ord='fro', axis=tuple(range(1, len(activations.shape) - 1))) return norm def _inf_norm(activations): norm = np.max(np.abs(activations), axis=tuple(range(1, len(activations.shape) - 1))) return norm model = del_extra_nodes(model) n_images = images.shape[0] n_batches_per_epoch = int(np.ceil(float(n_images) / batch_size)) # Get relevant layers if include_input: layer_regex = '({}|.*input.*)'.format(layer_regex) else: layer_regex = layer_regex layers = [layer.name for layer in model.layers if re.compile(layer_regex).match(layer.name)] # Initialize HDF5 to store the activations # filename = 'hdf5_aux_{}'.format(time.time()) # activations_hdf5_aux = h5py.File(filename, 'w') # hdf5_aux = [filename] # # grp_activations = activations_hdf5_aux.create_group('activations') if class_invariance: # grp_labels = activations_hdf5_aux.create_group('labels') labels_true_da = [] labels_pred_da = [] predictions_da = [] # labels_true = grp_labels.create_dataset( # 'labels_true', shape=(n_images, ), dtype=np.uint8) # labels_pred = grp_labels.create_dataset( # 'labels_pred', shape=(n_images, ), dtype=np.uint8) # predictions = grp_labels.create_dataset( # 'predictions', shape=labels.shape, dtype=K.floatx()) idx_softmax = model.output_names.index('softmax') store_labels = True else: store_labels = False # Initialize results dictionary results_dict = {'activations_norm': {}, 'summary': {}, 'class_invariance': {}, 'daug_invariance': {}} # Iterate over the layers for layer_name in layers: # Create batch generator image_gen = get_generator(images, **nodaug_params) batch_gen = generate_batches(image_gen, images, labels, batch_size, aug_per_im=1, shuffle=False) layer = model.get_layer(layer_name) layer_shape = layer.output_shape[1:] n_channels = layer_shape[-1] if re.compile('.*input.*').match(layer_name): layer_name = 'input' print('\nLayer {}\n'.format(layer_name)) # Create a Dataset for the activations of the layer # activations_layer = grp_activations.create_dataset( # layer_name, shape=(n_images, ) + layer_shape, # dtype=K.floatx()) # Create dask array for the activations of the layer activations_layer_da = [] # Initialize placeholders in the results dict for the layer results_dict['activations_norm'].update({layer_name: {n: {'mean': np.zeros(n_channels), 'std': np.zeros(n_channels)} for n in norms}}) layer_dict = results_dict['activations_norm'][layer_name] activation_function = K.function([model.input, K.learning_phase()], [layer.output]) # Iterate over the data set in batches init = 0 for batch_images, batch_labels in tqdm( batch_gen, total=n_batches_per_epoch): batch_size = batch_images.shape[0] end = init + batch_size # Store labels if store_labels: preds = model.predict_on_batch(batch_images) if isinstance(preds, list): preds = preds[idx_softmax] labels_pred_da.append(da.from_array( np.argmax(preds, axis=1))) labels_true_da.append(da.from_array( np.argmax(batch_labels, axis=1))) predictions_da.append(da.from_array(preds)) # labels_pred[init:end] = np.argmax(preds, axis=1) # labels_true[init:end] = np.argmax(batch_labels, axis=1) # predictions[init:end, :] = preds # Get and store activations activations = activation_function([batch_images, 0])[0] activations_layer_da.append(da.from_array( activations, chunks=activations.shape)) # activations_layer[init:end] = activations # Compute norms for norm_key in norms: mean_norm = layer_dict[norm_key]['mean'] std_norm = layer_dict[norm_key]['std'] if norm_key == 'fro': norm = _frobenius_norm(activations) elif norm_key == 'inf': norm = _inf_norm(activations) else: raise NotImplementedError('Implemented norms are fro ' 'and inf') mean_norm, std_norm = _update_stats(mean_norm, std_norm, norm) layer_dict[norm_key]['mean'] = mean_norm layer_dict[norm_key]['std'] = std_norm init = end if init == n_images: store_labels = False break # Concatenate dask arrays activations_layer_da = da.concatenate(activations_layer_da, axis=0) activations_layer_da = activations_layer_da.reshape((n_images, -1)) d_activations = activations_layer_da.shape[-1] if class_invariance: print('\nComputing class invariance\n') labels_pred_da = da.concatenate(labels_pred_da) labels_true_da = da.concatenate(labels_true_da) predictions_da = da.concatenate(predictions_da) n_classes = len(np.unique(labels_true_da)) # Compute MSE matrix of the activations r = da.reshape(da.sum(da.square(activations_layer_da), axis=1), (-1, 1)) mse_matrix_da = (r - 2 * da.dot(activations_layer_da, da.transpose(activations_layer_da)) \ + da.transpose(r)) / d_activations mse_matrix_da = mse_matrix_da.rechunk((mse_matrix_da.chunksize[0], mse_matrix_da.shape[-1])) # Compute class invariance time0 = time() results_dict['class_invariance'].update({layer_name: {}}) class_invariance_scores_da = [] if class_invariance: # mse_matrix_mean = da.mean(mse_matrix_da).compute() for cl in tqdm(range(n_classes)): labels_cl = labels_pred_da == cl labels_cl = labels_cl.compute() mse_class = mse_matrix_da[labels_cl, :][:, labels_cl] mse_class = mse_class.rechunk((-1, -1)) # mse_class_mean = da.mean(mse_class).compute() # class_invariance_score = 1. - np.divide( # mse_class_mean, mse_matrix_mean) # results_dict['class_invariance'][layer_name].update( # {cl: class_invariance_score}) class_invariance_scores_da.append( 1. - da.divide(da.mean(mse_class), da.mean(mse_matrix_da))) # Compute data augmentation invariance print('\nComputing data augmentation invariance\n') mse_daug_da = [] results_dict['daug_invariance'].update({layer_name: {}}) for r in range(n_daug_rep): print('Repetition {}'.format(r)) image_gen_daug = get_generator(images, **daug_params) batch_gen_daug = generate_batches(image_gen_daug, images, labels, batch_size, aug_per_im=1, shuffle=False) activations_layer_daug_da = [] # Iterate over the data set in batches to compute activations init = 0 for batch_images, batch_labels in tqdm( batch_gen, total=n_batches_per_epoch): batch_size = batch_images.shape[0] end = init + batch_size # Get and store activations activations = activation_function([batch_images, 0])[0] activations_layer_daug_da.append(da.from_array( activations, chunks=activations.shape)) init = end if init == n_images: break activations_layer_daug_da = da.concatenate( activations_layer_daug_da, axis=0) activations_layer_daug_da = activations_layer_daug_da.reshape( (n_images, -1)) activations_layer_daug_da = activations_layer_daug_da.rechunk( (activations_layer_daug_da.chunksize[0], activations_layer_daug_da.shape[-1])) # Compute MSE daug mse_daug_da.append(da.mean(da.square(activations_layer_da - \ activations_layer_daug_da), axis=1)) mse_daug_da = da.stack(mse_daug_da, axis=1) mse_sum = da.repeat(da.reshape(da.sum(mse_matrix_da, axis=1), (n_images, 1)), n_daug_rep, axis=1) daug_invariance_score_da = 1 - n_images * da.divide(mse_daug_da, mse_sum) time1 = time() # Compute dask results and update results dict results_dask = da.compute(class_invariance_scores_da, daug_invariance_score_da) time2 = time() results_dict['class_invariance'][layer_name].update( {cl: cl_inv_score for cl, cl_inv_score in enumerate(results_dask[0])}) results_dict['daug_invariance'].update({layer_name: {r: daug_inv_score for r, daug_inv_score in enumerate(results_dask[1].T)}}) # Compute summary statistics of the norms across the channels for layer, layer_dict in results_dict['activations_norm'].items(): results_dict['summary'].update({layer: {}}) for norm_key, norm_dict in layer_dict.items(): results_dict['summary'][layer].update({norm_key: { 'mean': np.mean(norm_dict['mean']), 'std': np.mean(norm_dict['std'])}}) return results_dict
def activations_norm(images, labels, batch_size, model, layer_regex, daug_params, norms=['fro']): """ Computes the norm of the activation of all feature maps Parameters ---------- images : h5py Dataset The set of images labels : h5py Dataset The ground truth labels batch_size : int Batch size model : Keras Model The model daug_params : dict Dictionary of data augmentation parameters Returns ------- results_dict : dict Dictionary containing some performance metrics """ def _update_stats(mean_norm, std_norm, norm): mean_norm_batch = np.mean(norm, axis=0) std_norm_batch = np.std(norm, axis=0) mean_norm = init / float(end) * mean_norm + \ batch_size / float(end) * mean_norm_batch std_norm = init / float(end) * std_norm ** 2 + \ batch_size / float(end) * std_norm_batch ** 2 + \ (init * batch_size) / float(end ** 2) * \ (mean_norm - mean_norm_batch) ** 2 std_norm = np.sqrt(std_norm) return mean_norm, std_norm def _frobenius_norm(activations): norm = np.linalg.norm( activations, ord='fro', axis=tuple(range(1, len(activations.shape) - 1))) return norm def _inf_norm(activations): norm = np.max(np.abs(activations), axis=tuple(range(1, len(activations.shape) - 1))) return norm n_images = images.shape[0] n_batches_per_epoch = int(np.ceil(float(n_images) / batch_size)) # Create batch generator image_gen = get_generator(images, **daug_params) batch_gen = batch_generator(image_gen, images, labels, batch_size, aug_per_im=1, shuffle=False) # Initialize list to store the mean norm of the activations results_dict = {'activations_norm': {}, 'summary': {}} # Iterate over the layers model = del_extra_nodes(model) for layer in model.layers: if re.match(layer_regex, layer.name): layer_name = layer.name.encode('utf-8') print('\nLayer {}'.format(layer_name)) output = model.get_layer(layer_name)\ .outbound_nodes[0].input_tensors[0] get_output = K.function([model.input, K.learning_phase()], [output]) n_channels = K.int_shape(output)[-1] results_dict['activations_norm'].update({layer_name: {n: {'mean': np.zeros(n_channels), 'std': np.zeros(n_channels)} for n in norms}}) layer_dict = results_dict['activations_norm'][layer_name] init = 0 batch_gen.image_gen.reset() for _ in tqdm(range(n_batches_per_epoch)): batch_images, _ = next(batch_gen()) batch_size = batch_images.shape[0] end = init + batch_size activations = get_output([batch_images, 0])[0] for norm_key in norms: mean_norm = layer_dict[norm_key]['mean'] std_norm = layer_dict[norm_key]['std'] if norm_key == 'fro': norm = _frobenius_norm(activations) elif norm_key == 'inf': norm = _inf_norm(activations) else: raise NotImplementedError('Implemented norms are fro ' 'and inf') mean_norm, std_norm = _update_stats(mean_norm, std_norm, norm) layer_dict[norm_key]['mean'] = mean_norm layer_dict[norm_key]['std'] = std_norm init = end # Compute summary statistics across the channels for layer, layer_dict in results_dict['activations_norm'].items(): results_dict['summary'].update({layer: {}}) for norm_key, norm_dict in layer_dict.items(): results_dict['summary'][layer].update({norm_key: { 'mean': np.mean(norm_dict['mean']), 'std': np.mean(norm_dict['std'])}}) return results_dict
def test_adv(images, labels, batch_size, model, adv_model, daug_params, attack_params): """ Tests the performance of a model on adversarial images. The adversarial images are computed according to the attack specified in the arguments. Parameters ---------- images : dask array The set of images labels : dask array The ground truth labels batch_size : int Batch size model : Keras Model The model adv_model : Keras Model The model used to generate adversarial examples daug_params : dict Dictionary of data augmentation parameters attack_params : dict Dictionary of the attack parameters Returns ------- results_dict : dict Dictionary containing some performance metrics """ # Get session sess = K.get_session() # Initialize adversarial attack attack, attack_params_cleverhans, bs = init_attack( adv_model, attack_params, sess) if bs: batch_size = bs n_images = images.shape[0] n_classes = labels.shape[1] n_batches_per_epoch = int(np.ceil(float(n_images) / batch_size)) # Create batch generator image_gen = get_generator(images, **daug_params) batch_gen = batch_generator(image_gen, images, labels, batch_size, aug_per_im=1, shuffle=False) # Define input TF placeholder if daug_params['crop_size']: image_shape = daug_params['crop_size'] else: image_shape = images.shape[1:] x = tf.placeholder(K.floatx(), shape=(bs,) + tuple(image_shape)) y = tf.placeholder(K.floatx(), shape=(bs,) + (n_classes,)) # Define adversarial predictions symbolically x_adv = attack.generate(x, **attack_params_cleverhans) x_adv = tf.stop_gradient(x_adv) predictions_adv = model(x_adv) # Define accuracy and mean squared error symbolically correct_preds = tf.equal(tf.argmax(y, axis=-1), tf.argmax(predictions_adv, axis=-1)) acc_value = tf.reduce_mean(tf.to_float(correct_preds)) mse_value = tf.reduce_mean(tf.square(tf.subtract(x, x_adv))) # Init results variables accuracy = 0.0 mse = 0.0 with sess.as_default(): init = 0 for _ in tqdm(range(n_batches_per_epoch)): batch = next(batch_gen()) this_batch_size = batch[0].shape[0] # Evaluate accuracy if isinstance(batch[1], (list, )): yy = batch[1][0] else: yy = batch[1] # Evaluate accuracy and MSE batch_acc = acc_value.eval(feed_dict={x: batch[0], y: yy, K.learning_phase(): 0}) accuracy += (this_batch_size * batch_acc) batch_mse = mse_value.eval(feed_dict={x: batch[0], K.learning_phase(): 0}) mse += (this_batch_size * batch_mse) init += this_batch_size accuracy /= n_images mse /= n_images results_dict = {'mean_acc': accuracy, 'mean_mse': mse} return results_dict
def test_rep(images, labels, batch_size, model, daug_params, repetitions, metrics=['accuracy']): """ Tests the performance of a model on a set of images, transformed according to the specified augmentation parameters, and computes statistics over a number of repetitions. Parameters ---------- images : h5py Dataset The set of images labels : h5py Dataset The ground truth labels batch_size : int Batch size model : Keras Model The model daug_params : dict Dictionary of data augmentation parameters repetitions : int Number of data augmentation repetitions Returns ------- results_dict : dict Dictionary containing some performance metrics """ n_images = images.shape[0] n_classes = labels.shape[1] n_batches_per_epoch = int(np.ceil(float(n_images) / batch_size)) # Create batch generator image_gen = get_generator(images, **daug_params) batch_gen = batch_generator(image_gen, images, labels, batch_size, aug_per_im=1, shuffle=False) # Initialize matrix to store the predictions. predictions = np.zeros([n_images, n_classes, repetitions]) # Iterate over the random repetitions for r in range(repetitions): print('Run %d/%d' % (r+1, repetitions)) init = 0 batch_gen.image_gen.reset() # Iterate over the whole data set batch by batch for _ in tqdm(range(n_batches_per_epoch)): batch_images, _ = next(batch_gen()) batch_size = batch_images.shape[0] end = init + batch_size predictions[init:end, :, r] = \ model.predict_on_batch(batch_images) init = end results_dict = _stats_from_pred(predictions, labels, metrics) return results_dict
def main(argv=None): # cluster = LocalCluster(dashboard_address=None) # client = Client(cluster, memory_limit='{}GB'.format(FLAGS.memory_limit), # processes=False) K.set_floatx('float32') chunk_size = FLAGS.chunk_size # Read data set hdf5_file = h5py.File(FLAGS.data_file, 'r') images, labels, _ = hdf52dask(hdf5_file, FLAGS.group, chunk_size, shuffle=FLAGS.shuffle, seed=FLAGS.seed, pct=FLAGS.pct) n_images = images.shape[0] n_batches = int(np.ceil(n_images / float(FLAGS.batch_size))) # Data augmentation parameters daug_params_file = get_daug_scheme_path(FLAGS.daug_params, FLAGS.data_file) daug_params = yaml.load(open(daug_params_file, 'r'), Loader=yaml.FullLoader) nodaug_params_file = get_daug_scheme_path('nodaug.yml', FLAGS.data_file) nodaug_params = yaml.load(open(nodaug_params_file, 'r'), Loader=yaml.FullLoader) # Initialize the network model model_filename = FLAGS.model model = load_model(model_filename) # Print the model summary model.summary() # Get relevant layers if FLAGS.store_input: layer_regex = '({}|.*input.*)'.format(FLAGS.layer_regex) else: layer_regex = FLAGS.layer_regex layers = [ layer.name for layer in model.layers if re.compile(layer_regex).match(layer.name) ] # Create batch generators n_daug_rep = FLAGS.n_daug_rep n_diff_per_batch = int(FLAGS.batch_size / n_daug_rep) image_gen_daug = get_generator(images, **daug_params) batch_gen_daug = batch_generator(image_gen_daug, images, labels, batch_size=n_diff_per_batch, aug_per_im=n_daug_rep, shuffle=False) image_gen_nodaug = get_generator(images, **nodaug_params) batch_gen_nodaug = batch_generator(image_gen_nodaug, images, labels, FLAGS.batch_size, aug_per_im=1, shuffle=False) # Outputs if FLAGS.output_dir == '-1': FLAGS.output_dir = os.path.dirname(FLAGS.model) output_hdf5 = h5py.File( os.path.join(FLAGS.output_dir, FLAGS.output_mse_matrix_hdf5), 'w') output_pickle = os.path.join(FLAGS.output_dir, FLAGS.output_pickle) df_init_idx = 0 df = pd.DataFrame() # Iterate over the layers for layer_idx, layer_name in enumerate(layers): # Reload the model if layer_idx > 0: K.clear_session() model = load_model(model_filename) layer = model.get_layer(layer_name) # Rename input layer if re.compile('.*input.*').match(layer_name): layer_name = 'input' hdf5_layer = output_hdf5.create_group(layer_name) activation_function = K.function( [model.input, K.learning_phase()], [layer.output]) print('\nComputing pairwise similarity at layer {}'.format(layer_name)) # Compute activations of original data (without augmentation) a_nodaug_da = get_activations(activation_function, batch_gen_nodaug) a_nodaug_da = da.squeeze(a_nodaug_da) a_nodaug_da = da.rechunk(a_nodaug_da, (chunk_size, ) + (a_nodaug_da.shape[1:])) dim_activations = a_nodaug_da.shape[1] # Comute matrix of similarities r = da.reshape(da.sum(da.square(a_nodaug_da), axis=1), (-1, 1)) mse_matrix = (r - 2 * da.dot(a_nodaug_da, da.transpose(a_nodaug_da)) \ + da.transpose(r)) / dim_activations # Compute activations with augmentation a_daug_da = get_activations(activation_function, batch_gen_daug) a_daug_da = da.rechunk(a_daug_da, (chunk_size, dim_activations, 1)) # Compute similarity of augmentations with respect to the # activations of the original data a_nodaug_da = da.repeat(da.reshape(a_nodaug_da, a_nodaug_da.shape + (1, )), repeats=n_daug_rep, axis=2) a_nodaug_da = da.rechunk(a_nodaug_da, (chunk_size, dim_activations, 1)) mse_daug = da.mean(da.square(a_nodaug_da - a_daug_da), axis=1) # Compute invariance score mse_sum = da.repeat(da.reshape(da.sum(mse_matrix, axis=1), (n_images, 1)), repeats=n_daug_rep, axis=1) mse_sum = da.rechunk(mse_sum, (chunk_size, 1)) invariance = 1 - n_images * da.divide(mse_daug, mse_sum) print('Dimensionality activations: {}x{}x{}'.format( n_images, dim_activations, n_daug_rep)) # Store HDF5 file if FLAGS.output_mse_matrix_hdf5: mse_matrix_ds = hdf5_layer.create_dataset( 'mse_matrix', shape=mse_matrix.shape, chunks=mse_matrix.chunksize, dtype=K.floatx()) mse_daug_ds = hdf5_layer.create_dataset('mse_daug', shape=mse_daug.shape, chunks=mse_daug.chunksize, dtype=K.floatx()) invariance_ds = hdf5_layer.create_dataset( 'invariance', shape=invariance.shape, chunks=invariance.chunksize, dtype=K.floatx()) time_init = time() with ProgressBar(dt=1): da.store([mse_matrix, mse_daug, invariance], [mse_matrix_ds, mse_daug_ds, invariance_ds]) time_end = time() print('Elapsed time: {}'.format(time_end - time_init)) invariance = np.ravel( np.asarray(output_hdf5[layer_name]['invariance'])) else: time_init = time() invariance = da.ravel(invariance).compute() time_end = time() print('Elapsed time: {}'.format(time_end - time_init)) # Update pandas data frame for plotting df_end_idx = df_init_idx + n_images * n_daug_rep d = pd.DataFrame( { 'Layer': layer_name, 'sample': np.repeat(np.arange(n_images), n_daug_rep), 'n_daug': np.tile(np.arange(n_daug_rep), n_images), 'invariance': invariance }, index=np.arange(df_init_idx, df_end_idx).tolist()) df = df.append(d) df_init_idx += df_end_idx pickle.dump(df, open(output_pickle, 'wb')) output_hdf5.close()