def imagenet_input(is_training): """Data reader for imagenet. Reads in imagenet data and performs pre-processing on the images. Args: is_training: bool specifying if train or validation dataset is needed. Returns: A batch of images and labels. """ if is_training: dataset = dataset_factory.get_dataset('imagenet', 'train', FLAGS.dataset_dir) else: dataset = dataset_factory.get_dataset('imagenet', 'validation', FLAGS.dataset_dir) provider = slim.dataset_data_provider.DatasetDataProvider( dataset, shuffle=is_training, common_queue_capacity=2 * FLAGS.batch_size, common_queue_min=FLAGS.batch_size) [image, label] = provider.get(['image', 'label']) image_preprocessing_fn = preprocessing_factory.get_preprocessing( 'mobilenet_v1', is_training=is_training) image = image_preprocessing_fn(image, FLAGS.image_size, FLAGS.image_size) images, labels = tf.train.batch(tensors=[image, label], batch_size=FLAGS.batch_size, num_threads=4, capacity=5 * FLAGS.batch_size) return images, labels
def parse_function(filename): image_string = tf.read_file(filename) image_decoded = tf.image.decode_jpeg(image_string, channels=args.num_channel) if args.preprocessing_name and args.preprocessing_name in preprocessing_factory.preprocessing_fn_map: image_preprocessing_fn = preprocessing_factory.get_preprocessing( args.preprocessing_name, is_training=False) image_decoded = image_preprocessing_fn(image_decoded, args.input_size, args.input_size) else: if not args.preprocessing_name: args.preprocessing_name = "base_preprocessing" preprocessing_f = util.get_attr( 'preprocessing.%s' % args.preprocessing_name, "preprocessing_inference") if not preprocessing_f: preprocessing_f = util.get_attr('preprocessing.base_preprocessing', "preprocessing_inference") image_decoded = preprocessing_f(image_decoded, tf.convert_to_tensor(args.input_size), tf.convert_to_tensor(args.input_size), args) return image_decoded
def pre_process(self, example_proto, is_training): features = {"image/encoded": tf.FixedLenFeature((), tf.string, default_value=""), "image/class/label": tf.FixedLenFeature((), tf.int64, default_value=0), 'image/height': tf.FixedLenFeature((), tf.int64, default_value=0), 'image/width': tf.FixedLenFeature((), tf.int64, default_value=0) } parsed_features = tf.parse_single_example(example_proto, features) if self.config.preprocessing_name: image_preprocessing_fn = preprocessing_factory.get_preprocessing(self.config.preprocessing_name, is_training=is_training) image = tf.image.decode_image(parsed_features["image/encoded"], self.config.num_channel) image = tf.clip_by_value( image_preprocessing_fn(image, tf.convert_to_tensor(self.config.input_size), tf.convert_to_tensor(self.config.input_size)), -1, 1.0) else: image = tf.clip_by_value(tf.image.per_image_standardization( tf.image.resize_images(tf.image.decode_jpeg(parsed_features["image/encoded"], self.config.num_channel), [tf.convert_to_tensor(self.config.input_size), tf.convert_to_tensor(self.config.input_size)])), -1., 1.0) if len(parsed_features["image/class/label"].get_shape()) == 0: label = tf.one_hot(parsed_features["image/class/label"], self.config.num_class) else: label = parsed_features["image/class/label"] return image, label
def inference_on_image(bot_id, suffix, setting_id, image_file, network_name='inception_v4', return_labels=1): """ Loads the corresponding model checkpoint, network function and preprocessing routine based on bot_id and network_name, restores the graph and runs it to the prediction enpoint with the image as input :param bot_id: bot_id, used to reference to correct model directory :param image_file: reference to the temporary image file to be classified :param network_name: name of the network type to be used :param return_labels: number of labels to return :return: the top n labels with probabilities, where n = return_labels """ # Get the model path model_path = dirs.get_transfer_model_dir(bot_id+suffix, setting_id) # Get number of classes to predict protobuf_dir = dirs.get_transfer_proto_dir(bot_id, setting_id) number_of_classes = dataset_utils.get_number_of_classes_by_labels(protobuf_dir) # Get the preprocessing and network construction functions preprocessing_fn = preprocessing_factory.get_preprocessing(network_name, is_training=False) network_fn = network_factory.get_network_fn(network_name, number_of_classes) # Process the temporary image file into a Tensor of shape [widht, height, channels] image_tensor = tf.gfile.FastGFile(image_file, 'rb').read() image_tensor = tf.image.decode_image(image_tensor, channels=0) # Perform preprocessing and reshape into [network.default_width, network.default_height, channels] network_default_size = network_fn.default_image_size image_tensor = preprocessing_fn(image_tensor, network_default_size, network_default_size) # Create an input batch of size one from the preprocessed image input_batch = tf.reshape(image_tensor, [1, 299, 299, 3]) # Create the network up to the Predictions Endpoint logits, endpoints = network_fn(input_batch) # Create a Saver() object to restore the network from the last checkpoint restorer = tf.train.Saver() with tf.Session() as sess: tf.global_variables_initializer().run() # Restore the variables of the network from the last checkpoint and run the graph restorer.restore(sess, tf.train.latest_checkpoint(model_path)) sess.run(endpoints) # Get the numpy array of predictions out of the predictions = endpoints['Predictions'].eval()[0] return map_predictions_to_labels(protobuf_dir, predictions, return_labels)
def build_imagenet_graph(path): tf.reset_default_graph() print(path) filename_queue = tf.train.string_input_producer( tf.train.match_filenames_once(path + "/*.jpg"), num_epochs=1, shuffle=False, capacity=100) image_reader = tf.WholeFileReader() image_file_name, image_file = image_reader.read(filename_queue) image = tf.image.decode_jpeg(image_file, channels=3, fancy_upscaling=True) model_name = 'inception_resnet_v2' network_fn = nets_factory.get_network_fn(model_name, is_training=False, num_classes=1001) preprocessing_name = model_name image_preprocessing_fn = preprocessing_factory.get_preprocessing( preprocessing_name, is_training=False) eval_image_size = network_fn.default_image_size image = image_preprocessing_fn(image, eval_image_size, eval_image_size) filenames, images = tf.train.batch([image_file_name, image], batch_size=100, num_threads=2, capacity=500) logits, _ = network_fn(images) variables_to_restore = slim.get_variables_to_restore() predictions = tf.argmax(logits, 1) return filenames, logits, predictions, variables_to_restore
def eval_model(candidate, N, F, save_dir, model_name): print("eval model") tf.logging.set_verbosity(tf.logging.INFO) with tf.Graph().as_default(): tf_global_step = slim.get_or_create_global_step() ###################### # Select the dataset # ###################### dataset = dataset_factory.get_dataset( FLAGS.dataset_name, 'test', FLAGS.dataset_dir) #################### # Select the model # #################### network_fn = nets_factory.get_network_fn( FLAGS.model_name, candidate, N, F, num_classes=(dataset.num_classes - FLAGS.labels_offset), is_training=False) ############################################################## # Create a dataset provider that loads data from the dataset # ############################################################## provider = slim.dataset_data_provider.DatasetDataProvider( dataset, shuffle=False, common_queue_capacity=2 * FLAGS.batch_size, common_queue_min=FLAGS.batch_size) [image, label] = provider.get(['image', 'label']) label -= FLAGS.labels_offset ##################################### # Select the preprocessing function # ##################################### preprocessing_name = FLAGS.preprocessing_name or FLAGS.model_name image_preprocessing_fn = preprocessing_factory.get_preprocessing( preprocessing_name, is_training=False) eval_image_size = network_fn.default_image_size image = image_preprocessing_fn(image, eval_image_size, eval_image_size) FLAGS.batch_size = 100 images, labels = tf.train.batch( [image, label], batch_size=FLAGS.batch_size, num_threads=FLAGS.num_preprocessing_threads, capacity=5 * FLAGS.batch_size) #################### # Define the model # #################### logits, _ = network_fn(images) if FLAGS.moving_average_decay: variable_averages = tf.train.ExponentialMovingAverage( FLAGS.moving_average_decay, tf_global_step) variables_to_restore = variable_averages.variables_to_restore( slim.get_model_variables()) variables_to_restore[tf_global_step.op.name] = tf_global_step else: variables_to_restore = slim.get_variables_to_restore() predictions = tf.argmax(logits, 1) labels = tf.squeeze(labels) # Define the metrics: names_to_values, names_to_updates = slim.metrics.aggregate_metric_map({ 'Accuracy': slim.metrics.streaming_accuracy(predictions, labels), # 'Recall_5': slim.metrics.streaming_recall_at_k( # logits, labels, 5), }) # Print the summaries to screen. for name, value in names_to_values.items(): summary_name = 'eval/%s' % name op = tf.summary.scalar(summary_name, value, collections=[]) op = tf.Print(op, [value], summary_name) tf.add_to_collection(tf.GraphKeys.SUMMARIES, op) # TODO(sguada) use num_epochs=1 if FLAGS.max_num_batches: num_batches = FLAGS.max_num_batches else: # This ensures that we make a single pass over all of the data. num_batches = math.ceil(dataset.num_samples / float(FLAGS.batch_size)) FLAGS.checkpoint_path = FLAGS.train_dir if tf.gfile.IsDirectory(FLAGS.checkpoint_path): checkpoint_path = tf.train.latest_checkpoint(FLAGS.checkpoint_path) else: checkpoint_path = FLAGS.checkpoint_path tf.logging.info('Evaluating %s' % checkpoint_path) final_op = [names_to_values['Accuracy']] #top1 accuracy to return config = tf.ConfigProto() config.gpu_options.allow_growth = True #time.sleep(60) pl.start() start_time = time.time() a = slim.evaluation.evaluate_once( master=FLAGS.master, checkpoint_path=checkpoint_path, logdir=FLAGS.eval_dir, session_config=config, num_evals=num_batches, eval_op=list(names_to_updates.values()), final_op = final_op, variables_to_restore=variables_to_restore) duration = time.time() - start_time pl.stop() data_list = pl.getDataTrace(nodeName='module/gpu', valType='power') pickle.dump(data_list, open(os.path.join(save_dir, model_name + '_data_list_final_{}_{}.pkl'.format(N,F)),'wb')) power_list = data_list[1] time_list = data_list[0] start, end = get_start_end(power_list) integration_time = time_list[end] - time_list[start] integration_energy = integrate_power(power_list, time_list, start, end) return integration_time, integration_energy
def train_model(candidate, N, F): print("train model") print(FLAGS.dataset_name) if not FLAGS.dataset_dir: raise ValueError('You must supply the dataset directory with --dataset_dir') tf.logging.set_verbosity(tf.logging.INFO) with tf.Graph().as_default(): ####################### # Config model_deploy # ####################### deploy_config = model_deploy.DeploymentConfig( num_clones=FLAGS.num_clones, clone_on_cpu=FLAGS.clone_on_cpu, replica_id=FLAGS.task, num_replicas=FLAGS.worker_replicas, num_ps_tasks=FLAGS.num_ps_tasks) # Create global_step with tf.device(deploy_config.variables_device()): global_step = slim.create_global_step() ###################### # Select the dataset # ###################### dataset = dataset_factory.get_dataset( FLAGS.dataset_name, FLAGS.dataset_split_name, FLAGS.dataset_dir) ###################### # Select the network # ###################### network_fn = nets_factory.get_network_fn( FLAGS.model_name, candidate, N, F, num_classes=(dataset.num_classes - FLAGS.labels_offset), weight_decay=FLAGS.weight_decay, is_training=True) ##################################### # Select the preprocessing function # ##################################### preprocessing_name = FLAGS.preprocessing_name or FLAGS.model_name image_preprocessing_fn = preprocessing_factory.get_preprocessing( preprocessing_name, is_training=True) ############################################################## # Create a dataset provider that loads data from the dataset # ############################################################## with tf.device(deploy_config.inputs_device()): provider = slim.dataset_data_provider.DatasetDataProvider( dataset, num_readers=FLAGS.num_readers, common_queue_capacity=20 * FLAGS.batch_size, common_queue_min=10 * FLAGS.batch_size) [image, label] = provider.get(['image', 'label']) label -= FLAGS.labels_offset train_image_size = FLAGS.train_image_size or network_fn.default_image_size image = image_preprocessing_fn(image, train_image_size, train_image_size) images, labels = tf.train.batch( [image, label], batch_size=FLAGS.batch_size, num_threads=FLAGS.num_preprocessing_threads, capacity=5 * FLAGS.batch_size) labels = slim.one_hot_encoding( labels, dataset.num_classes - FLAGS.labels_offset) batch_queue = slim.prefetch_queue.prefetch_queue( [images, labels], capacity=2 * deploy_config.num_clones) #################### # Define the model # #################### def clone_fn(batch_queue): """Allows data parallelism by creating multiple clones of network_fn.""" images, labels = batch_queue.dequeue() logits, end_points = network_fn(images) ############################# # Specify the loss function # ############################# if 'AuxLogits' in end_points: slim.losses.softmax_cross_entropy( end_points['AuxLogits'], labels, label_smoothing=FLAGS.label_smoothing, weights=0.4, scope='aux_loss') slim.losses.softmax_cross_entropy( logits, labels, label_smoothing=FLAGS.label_smoothing, weights=1.0) return end_points # Gather initial summaries. summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES)) clones = model_deploy.create_clones(deploy_config, clone_fn, [batch_queue]) first_clone_scope = deploy_config.clone_scope(0) # Gather update_ops from the first clone. These contain, for example, # the updates for the batch_norm variables created by network_fn. update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS, first_clone_scope) # Add summaries for end_points. end_points = clones[0].outputs for end_point in end_points: x = end_points[end_point] summaries.add(tf.summary.histogram('activations/' + end_point, x)) summaries.add(tf.summary.scalar('sparsity/' + end_point, tf.nn.zero_fraction(x))) # Add summaries for losses. for loss in tf.get_collection(tf.GraphKeys.LOSSES, first_clone_scope): summaries.add(tf.summary.scalar('losses/%s' % loss.op.name, loss)) # Add summaries for variables. for variable in slim.get_model_variables(): summaries.add(tf.summary.histogram(variable.op.name, variable)) ################################# # Configure the moving averages # ################################# if FLAGS.moving_average_decay: moving_average_variables = slim.get_model_variables() variable_averages = tf.train.ExponentialMovingAverage( FLAGS.moving_average_decay, global_step) else: moving_average_variables, variable_averages = None, None ######################################### # Configure the optimization procedure. # ######################################### with tf.device(deploy_config.optimizer_device()): learning_rate = _configure_learning_rate(dataset.num_samples, global_step) optimizer = _configure_optimizer(learning_rate) summaries.add(tf.summary.scalar('learning_rate', learning_rate)) if FLAGS.sync_replicas: # If sync_replicas is enabled, the averaging will be done in the chief # queue runner. optimizer = tf.train.SyncReplicasOptimizer( opt=optimizer, replicas_to_aggregate=FLAGS.replicas_to_aggregate, total_num_replicas=FLAGS.worker_replicas, variable_averages=variable_averages, variables_to_average=moving_average_variables) elif FLAGS.moving_average_decay: # Update ops executed locally by trainer. update_ops.append(variable_averages.apply(moving_average_variables)) # Variables to train. variables_to_train = _get_variables_to_train() # and returns a train_tensor and summary_op total_loss, clones_gradients = model_deploy.optimize_clones( clones, optimizer, var_list=variables_to_train) # Add total_loss to summary. summaries.add(tf.summary.scalar('total_loss', total_loss)) # Create gradient updates. grad_updates = optimizer.apply_gradients(clones_gradients, global_step=global_step) update_ops.append(grad_updates) update_op = tf.group(*update_ops) with tf.control_dependencies([update_op]): train_tensor = tf.identity(total_loss, name='train_op') # Add the summaries from the first clone. These contain the summaries # created by model_fn and either optimize_clones() or _gather_clone_loss(). summaries |= set(tf.get_collection(tf.GraphKeys.SUMMARIES, first_clone_scope)) # Merge all summaries together. summary_op = tf.summary.merge(list(summaries), name='summary_op') ########################### # Kicks off the training. # ########################### slim.learning.train( train_tensor, logdir=FLAGS.train_dir, master=FLAGS.master, is_chief=(FLAGS.task == 0), init_fn=_get_init_fn(), summary_op=summary_op, number_of_steps=FLAGS.max_number_of_steps, log_every_n_steps=FLAGS.log_every_n_steps, save_summaries_secs=FLAGS.save_summaries_secs, save_interval_secs=FLAGS.save_interval_secs, sync_optimizer=optimizer if FLAGS.sync_replicas else None)
def main(_): if not FLAGS.dataset_dir: raise ValueError( 'You must supply the dataset directory with --dataset_dir') tf.logging.set_verbosity(tf.logging.INFO) with tf.Graph().as_default(): tf_global_step = slim.get_or_create_global_step() ###################### # Select the dataset # ###################### dataset = dataset_factory.get_dataset(FLAGS.dataset_name, FLAGS.dataset_split_name, FLAGS.dataset_dir) #################### # Select the model # #################### network_fn = nets_factory.get_network_fn( FLAGS.model_name, num_classes=(dataset.num_classes - FLAGS.labels_offset), is_training=False) ############################################################## # Create a dataset provider that loads data from the dataset # ############################################################## provider = slim.dataset_data_provider.DatasetDataProvider( dataset, shuffle=False, common_queue_capacity=2 * FLAGS.batch_size, common_queue_min=FLAGS.batch_size) [image, label] = provider.get(['image', 'label']) label -= FLAGS.labels_offset ##################################### # Select the preprocessing function # ##################################### preprocessing_name = FLAGS.preprocessing_name or FLAGS.model_name image_preprocessing_fn = preprocessing_factory.get_preprocessing( preprocessing_name, is_training=False) eval_image_size = FLAGS.eval_image_size or network_fn.default_image_size image = image_preprocessing_fn(image, eval_image_size, eval_image_size) images, labels = tf.train.batch( [image, label], batch_size=FLAGS.batch_size, num_threads=FLAGS.num_preprocessing_threads, capacity=5 * FLAGS.batch_size) #################### # Define the model # #################### logits, _ = network_fn(images) if FLAGS.moving_average_decay: variable_averages = tf.train.ExponentialMovingAverage( FLAGS.moving_average_decay, tf_global_step) variables_to_restore = variable_averages.variables_to_restore( slim.get_model_variables()) variables_to_restore[tf_global_step.op.name] = tf_global_step else: variables_to_restore = slim.get_variables_to_restore() predictions = tf.argmax(logits, 1) labels = tf.squeeze(labels) # Define the metrics: names_to_values, names_to_updates = slim.metrics.aggregate_metric_map({ 'Accuracy': slim.metrics.streaming_accuracy(predictions, labels), 'Recall_5': slim.metrics.streaming_recall_at_k(logits, labels, 5), }) # Print the summaries to screen. for name, value in names_to_values.items(): summary_name = 'eval/%s' % name op = tf.summary.scalar(summary_name, value, collections=[]) op = tf.Print(op, [value], summary_name) tf.add_to_collection(tf.GraphKeys.SUMMARIES, op) # TODO(sguada) use num_epochs=1 if FLAGS.max_num_batches: num_batches = FLAGS.max_num_batches else: # This ensures that we make a single pass over all of the data. num_batches = math.ceil(dataset.num_samples / float(FLAGS.batch_size)) if tf.gfile.IsDirectory(FLAGS.checkpoint_path): checkpoint_path = tf.train.latest_checkpoint(FLAGS.checkpoint_path) else: checkpoint_path = FLAGS.checkpoint_path tf.logging.info('Evaluating %s' % checkpoint_path) slim.evaluation.evaluate_once( master=FLAGS.master, checkpoint_path=checkpoint_path, logdir=FLAGS.eval_dir, num_evals=num_batches, eval_op=list(names_to_updates.values()), variables_to_restore=variables_to_restore)
def run_one_cnn(args): num_inj_per_layer = 1 # Obtain target layer information layer_dict = {} with open(args.layer_path) as layer_csv: layer_reader = csv.reader(layer_csv, delimiter='\t') for row in layer_reader: layer_dict[row[0]] = row[1] print(layer_dict) layer_names = [ layer_dict["Input tensor name:"], layer_dict["Weight tensor name:"], layer_dict["Output tensor name:"] ] layer_dims = [ str2list(layer_dict["Input shape:"]), str2list(layer_dict["Weight shape:"]), str2list(layer_dict["Output shape:"]) ] layer_stride = int(layer_dict["Layer stride:"]) layer_padding = layer_dict["Layer padding:"] quant_min_max = str2list(layer_dict["Quant min max:"], True) # Obtain delta delta_set, inj_pos = delta_generator(args.network, args.precision, args.inj_type, [layer_names[2]], layer_dims[2], quant_min_max) # Then start running injection tf.reset_default_graph() (x_train, y_train), (x_test, y_test) = cifar10.load_data() np_image = x_test[args.image_id] image_label = y_test[args.image_id] image = tf.placeholder(tf.uint8, shape=[32, 32, 3]) if 'ic' in args.network or 'mb' in args.network: pre_fn = get_preprocessing('inception', is_training=False) else: pre_fn = get_preprocessing('resnet_v1_50', is_training=False) post_image = pre_fn(image, 224, 224) images = tf.expand_dims(post_image, 0) # Cast the image for fp16 if 'fp16' in args.precision: images = tf.cast(images, tf.float16) # Deploy the network arg_scope_fn = get_arg_scope(args.network, args.precision) network_fn = get_network(args.network, args.precision) # Need to feed in injection relating arguments with tf.contrib.slim.arg_scope(arg_scope_fn()): net, endpoints = network_fn(images, num_classes=10, is_training=False, inj_type=get_network_inj_type( args.precision, args.inj_type), inj_layer=[layer_names[2]], inj_pos=inj_pos, quant_min_max=quant_min_max) # Quantize the network if necessary if 'int8' in args.precision: tf.contrib.quantize.create_eval_graph() elif 'int16' in args.precision: tf.contrib.quantize.experimental_create_eval_graph(weight_bits=16, activation_bits=16) # Create saver: For FP16, need extra handling for Logits all_variables = tf.get_collection_ref(tf.GraphKeys.GLOBAL_VARIABLES) if 'fp16' in args.precision: v_list = [ v for v in all_variables if 'dense' not in v.name and 'delta' not in v.name ] elif 'rs' in args.network and 'int' in args.precision: v_list = [ v for v in all_variables if 'delta' not in v.name and 'unit_1/bottleneck_v1/shortcut/act_quant' not in v.name and 'unit_2/bottleneck_v1/conv3/act_quant' not in v.name and 'unit_3/bottleneck_v1/conv3/act_quant' not in v.name and 'unit_4/bottleneck_v1/conv3/act_quant' not in v.name and 'unit_5/bottleneck_v1/conv3/act_quant' not in v.name and 'unit_6/bottleneck_v1/conv3/act_quant' not in v.name ] else: v_list = [v for v in all_variables if 'delta' not in v.name] saver = tf.train.Saver(var_list=v_list) # Create a session and run it with tf.Session() as sess: saver.restore(sess, args.ckpt_path) # For fp16, restore the dense part of the network if 'fp16' in args.precision: dense_var_dict = { 'mb': 'MobilenetV1/Logits/Conv2d_1c_1x1/', 'rs': 'resnet_v1_50/logits/', 'ic': 'InceptionV1/Logits/Conv2d_0c_1x1/' } for variable in all_variables: if 'dense/kernel' in variable.name: var = tf.contrib.framework.load_variable( args.ckpt_path, dense_var_dict[args.network[:2]] + 'weights') sess.run(variable.assign(var[0, 0, :, :])) if 'dense/bias' in variable.name: var = tf.contrib.framework.load_variable( args.ckpt_path, dense_var_dict[args.network[:2]] + 'biases') sess.run(variable.assign(var)) elif 'rs' in args.network and 'int' in args.precision: for variable in all_variables: if 'unit_1/bottleneck_v1/shortcut/act_quant' in variable.name: var = tf.contrib.framework.load_variable( args.ckpt_path, re.sub('act_quant', 'conv_quant', variable.name)) sess.run(variable.assign(var)) if 'bottleneck_v1/conv3/act_quant' in variable.name and 'unit_1' not in variable.name: var = tf.contrib.framework.load_variable( args.ckpt_path, re.sub('act_quant', 'conv_quant', variable.name)) sess.run(variable.assign(var)) # If we inject to input/weights or local controls if 'INPUT' in args.inj_type or 'WEIGHT' in args.inj_type or 'RD_BFLIP' in args.inj_type: layer = layer_names[2] delta_np = np.zeros(shape=layer_dims[2], dtype=np.float32) scope_string = '' if 'mb' in args.network: scope_string = 'MobilenetV1' elif 'rs' in args.network: scope_string = layer[:layer.rfind('/')] else: scope_string = layer[layer.find('/') + 1:layer.rfind('/')] with tf.variable_scope(scope_string, reuse=True): sess.run( tf.get_variable('delta_{}'.format( re.sub('\/', '_', layer[layer.rfind('/') + 1:])), trainable=False).assign(delta_np)) # Get this layer's weight weight_tensor = tf.get_default_graph().get_tensor_by_name( layer_names[1]) if 'RD_BFLIP' in args.inj_type: # In RD_BFLIP, input tensor means this layer's output input_tensor = tf.get_default_graph().get_tensor_by_name( layer + '/Conv2D:0') else: # Get this layer's input for injecting to input or psum input_tensor = tf.get_default_graph().get_tensor_by_name( layer_names[0]) # Run the golden network wt, inp = sess.run([weight_tensor, input_tensor], feed_dict={image: np_image}) if 'INPUT' in args.inj_type or 'RD_BFLIP' in args.inj_type: if 'INPUT' in args.inj_type or 'RD_BFLIP' in args.inj_type: t_a, t_b, t_c, t_d = inp.shape else: t_a, t_b, t_c, t_d = layer_dims[2] p_a = np.random.randint(t_a) p_b = np.random.randint(t_b) p_c = np.random.randint(t_c) p_d = np.random.randint(t_d) golden_d = inp[p_a][p_b][p_c][p_d] if 'RD_BFLIP' in args.inj_type: flip_bit, perturb = get_bit_flip_perturbation( args.network, args.precision, golden_d, layer, 'rd_bflip') else: flip_bit, perturb = get_bit_flip_perturbation( args.network, args.precision, golden_d, layer, 'input') inp_perturb = np.zeros(inp.shape) inp_perturb[p_a][p_b][p_c][p_d] = perturb if 'RD_BFLIP' in args.inj_type: delta_perturb = inp_perturb else: delta_perturb = perturb_conv(inp_perturb, wt, layer_stride, layer_padding == 'SAME', layer_dims[2][-1]) else: t_a, t_b, t_c, t_d = wt.shape p_a = np.random.randint(t_a) p_b = np.random.randint(t_b) p_c = np.random.randint(t_c) p_d = np.random.randint(t_d) golden_d = wt[p_a][p_b][p_c][p_d] flip_bit, perturb = get_bit_flip_perturbation( args.network, args.precision, golden_d, layer, 'weight') wt_perturb = np.zeros(wt.shape) wt_perturb[p_a][p_b][p_c][p_d] = perturb delta_perturb = perturb_conv(inp, wt_perturb, layer_stride, layer_padding == 'SAME', layer_dims[2][-1]) # If we only inject to 16W or 16C we need to reconfig the delta if '16' in args.inj_type and 'PSUM' not in args.inj_type: _, d_h, d_w, d_c = delta_perturb.shape delta_16 = np.zeros(delta_perturb.shape) pos_16 = [] # Injecting to input: 16 neurons in 16 channels at a time if 'INPUT' in args.inj_type: weight_d = layer_dims[1] pad_type = layer_padding if pad_type is 'VALID': pad = 0 else: pad = weight_d // 2 stride = layer_stride start_h = np.random.randint( max(0, (p_b + pad) // stride - weight_d + 1), min((p_b + pad) // stride + 1, d_h)) start_w = np.random.randint( max(0, (p_c + pad) // stride - weight_d + 1), min((p_c + pad) // stride + 1, d_w)) start_c = np.random.randint(d_c // 16) for i in range(16): delta_16[0][start_h][start_w][ start_c + i] = delta_perturb[0][start_h][start_w][start_c + i] pos_16.append(start_h) pos_16.append(start_w) pos_16.append(16 * start_c + i) # Injecting to weight: 16 W at a time else: start_p = np.random.randint(d_h * d_w // 16) # It will only affect neurons in p_d start_c = p_d for i in range(16): # If it doesn't have 16, then just break if start_p * 16 + i >= d_h * d_w: break elem_h = (start_p * 16 + i) // d_w elem_w = (start_p * 16 + i) % d_w delta_16[0][elem_h][elem_w][start_c] = delta_perturb[ 0][elem_h][elem_w][start_c] pos_16.append(elem_h) pos_16.append(elem_w) pos_16.append(start_c) delta_perturb = delta_16 # Assign delta_perturb back to the variable delta with tf.variable_scope(scope_string, reuse=True): sess.run( tf.get_variable('delta_{}'.format( re.sub('\/', '_', layer[layer.rfind('/') + 1:])), trainable=False).assign(delta_perturb)) # Then run the network again if 'mb' in args.network or 'ic' in args.network: lgt, prd = sess.run( [endpoints['Logits'][0], endpoints['Predictions'][0]], feed_dict={image: np_image}) else: lgt, prd = sess.run([ endpoints['resnet_v1_50/spatial_squeeze'][0], endpoints['predictions'][0] ], feed_dict={image: np_image}) # Get a sorted label network_labels = np.argsort(lgt)[::-1] # If we inject to neuron directly else: layer = layer_names[2] delta_np = np.zeros(shape=layer_dims[2], dtype=np.float32) for n_j in range(num_inj_per_layer): layer_pos = inj_pos[layer] delta_np[0][layer_pos[n_j][0]][layer_pos[n_j][1]][ layer_pos[n_j][2]] = delta_set[layer][n_j] scope_string = '' if 'mb' in args.network: scope_string = 'MobilenetV1' elif 'rs' in args.network: scope_string = layer[:layer.rfind('/')] else: scope_string = layer[layer.find('/') + 1:layer.rfind('/')] with tf.variable_scope(scope_string, reuse=True): sess.run( tf.get_variable('delta_{}'.format( re.sub('\/', '_', layer[layer.rfind('/') + 1:])), trainable=False).assign(delta_np)) op_list = [] for node in tf.get_default_graph().as_graph_def().node: if 'Conv2D' in node.name: op_list.append(node.name + ':0') # Run the network if 'mb' in args.network or 'ic' in args.network: ops, lgt, prd = sess.run([ op_list, endpoints['Logits'][0], endpoints['Predictions'][0] ], feed_dict={image: np_image}) else: ops, lgt, prd = sess.run([ op_list, endpoints['resnet_v1_50/spatial_squeeze'][0], endpoints['predictions'][0] ], feed_dict={image: np_image}) # Get a sorted label network_labels = np.argsort(lgt)[::-1] print( "After injection, the network label becomes {}".format(network_labels))
def run_transfer_learning(root_model_dir, bot_model_dir, protobuf_dir, model_name='inception_v4', dataset_split_name='train', dataset_name='bot', checkpoint_exclude_scopes=None, trainable_scopes=None, max_train_time_sec=None, max_number_of_steps=None, log_every_n_steps=None, save_summaries_secs=None, optimization_params=None): """ Starts the transfer learning of a model in a tensorflow session :param root_model_dir: Directory containing the root models pretrained checkpoint files :param bot_model_dir: Directory where the transfer learned model's checkpoint files are written to :param protobuf_dir: Directory for the dataset factory to load the bot's training data from :param model_name: name of the network model for the net factory to provide the correct network and preprocesing fn :param dataset_split_name: 'train' or 'validation' :param dataset_name: triggers the dataset factory to load a bot dataset :param checkpoint_exclude_scopes: Layers to exclude when restoring the models variables :param trainable_scopes: Layers to train from the restored model :param max_train_time_sec: time boundary to stop training after in seconds :param max_number_of_steps: maximum number of steps to run :param log_every_n_steps: write a log after every nth optimization step :param save_summaries_secs: save summaries to disc every n seconds :param optimization_params: parameters for the optimization :return: """ if not optimization_params: optimization_params = OPTIMIZATION_PARAMS if not max_number_of_steps: max_number_of_steps = _MAX_NUMBER_OF_STEPS if not checkpoint_exclude_scopes: checkpoint_exclude_scopes = _CHECKPOINT_EXCLUDE_SCOPES if not trainable_scopes: trainable_scopes = _TRAINABLE_SCOPES if not max_train_time_sec: max_train_time_sec = _MAX_TRAIN_TIME_SECONDS if not log_every_n_steps: log_every_n_steps = _LOG_EVERY_N_STEPS if not save_summaries_secs: save_summaries_secs = _SAVE_SUMMARRIES_SECS tf.logging.set_verbosity(tf.logging.INFO) with tf.Graph().as_default(): ####################### # Config model_deploy # ####################### deploy_config = model_deploy.DeploymentConfig( num_clones=_NUM_CLONES, clone_on_cpu=_CLONE_ON_CPU, replica_id=_TASK, num_replicas=_WORKER_REPLICAS, num_ps_tasks=_NUM_PS_TASKS) # Create global_step with tf.device(deploy_config.variables_device()): global_step = slim.create_global_step() ###################### # Select the dataset # ###################### dataset = dataset_factory.get_dataset( dataset_name, dataset_split_name, protobuf_dir) ###################### # Select the network # ###################### network_fn = nets_factory.get_network_fn( model_name, num_classes=(dataset.num_classes - _LABELS_OFFSET), weight_decay=OPTIMIZATION_PARAMS['weight_decay'], is_training=True, dropout_keep_prob=OPTIMIZATION_PARAMS['dropout_keep_prob']) ##################################### # Select the preprocessing function # ##################################### image_preprocessing_fn = preprocessing_factory.get_preprocessing( model_name, is_training=True) ############################################################## # Create a dataset provider that loads data from the dataset # ############################################################## with tf.device(deploy_config.inputs_device()): provider = slim.dataset_data_provider.DatasetDataProvider( dataset, num_readers=_NUM_READERS, common_queue_capacity=20 * _BATCH_SIZE, common_queue_min=10 * _BATCH_SIZE) [image, label] = provider.get(['image', 'label']) label -= _LABELS_OFFSET train_image_size = network_fn.default_image_size image = image_preprocessing_fn(image, train_image_size, train_image_size) images, labels = tf.train.batch( [image, label], batch_size=_BATCH_SIZE, num_threads=_NUM_PREPROCESSING_THREADS, capacity=5 * _BATCH_SIZE) labels = slim.one_hot_encoding( labels, dataset.num_classes - _LABELS_OFFSET) batch_queue = slim.prefetch_queue.prefetch_queue( [images, labels], capacity=2 * deploy_config.num_clones) #################### # Define the model # #################### def clone_fn(batch_queue): """Allows data parallelism by creating multiple clones of network_fn.""" images, labels = batch_queue.dequeue() logits, end_points = network_fn(images) ############################# # Specify the loss function # ############################# if 'AuxLogits' in end_points: tf.losses.softmax_cross_entropy( logits=end_points['AuxLogits'], onehot_labels=labels, label_smoothing=_LABEL_SMOOTHING, weights=0.4, scope='aux_loss') tf.losses.softmax_cross_entropy( logits=logits, onehot_labels=labels, label_smoothing=_LABEL_SMOOTHING, weights=1.0) return end_points # Gather initial summaries. summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES)) clones = model_deploy.create_clones(deploy_config, clone_fn, [batch_queue]) first_clone_scope = deploy_config.clone_scope(0) # Gather update_ops from the first clone. These contain, for example, # the updates for the batch_norm variables created by network_fn. update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS, first_clone_scope) # Add summaries for end_points. end_points = clones[0].outputs for end_point in end_points: x = end_points[end_point] summaries.add(tf.summary.histogram('activations/' + end_point, x)) summaries.add(tf.summary.scalar('sparsity/' + end_point, tf.nn.zero_fraction(x))) # Add summaries for losses. for loss in tf.get_collection(tf.GraphKeys.LOSSES, first_clone_scope): summaries.add(tf.summary.scalar('losses/%s' % loss.op.name, loss)) # Add summaries for variables. for variable in slim.get_model_variables(): summaries.add(tf.summary.histogram(variable.op.name, variable)) ################################# # Configure the moving averages # ################################# if OPTIMIZATION_PARAMS['moving_average_decay']: moving_average_variables = slim.get_model_variables() variable_averages = tf.train.ExponentialMovingAverage( OPTIMIZATION_PARAMS['moving_average_decay'], global_step) else: moving_average_variables, variable_averages = None, None ######################################### # Configure the optimization procedure. # ######################################### with tf.device(deploy_config.optimizer_device()): learning_rate = _configure_learning_rate(dataset.num_samples, global_step) optimizer = _configure_optimizer(learning_rate) summaries.add(tf.summary.scalar('learning_rate', learning_rate)) if _SYNC_REPLICAS: # If sync_replicas is enabled, the averaging will be done in the chief # queue runner. optimizer = tf.train.SyncReplicasOptimizer( opt=optimizer, replicas_to_aggregate=_REPLICAS_TO_AGGREGATE, variable_averages=variable_averages, variables_to_average=moving_average_variables, replica_id=tf.constant(_TASK, tf.int32, shape=()), total_num_replicas=_WORKER_REPLICAS) elif OPTIMIZATION_PARAMS['moving_average_decay']: # Update ops executed locally by trainer. update_ops.append(variable_averages.apply(moving_average_variables)) # Variables to train. variables_to_train = _get_variables_to_train(trainable_scopes) # and returns a train_tensor and summary_op total_loss, clones_gradients = model_deploy.optimize_clones( clones, optimizer, var_list=variables_to_train) # Add total_loss to summary. summaries.add(tf.summary.scalar('total_loss', total_loss)) # Create gradient updates. grad_updates = optimizer.apply_gradients(clones_gradients, global_step=global_step) update_ops.append(grad_updates) update_op = tf.group(*update_ops) train_tensor = control_flow_ops.with_dependencies([update_op], total_loss, name='train_op') # Add the summaries from the first clone. These contain the summaries # created by model_fn and either optimize_clones() or _gather_clone_loss(). summaries |= set(tf.get_collection(tf.GraphKeys.SUMMARIES, first_clone_scope)) # Merge all summaries together. summary_op = tf.summary.merge(list(summaries), name='summary_op') ########################### # Kicks off the training. # ########################### slim.learning.train( train_tensor, logdir=bot_model_dir, train_step_fn=train_step, # Manually added a custom train step to stop after max_time train_step_kwargs=_train_step_kwargs(logdir=bot_model_dir, max_train_time_seconds=max_train_time_sec), master=_MASTER, is_chief=(_TASK == 0), init_fn=_get_init_fn(root_model_dir, bot_model_dir, checkpoint_exclude_scopes), summary_op=summary_op, # number_of_steps=max_number_of_steps, log_every_n_steps=log_every_n_steps, save_summaries_secs=save_summaries_secs, save_interval_secs=_SAVE_INTERNAL_SECS, sync_optimizer=optimizer if _SYNC_REPLICAS else None)
def RCNN(inputs, proposals, options, is_training=True): """Runs RCNN model on the `inputs`. Args: inputs: Input image, a [batch, height, width, 3] uint8 tensor. The pixel values are in the range of [0, 255]. proposals: Boxes used to crop the image features, using normalized coordinates. It should be a [batch, max_num_proposals, 4] float tensor denoting [y1, x1, y2, x2]. options: A fast_rcnn_pb2.FastRCNN proto. is_training: If true, the model shall be executed in training mode. Returns: A [batch, max_num_proposals, feature_dims] tensor. Raises: ValueError if options is invalid. """ if not isinstance(options, rcnn_pb2.RCNN): raise ValueError('The options has to be a rcnn_pb2.RCNN proto!') if inputs.dtype != tf.uint8: raise ValueError('The inputs has to be a tf.uint8 tensor.') net_fn = nets_factory.get_network_fn(name=options.feature_extractor_name, num_classes=1001) default_image_size = getattr(net_fn, 'default_image_size', 224) # Preprocess image. preprocess_fn = preprocessing_factory.get_preprocessing( options.feature_extractor_name, is_training=False) inputs = preprocess_fn(inputs, output_height=None, output_width=None, crop_image=False) # Crop and resize images. batch = proposals.shape[0] max_num_proposals = tf.shape(proposals)[1] box_ind = tf.expand_dims(tf.range(batch), axis=-1) box_ind = tf.tile(box_ind, [1, max_num_proposals]) cropped_inputs = tf.image.crop_and_resize( inputs, boxes=tf.reshape(proposals, [-1, 4]), box_ind=tf.reshape(box_ind, [-1]), crop_size=[default_image_size, default_image_size]) # Run CNN. _, end_points = net_fn(cropped_inputs) outputs = end_points[options.feature_extractor_endpoint] outputs = tf.reshape(outputs, [batch, max_num_proposals, -1]) init_fn = slim.assign_from_checkpoint_fn( options.feature_extractor_checkpoint, slim.get_model_variables(options.feature_extractor_scope)) def _init_from_ckpt_fn(_, sess): return init_fn(sess) return outputs, _init_from_ckpt_fn
def eval(bot_id, bot_suffix, setting_id=None, dataset_split='train', dataset_name='bot', model_name='inception_v4', preprocessing=None, moving_average_decay=None, tf_master=''): full_id = bot_id + bot_suffix if setting_id: protobuf_dir = dirs.get_transfer_proto_dir(bot_id, setting_id) else: protobuf_dir = dirs.get_protobuf_dir(bot_id) _check_dir(protobuf_dir) print("READIND FROM %s" % (protobuf_dir)) performance_data_dir = dirs.get_performance_data_dir(bot_id) # if os.listdir(performance_data_dir): # raise ValueError('%s is not empty' % performance_data_dir) tf.logging.set_verbosity(tf.logging.INFO) with tf.Graph().as_default(): tf_global_step = slim.get_or_create_global_step() ###################### # Select the dataset # ###################### dataset = dataset_factory.get_dataset(dataset_name, dataset_split, protobuf_dir) #################### # Select the model # #################### network_fn = nets_factory.get_network_fn( model_name, num_classes=(dataset.num_classes - LABELS_OFFSET), is_training=False) ############################################################## # Create a dataset provider that loads data from the dataset # ############################################################## provider = slim.dataset_data_provider.DatasetDataProvider( dataset, shuffle=False, common_queue_capacity=2 * BATCH_SIZE, common_queue_min=BATCH_SIZE) [image, label] = provider.get(['image', 'label']) label -= LABELS_OFFSET ##################################### # Select the preprocessing function # ##################################### preprocessing_name = preprocessing or model_name image_preprocessing_fn = preprocessing_factory.get_preprocessing( preprocessing_name, is_training=False) eval_image_size = EVAL_IMAGE_SIZE or network_fn.default_image_size image = image_preprocessing_fn(image, eval_image_size, eval_image_size) images, labels = tf.train.batch([image, label], batch_size=BATCH_SIZE, num_threads=NUM_THREADS, capacity=5 * BATCH_SIZE) #################### # Define the model # #################### logits, _ = network_fn(images) if moving_average_decay: variable_averages = tf.train.ExponentialMovingAverage( moving_average_decay, tf_global_step) variables_to_restore = variable_averages.variables_to_restore( slim.get_model_variables()) variables_to_restore[tf_global_step.op.name] = tf_global_step else: variables_to_restore = slim.get_variables_to_restore() predictions = tf.argmax(logits, 1) labels = tf.squeeze(labels) # Define the metrics: names_to_values, names_to_updates = slim.metrics.aggregate_metric_map({ 'Accuracy': slim.metrics.streaming_accuracy(predictions, labels), 'Recall_5': slim.metrics.streaming_recall_at_k(logits, labels, 5), }) # Print the summaries to screen. for name, value in names_to_values.items(): summary_name = 'eval/%s' % name op = tf.summary.scalar(summary_name, value, collections=[]) op = tf.Print(op, [value], summary_name) tf.add_to_collection(tf.GraphKeys.SUMMARIES, op) # TODO(sguada) use num_epochs=1 if MAX_NUM_BATCHES: num_batches = MAX_NUM_BATCHES else: # This ensures that we make a single pass over all of the data. num_batches = math.ceil(dataset.num_samples / float(BATCH_SIZE)) print(dataset.num_samples) print(dataset.num_classes)