def setUp(self): self.inp = (np.random.randn(10*10*51*51) .reshape(10,10,51,51)) self.keras_model = keras.models.Sequential() conv_layer = keras.layers.convolutional.Conv2D( filters=2, kernel_size=(4,4), strides=(2,2), activation="relu", input_shape=(10,51,51), data_format="channels_first") self.keras_model.add(conv_layer) self.keras_model.add(keras.layers.pooling.MaxPooling2D( pool_size=(4,4), strides=(2,2), data_format="channels_first")) self.keras_model.add(keras.layers.pooling.AveragePooling2D( pool_size=(4,4), strides=(2,2), data_format="channels_first")) self.keras_model.add(keras.layers.Flatten()) self.keras_model.add(keras.layers.Dense(output_dim=1)) self.keras_model.add(keras.layers.core.Activation("sigmoid")) self.keras_model.compile(loss="mse", optimizer="sgd") self.keras_output_fprop_func = compile_func( [self.keras_model.layers[0].input, K.learning_phase()], self.keras_model.layers[-1].output) grad = tf.gradients(tf.reduce_sum( self.keras_model.layers[-2].output[:,0]), [self.keras_model.layers[0].input])[0] self.grad_func = compile_func( [self.keras_model.layers[0].input, K.learning_phase()], grad) self.saved_file_path = "conv2model_channelsfirst.h5" if (os.path.isfile(self.saved_file_path)): os.remove(self.saved_file_path) self.keras_model.save(self.saved_file_path)
def main(): args = get_args(__doc__) with get_session() as sess, get_device(): tf.set_random_seed(1234) np.random.seed(1234) model = load_model(args.modelconf) learn_opts = load_learn_opts(args.learnconf) mnist_data = load_mnist() train_feeder = Feeder({ model.view.x_bhh: FeedArray(mnist_data.x_train.reshape((-1, 28, 28))), model.view.y_b: FeedArray(mnist_data.y_train), K.learning_phase(): FeedRandomStream(lambda b: 1) }, batch_size=learn_opts.batch_size ) valid_feeder = Feeder({ model.view.x_bhh: FeedArray(mnist_data.x_valid.reshape((-1, 28, 28))), model.view.y_b: FeedArray(mnist_data.y_valid), K.learning_phase(): FeedRandomStream(lambda b: 0) }, batch_size=learn_opts.batch_size ) optimizer = tf.train.AdamOptimizer(learning_rate=1e-4) callbacks = [DefaultLogger(['loss', 'err']), DeepDashboardLogger( learn_opts.exp_id, [RawLogger('out.log', 'Log', ['loss', 'err']), CSVLogger('loss.csv', 'Loss', ['loss']), CSVLogger('err.csv', 'Classification Error', ['err'])] ), EarlyStopping('loss', learn_opts.patience), ModelCheckpoint(os.path.join(os.getenv('MYSHKIN_CHECKPOINTDIR', '.'), learn_opts.exp_id), 'loss', verbose=True)] if not args.nodash: call(["open", "http://localhost/deep-dashboard/?id={:s}".format(learn_opts.exp_id)]) else: print "experiment id: {:s}".format(learn_opts.exp_id) fit(model, optimizer, train_feeder, valid_feeder, sess, callbacks=callbacks, n_epochs=learn_opts.n_epochs)
def __init__(self, replay_filename, group_name, model_filename=''): # Set learning phase to TEST self.learning_phase = TEST_MODE # If not informed, defaults to '_model' suffix if model_filename == '': model_filename = '{}_model.h5'.format(group_name) # Loads Keras model self.model = load_model(model_filename) # Loads ReplayData file self.replay_data = h5py.File('{}'.format(replay_filename), 'r') self.group_name = group_name self.group = self.replay_data[self.group_name] # Retrieves some basic information from the replay data self.inputs = self.group['inputs'][:] self.targets = self.group['targets'][:] self.n_epochs = self.group.attrs['n_epochs'] self.n_layers = self.group.attrs['n_layers'] # Retrieves weights as a list, each element being one epoch self.weights = self._retrieve_weights() # Gets Tensors for the weights in the same order as the layers # Keras' model.weights returns the Tensors in a different order! self._model_weights = [w for layer in self.model.layers for w in layer.weights] ### Functions # Keras function to get the outputs, given inputs and weights self._get_output = K.function(inputs=[K.learning_phase()] + self.model.inputs + self._model_weights, outputs=[self.model.layers[-1].output]) # Keras function to get the loss and metrics, given inputs, targets, weights and sample weights self._get_metrics = K.function(inputs=[K.learning_phase()] + self.model.inputs + self.model.targets + self._model_weights + self.model.sample_weights, outputs=[self.model.total_loss] + self.model.metrics_tensors) # Keras function to compute the binary cross entropy, given inputs, targets, weights and sample weights self._get_binary_crossentropy = K.function(inputs=[K.learning_phase()] + self.model.inputs + self.model.targets + self._model_weights + self.model.sample_weights, outputs=[K.binary_crossentropy(self.model.targets[0], self.model.outputs[0])]) # Attributes for the visualizations - Data self._feature_space_data = None self._loss_hist_data = None self._loss_and_metric_data = None self._prob_hist_data = None self._decision_boundary_data = None # Attributes for the visualizations - Plot objects self._feature_space_plot = None self._loss_hist_plot = None self._loss_and_metric_plot = None self._prob_hist_plot = None self._decision_boundary_plot = None
def __init__(self, state_size, num_actuators, args): # remember parameters self.state_size = state_size self.num_actuators = num_actuators self.discount_rate = args.discount_rate self.target_rate = args.target_rate self.noise = args.noise self.noise_scale = args.noise_scale x, u, m, v, q, p, a = self._createLayers(args) # wrappers around computational graph fmu = K.function([K.learning_phase(), x], m) self.mu = lambda x: fmu([0, x]) fP = K.function([K.learning_phase(), x], p) self.P = lambda x: fP([0, x]) fA = K.function([K.learning_phase(), x, u], a) self.A = lambda x, u: fA([0, x, u]) fQ = K.function([K.learning_phase(), x, u], q) self.Q = lambda x, u: fQ([0, x, u]) # main model self.model = Model(input=[x,u], output=q) self.model.summary() if args.optimizer == 'adam': optimizer = Adam(args.learning_rate) elif args.optimizer == 'rmsprop': optimizer = RMSprop(args.learning_rate) else: assert False self.model.compile(optimizer=optimizer, loss='mse') if args.optimizer == 'adam': optimizer = Adam(args.learning_rate) elif args.optimizer == 'rmsprop': optimizer = RMSprop(args.learning_rate) else: assert False self.model.compile(optimizer=optimizer, loss='mse') # another set of layers for target model x, u, m, v, q, p, a = self._createLayers(args) # V() function uses target model weights fV = K.function([K.learning_phase(), x], v) self.V = lambda x: fV([0, x]) # target model is initialized from main model self.target_model = Model(input=[x,u], output=q) self.target_model.set_weights(self.model.get_weights())
def _make_loss_pred_function(self): if not hasattr(self, 'loss_pred_function'): raise RuntimeError('You must compile your model before using it.') if self.loss_pred_function is None: inputs = self._feed_inputs + self._feed_targets + self._feed_sample_weights if self.uses_learning_phase and not isinstance(K.learning_phase(), int): inputs += [K.learning_phase()] # Return loss and metrics, no gradient updates. # Does update the network states. self.loss_pred_function = K.function(inputs, [self.total_loss] + self.outputs, updates=self.state_updates, **self._function_kwargs)
def _make_test_function(self): if not hasattr(self, 'test_function'): raise Exception('You must compile your model before using it.') if self.test_function is None: inputs = self.inputs + self.targets + self.sample_weights if self.uses_learning_phase and not isinstance(K.learning_phase(), int): inputs += [K.learning_phase()] outputs = [self.total_loss] outputs += list(itertools.chain.from_iterable( [model.total_loss] + model.metrics_tensors for model in self.layers)) self.test_function = K.function(inputs, outputs, updates=self.state_updates, **self._function_kwargs)
def visualize(model, datas): for img_name in range(len(datas)): data = [datas[img_name]] layers = ['input', 'conv1', 'pool1', 'conv2', 'pool2', 'conv3', 'pool3', 'conv4', 'pool4'] save_dir = 'visual_pic/%d/'%(img_name+1) if os.path.exists(save_dir) == False: os.mkdir(save_dir) for layer_name in layers: layer = K.function([model.input, K.learning_phase()], [model.get_layer(layer_name).output]) layer_out = layer([data, 0])[0][0] nb_filter = layer_out.shape[0] layer_dir = save_dir +layer_name for k in range(nb_filter): img_data = layer_out[k] width, height = img_data.shape[0], img_data.shape[1] max_val = 1e-9 for i in range(width): for j in range(height): max_val = max(max_val, img_data[i][j]) im = Image.new('L', img_data.shape) for i in range(width): for j in range(height): val = int(img_data[i][j] / max_val * 255)# scale pixel value back to 255 im.putpixel((i,j), val) if os.path.exists(layer_dir) == False: os.mkdir(layer_dir) save_path = layer_dir+'/%d.jpg'%k im.save(save_path) print layer_dir
def get_activations(model, model_inputs, print_shape_only=False, layer_name=None): import keras.backend as K print('----- activations -----') activations = [] inp = model.input model_multi_inputs_cond = True if not isinstance(inp, list): # only one input! let's wrap it in a list. inp = [inp] model_multi_inputs_cond = False outputs = [layer.output for layer in model.layers if layer.name == layer_name or layer_name is None] # all layer outputs funcs = [K.function(inp + [K.learning_phase()], [out]) for out in outputs] # evaluation functions if model_multi_inputs_cond: list_inputs = [] list_inputs.extend(model_inputs) list_inputs.append(1.) else: list_inputs = [model_inputs, 1.] # Learning phase. 1 = Test mode (no dropout or batch normalization) # layer_outputs = [func([model_inputs, 1.])[0] for func in funcs] layer_outputs = [func(list_inputs)[0] for func in funcs] for layer_activations in layer_outputs: activations.append(layer_activations) if print_shape_only: print(layer_activations.shape) else: print(layer_activations) return activations
def get_activations1(model, layer, X_batch): if layer == -1: return [X_batch] else: get_activations = K.function([model.layers[0].input, K.learning_phase()], [model.layers[layer].output]) activations = get_activations([X_batch, 0]) return activations
def on_epoch_end(self, epoch, logs={}): import tensorflow as tf if self.model.validation_data and self.histogram_freq: if epoch % self.histogram_freq == 0: # TODO: implement batched calls to sess.run # (current call will likely go OOM on GPU) if self.model.uses_learning_phase: cut_v_data = len(self.model.inputs) val_data = self.model.validation_data[:cut_v_data] + [0] tensors = self.model.inputs + [K.learning_phase()] else: val_data = self.model.validation_data tensors = self.model.inputs feed_dict = dict(zip(tensors, val_data)) result = self.sess.run([self.merged], feed_dict=feed_dict) summary_str = result[0] self.writer.add_summary(summary_str, epoch) for name, value in logs.items(): if name in ['batch', 'size']: continue summary = tf.Summary() summary_value = summary.value.add() summary_value.simple_value = value.item() summary_value.tag = name self.writer.add_summary(summary, epoch) self.writer.flush()
def ANN_supervisor(predictor, descriptors, descriptor_names, debug=False): print('ANN activated for ' + str(predictor)) # _start = time.time() ## form the excitation in the corrrect order/variables excitation = tf_ANN_excitation_prepare(predictor, descriptors, descriptor_names) if debug: print('excitation is ' + str(excitation.shape)) print('fetching non-dimensionalization data... ') train_mean_x, train_mean_y, train_var_x, train_var_y = load_normalization_data(predictor) if debug: print('rescaling input excitation...') excitation = data_normalize(excitation, train_mean_x, train_var_x) ## fetch ANN # print('This is the predictor......',predictor) loaded_model = load_keras_ann(predictor) result = data_rescale(loaded_model.predict(excitation), train_mean_y, train_var_y) if not "clf" in predictor: if debug: print('LOADED MODEL HAS ' + str( len(loaded_model.layers)) + ' layers, so latent space measure will be from first ' + str( len(loaded_model.layers) - 1) + ' layers') get_outputs = K.function([loaded_model.layers[0].input, K.learning_phase()], [loaded_model.layers[len(loaded_model.layers) - 2].output]) latent_space_vector = get_outputs([excitation, 0]) # Using test phase. if debug: print('calling ANN model...') else: latent_space_vector = find_clf_lse(predictor, excitation, loaded_model=loaded_model, ensemble=False, modelname=False) # print("Finished in %f s" % (time.time() - _start)) return result, latent_space_vector
def main(_): cluster,server,job_name,task_index,num_workers = get_mpi_cluster_server_jobname(num_ps = 4, num_workers = 5) MY_GPU = task_index % NUM_GPUS if job_name == "ps": server.join() elif job_name == "worker": is_chief = (task_index == 0) # Assigns ops to the local worker by default. with tf.device(tf.train.replica_device_setter(\ worker_device='/job:worker/task:{}/gpu:{}'.format(task_index,MY_GPU), cluster=cluster)): loss,accuracy,input_tensor,true_output_tensor = get_loss_accuracy_ops() global_step = tf.Variable(0,trainable=False) optimizer = tf.train.AdagradOptimizer(0.01) if sync_mode: optimizer = tf.train.SyncReplicasOptimizer(optimizer,replicas_to_aggregate=num_workers, replica_id=task_index,total_num_replicas=num_workers) train_op = optimizer.minimize(loss, global_step=global_step) if sync_mode and is_chief: # Initial token and chief queue runners required by the sync_replicas mode chief_queue_runner = optimizer.get_chief_queue_runner() init_tokens_op = optimizer.get_init_tokens_op() saver = tf.train.Saver() summary_op = tf.merge_all_summaries() init_op = tf.initialize_all_variables() # Create a "supervisor", which oversees the training process. sv = tf.train.Supervisor(is_chief=is_chief,logdir="/tmp/train_logs",init_op=init_op,summary_op=summary_op, saver=saver,global_step=global_step,save_model_secs=600) mnist = input_data.read_data_sets(data_dir, one_hot=True) # The supervisor takes care of session initialization, restoring from # a checkpoint, and closing when done or an error occurs. config = tf.ConfigProto(allow_soft_placement=True) with sv.prepare_or_wait_for_session(server.target,config=config) as sess: if sync_mode and is_chief: sv.start_queue_runners(sess,[chief_queue_runner]) sess.run(init_tokens_op) step = 0 start = time.time() while not sv.should_stop() and step < 1000: batch_xs, batch_ys = mnist.train.next_batch(batch_size) train_feed = {input_tensor: batch_xs, true_output_tensor: batch_ys,K.learning_phase(): 1} _, step, curr_loss, curr_accuracy = sess.run([train_op, global_step, loss, accuracy], feed_dict=train_feed) sys.stdout.write('\rWorker {}, step: {}, loss: {}, accuracy: {}'.format(task_index,step,curr_loss,curr_accuracy)) sys.stdout.flush() # Ask for all the services to stop. sv.stop() print('Elapsed: {}'.format(time.time() - start))
def test_attention_layer_by_run(self): input_shape = (7, 8, 5, 6, 9) # record, document,section,sentence,word input_feature_dims = (20, 10, 50, 60, 30) # document, section, sentence, word attention_output_dims = (45, 35, 25, 65) # document, section, sentence, word attention_weight_vector_dims = (82, 72, 62, 52) # embedding embedding_rows = 200 embedding_dim = 50 # classifier initial_embedding = np.random.random((embedding_rows, embedding_dim)) inputs = HierarchicalAttention.build_inputs(input_shape, input_feature_dims) hierarchical_attention = HierarchicalAttention(input_feature_dims[0], attention_output_dims, attention_weight_vector_dims, embedding_rows, embedding_dim, initial_embedding, use_sequence_to_vector_encoder=False) output = hierarchical_attention(inputs) total = 2 output_dim = 10 timesteps = input_shape[0] x_train, _ = faked_dataset(inputs, total, timesteps, embedding_rows, output_dim) # build feed_dic feed_dict = {} for i in range(len(inputs)): feed_dict[inputs[i]] = x_train[i] feed_dict[K.learning_phase()] = 1 # tf.initialize_all_variables() # y_out is fine 2,7, 110 y_out = K.get_session().run(output, feed_dict=feed_dict) self.assertEquals(y_out.shape , (2, 7, 110), "y_out")
def build_train_fn(model): # cost lr = T.scalar() labels = K.placeholder(ndim=2, dtype='int32') ob_input = model.inputs[0] raw_softmax_outputs = model.outputs[0] softmax_outputs = raw_softmax_outputs.dimshuffle((2,0,1)) softmax_outputs = softmax_outputs.reshape((softmax_outputs.shape[0], softmax_outputs.shape[1]*softmax_outputs.shape[2])) softmax_outputs = softmax_outputs.dimshuffle((1,0)) cost = categorical_crossentropy(softmax_outputs, labels).mean() # gradients trainable_vars = model.trainable_weights grads = K.gradients(cost, trainable_vars) grads = lasagne.updates.total_norm_constraint(grads, 100) updates = lasagne.updates.nesterov_momentum(grads, trainable_vars, lr, 0.99) for key, val in model.updates: updates[key] = val # train_fn train_fn = K.function([ob_input, labels, K.learning_phase(), lr], [softmax_outputs, cost], updates=updates) return train_fn
def build_model(self): """Build a Critic (Value) model that maps (state, action)> Q_values.""" # Input layers states = layers.Input(shape=(self.state_size,), name="states") actions = layers.Input(shape=(self.action_size,), name="actions") # Add some hidden layers to state pathway net_states = layers.Dense(units=32, activation="relu")(states) net_states = layers.Dense(units=64, activation="relu")(net_states) # Add some hidden layers to action pathway net_actions = layers.Dense(units=32, activation="relu")(actions) net_actions = layers.Dense(units=64, activation="relu")(net_actions) # Combine both pathways net = layers.Add()([net_states, net_actions]) net = layers.Activation('relu')(net) Q_values = layers.Dense(units=1, name='q_values')(net) self.model = models.Model(inputs=[states, actions], outputs=Q_values) optimizer = optimizers.Adam() self.model.compile(optimizer=optimizer, loss='mse') # Compute action gradients (derivative of Q values w.r.t. to actions) action_gradients = K.gradients(Q_values, actions) # Define an additional function to fetch action gradients (to be used # by actor model) self.get_action_gradients = K.function( inputs=[*self.model.inputs, K.learning_phase()], outputs=action_gradients)
def main(): args = get_args(__doc__) with get_session() as sess, get_device(): tf.set_random_seed(1234) np.random.seed(1234) model = load_model(os.path.join(args.checkpointdir, 'model_conf.yaml')) mnist_data = load_mnist() test_feeder = Feeder({ model.view.x_bk: FeedArray(mnist_data.x_test), model.view.y_b: FeedArray(mnist_data.y_test), K.learning_phase(): FeedRandomStream(lambda b: 0) }, batch_size=128 ) saver = tf.train.Saver() sess.run(tf.initialize_all_variables()) # restore model parameters checkpoint_file = os.path.join(args.checkpointdir, 'checkpoint') with open(checkpoint_file, 'r') as f: param_file = os.path.join(args.checkpointdir, yaml.load(f)['model_checkpoint_path']) saver.restore(sess, param_file) test_fields = ['loss', 'err'] test_stats = evaluate(sess, model, test_feeder, test_fields) print ", ".join(["test {:s} = {:0.8f}".format(field, test_stats[field]) for field in test_fields])
def get_mc_predictions(model, X, nb_iter=50, batch_size=256): """ TODO :param model: :param X: :param nb_iter: :param batch_size: :return: """ output_dim = model.layers[-1].output.shape[-1].value get_output = K.function( [model.layers[0].input, K.learning_phase()], [model.layers[-1].output] ) def predict(): n_batches = int(np.ceil(X.shape[0] / float(batch_size))) output = np.zeros(shape=(len(X), output_dim)) for i in range(n_batches): output[i * batch_size:(i + 1) * batch_size] = \ get_output([X[i * batch_size:(i + 1) * batch_size], 1])[0] return output preds_mc = [] for i in tqdm(range(nb_iter)): preds_mc.append(predict()) return np.asarray(preds_mc)
def train(): global_step = tf.Variable(0, trainable=False) img = tf.placeholder(tf.float32, shape=(None, 120, 160, 3)) lbs = tf.placeholder(tf.float32, shape=(None, 10)) preds = cnn_model.load_model(img) loss = tf.reduce_mean(categorical_crossentropy(lbs, preds)) tf.scalar_summary('loss', loss) lr = tf.train.exponential_decay(INITIAL_LEARNING_RATE, global_step, cnn_input.decay_steps, LEARNING_RATE_DECAY_FACTOR, staircase=True) tf.scalar_summary('learning_rate', lr) train_step = tf.train.GradientDescentOptimizer(lr).minimize(loss, global_step=global_step) sess = tf.Session() K.set_session(sess) init = tf.initialize_all_variables() sess.run(init) summary_writer = tf.train.SummaryWriter(FLAGS.train_dir, sess.graph) with sess.as_default(): train_data = cnn_input.load_train_data() for epoch in cnn_input.epochs(train_data): for batch in cnn_input.batches(epoch): train_step.run(feed_dict={img: batch[0], lbs: batch[1], K.learning_phase(): 1})
def compile_saliency_function(model, activation_layer='block5_conv3'): input_img = model.input layer_dict = dict([(layer.name, layer) for layer in model.layers[1:]]) layer_output = layer_dict[activation_layer].output max_output = K.max(layer_output, axis=3) saliency = K.gradients(K.sum(max_output), input_img)[0] return K.function([input_img, K.learning_phase()], [saliency])
def __init__(self, model, outchannels=[], verbose=1): # Bacnend: either tensorflow or theano) self.backend = K.backend() # load model supports keras.Model and keras.Sequential if isinstance(model, Sequential): self.model = model.model elif isinstance(model, Model): self.model = model else: print("Invalid input model") return -1 # load input tensors self.input_tensors = [] for i in self.model.inputs: self.input_tensors.append(i) # The learning phase flag is a bool tensor (0 = test, 1 = train) # to be passed as input to any Keras function that uses # a different behavior at train time and test time. self.input_tensors.append(K.learning_phase()) # If outputchanel is specified, use it. # Otherwise evalueate all outputs. self.outchannels = outchannels if len(self.outchannels) == 0: if verbose: print("Evaluated output channel (0-based index): All") if K.backend() == "tensorflow": self.outchannels = range(self.model.output.shape[1]._value) elif K.backend() == "theano": self.outchannels = range(model1.output._keras_shape[1]) else: if verbose: print("Evaluated output channels (0-based index):") print(','.join([str(i) for i in self.outchannels])) # Build gradient functions for desired output channels. self.get_gradients = {} if verbose: print("Building gradient functions") # Evaluate over all channels. for c in self.outchannels: # Get tensor that calcuates gradient if K.backend() == "tensorflow": gradients = self.model.optimizer.get_gradients(self.model.output[:, c], self.model.input) if K.backend() == "theano": gradients = self.model.optimizer.get_gradients(self.model.output[:, c].sum(), self.model.input) # Build computational graph that calculates the tensfor given inputs self.get_gradients[c] = K.function(inputs=self.input_tensors, outputs=gradients) # This takes a lot of time for a big model with many tasks. # So lets pring the progress. if verbose: sys.stdout.write('\r') sys.stdout.write("Progress: " + str(int((c + 1) * 1.0 / len(self.outchannels) * 1000) * 1.0 / 10) + "%") sys.stdout.flush() # Done if verbose: print("\nDone.")
def on_batch_end(self, batch, logs=None): logs = logs or {} if batch % self.histogram_freq == 0: if self.model.uses_learning_phase: try: cut_v_data = self.model.input.shape[0] val_data = self.X_test[:cut_v_data] + [0] tensors = self.model.input + [K.learning_phase()] except: print "Tensor batch size not defined. Skipping" return else: val_data = self.X_test tensors = self.model.input feed_dict = {tensors: val_data} result = self.sess.run([self.merged], feed_dict=feed_dict) summary_str = result[0] self.writer.add_summary(summary_str, self.global_step) for name, value in logs.items(): if name in ['batch', 'size']: continue summary = tf.Summary() summary_value = summary.value.add() summary_value.simple_value = value.item() #print(name) #print(summary_value.simple_value) summary_value.tag = name self.writer.add_summary(summary, self.global_step) self.writer.flush() self.global_step = self.global_step +1
def grad_cam(input_model, model_x, orig_x, category_index, layer_name, class_names): output = input_model.output final_layer = Lambda(lambda x: target_category_loss(x, category_index, len(class_names))) output = final_layer(output) model = Model(inputs=input_model.input, outputs=output) loss = K.sum(model.layers[-1].output) conv_output = model.get_layer(layer_name).output grads = normalize(K.gradients(loss, conv_output)[0]) gradient_function = K.function([model.layers[0].input, K.learning_phase()], [conv_output, grads]) output, grads_val = gradient_function([model_x, 0]) output, grads_val = output[0, :], grads_val[0, :, :, :] weights = np.mean(grads_val, axis=(0, 1)) cam = np.zeros(output.shape[0: 2], dtype=np.float32) for i, w in enumerate(weights): cam += w * output[:, :, i] cam = np.maximum(cam, np.zeros(output.shape[0: 2], dtype=np.float32)) cam = cam.squeeze() cam = cv2.applyColorMap(np.uint8(255 * cam / np.max(cam)), cv2.COLORMAP_JET) cam = cv2.resize(cam, (np.shape(orig_x)[0], np.shape(orig_x)[1])) cam = 0.4 * cam + 0.6 * orig_x return np.uint8(cam)
def test_hierarchical_attention_layer(self): input_shape = (3, 2, 4, 2, 6) # record, document,section,sentence,word input_feature_dims = (2, 3, 4, 5, 6) # document, section, sentence, word attention_output_dims = (3, 4, 5, 8) # document, section, sentence, word attention_weight_vector_dims = (3, 4, 5, 6) # embedding embedding_rows = 10 embedding_dim = 2 # classifier initial_embedding = np.random.random((embedding_rows, embedding_dim)) inputs = HierarchicalAttention.build_inputs(input_shape, input_feature_dims) hierarchical_attention = HierarchicalAttention(input_shape,input_feature_dims[0], attention_output_dims, attention_weight_vector_dims, embedding_rows, embedding_dim, initial_embedding, use_sequence_to_vector_encoder = False) output = hierarchical_attention(inputs) self.assertEqual(shape(output), (None, 3, 8), "y") total = 2 output_dim = 10 timesteps = input_shape[0] x_train, _ = faked_dataset(inputs, total, timesteps, embedding_rows, output_dim) # build feed_dic feed_dict = {} for i in range(len(inputs)): feed_dict[inputs[i]] = x_train[i] feed_dict[K.learning_phase()] = 1 y_out = run(output, feed_dict = feed_dict) self.assertEquals(y_out.shape , (2, 3, 8), "y_out")
def build_model_policy_value(self, model, max_cache_size=100000): from collections import OrderedDict cache = OrderedDict() from tensorflow.python.keras import backend as K get_output = K.function([model.input, K.learning_phase()], [model.output[0], model.output[1]]) def model_policy_value(input_array): key = input_array.tobytes() if key in cache: cache.move_to_end(key, last=True) return cache[key] input_array = input_array.reshape((-1, 54, 6)) #input_array = np.rollaxis(input_array, 2, 1) policy, value = model.predict(input_array) #policy, value = get_output([input_array, 0]) policy = policy.reshape((12,)) value = value[0, 0] cache[key] = (policy, value) if len(cache) > max_cache_size: cache.popitem(last=False) return policy, value return model_policy_value
def test_hierarchical_attention_with_classifier_layer(self): input_shape = (3, 2, 4, 2, 6) # record, document,section,sentence,word input_feature_dims = (2, 3, 4, 5, 6) # document, section, sentence, word attention_output_dims = (3, 4, 5, 8) # document, section, sentence, word attention_weight_vector_dims = (3, 4, 5, 6) # embedding embedding_rows = 10 embedding_dim = 2 initial_embedding = np.random.random((embedding_rows, embedding_dim)) # classifier output_dim = 5 hidden_unit_numbers = (5, 20) # 5--> first hidden layer, 20 --> second hidden layer hidden_unit_activation_functions = ("relu", "relu") use_sequence_to_vector_encoder = False inputs = HierarchicalAttention.build_inputs(input_shape, input_feature_dims) classifier = ClassifierWithHierarchicalAttention(input_shape,input_feature_dims[0], attention_output_dims, attention_weight_vector_dims, embedding_rows, embedding_dim, initial_embedding, use_sequence_to_vector_encoder, output_dim, hidden_unit_numbers, hidden_unit_activation_functions) output = classifier(inputs) total = 2 timesteps = input_shape[0] x_train, _ = faked_dataset(inputs, total, timesteps, embedding_rows, output_dim) # build feed_dic feed_dict = {} for i in range(len(inputs)): feed_dict[inputs[i]] = x_train[i] feed_dict[K.learning_phase()] = 1 y_out = run(output, feed_dict = feed_dict) self.assertEqual(y_out.shape , (total , timesteps, output_dim), "y_out")
def get_network_layer_output(model, dataInput, layerNum, **kwargs): """ :param model: :param dataInput: :param layerNum: :param kwargs: :return: """ get_output = K.function([model.layers[0].input, K.learning_phase()], [model.layers[layerNum].output]) phase = kwargs.get('phase', None) if phase is None or phase == 'test': # output in test mode = 0 layer_output = get_output([dataInput, 0])[0] elif phase == 'train': # output in train mode = 1 layer_output = get_output([dataInput, 1])[0] else: raise RuntimeError("invalid phase passed to get_network_layer_output") return layer_output
def get_training_function(self, x, y): model = self.model cost, grads = self.get_cost_grads() outs = [cost] if type(grads) in {list, tuple}: outs += grads else: outs.append(grads) fn = K.function( inputs=[], outputs=outs, givens={ model.inputs[0]: x, model.targets[0]: y, model.sample_weights[0]: np.ones( (x.shape[0], ), dtype=np.float32), K.learning_phase(): np.uint8(1)}) def train_fn(theta): self.set_model_params(theta) cost_grads = fn([]) cost = np.asarray(cost_grads[0], dtype=np.float64) grads = np.asarray(self.flatten_grads(cost_grads[1:]), dtype=np.float64) return cost, grads return train_fn
def extract(matlabfile,outfile): res_ims = [] with open(matlabfile) as matfile: mat = sio.loadmat(matfile) # ys = mat['labels'] ims = mat['ims'][0] for im in ims: im = cv2.resize(im, (224, 224)).astype(np.float32) im[:,:,0] -= 103.939 im[:,:,1] -= 116.779 im[:,:,2] -= 123.68 im = im.transpose((2,0,1)) im = np.expand_dims(im, axis=0) res_ims.append(im) #%% layer_outputs = [] model = vgg.VGG_19('vgg19_weights.h5') sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True) #not used since the weights are pre-trained model.compile(optimizer=sgd, loss='categorical_crossentropy') #get_37th_layer_output = K.function([model.layers[0].input], [model.layers[38].output]) #37 = flatten get_layer_output = K.function([model.layers[0].input, K.learning_phase()], [model.layers[41].output]) #41 = last layer-1 i = 0 l = '('+str(len(res_ims))+')' for im in res_ims: print 'progress',str(i)+l i += 1 # layer_output = get_37th_layer_output([im])[0][0] layer_output = get_layer_output([im, 1])[0][0] layer_outputs.append(layer_output) Xtrain = np.array(layer_outputs).T np.save(outfile,Xtrain)
def step(self, encoder_masks, img_data, zero_paddings, decoder_inputs, target_weights, bucket_id, forward_only): # Check if the sizes match. encoder_size, decoder_size = self.buckets[bucket_id] if len(decoder_inputs) != decoder_size: raise ValueError("Decoder length must be equal to the one in bucket," " %d != %d." % (len(decoder_inputs), decoder_size)) if len(target_weights) != decoder_size: raise ValueError("Weights length must be equal to the one in bucket," " %d != %d." % (len(target_weights), decoder_size)) # Input feed: encoder inputs, decoder inputs, target_weights, as provided. input_feed = {} if not forward_only: input_feed[K.learning_phase()] = 0 else: input_feed[K.learning_phase()] = 0 input_feed[self.img_data.name] = img_data input_feed[self.zero_paddings.name] = zero_paddings for l in xrange(decoder_size): input_feed[self.decoder_inputs[l].name] = decoder_inputs[l] input_feed[self.target_weights[l].name] = target_weights[l] for l in xrange(encoder_size): input_feed[self.encoder_masks[l].name] = encoder_masks[l] # Since our targets are decoder inputs shifted by one, we need one more. last_target = self.decoder_inputs[decoder_size].name input_feed[last_target] = np.zeros([self.batch_size], dtype=np.int32) # Output feed: depends on whether we do a backward step or not. if not forward_only: output_feed = [self.updates[bucket_id], # Update Op that does SGD. #self.gradient_norms[bucket_id], # Gradient norm. self.attention_decoder_model.losses[bucket_id]] output_feed += self.keras_updates else: output_feed = [self.attention_decoder_model.losses[bucket_id]] # Loss for this batch. for l in xrange(decoder_size): # Output logits. output_feed.append(self.attention_decoder_model.outputs[bucket_id][l]) if self.visualize: output_feed += self.attention_decoder_model.attention_weights_histories[bucket_id] outputs = self.sess.run(output_feed, input_feed) if not forward_only: return None, outputs[1], None, None # Gradient norm, loss, no outputs, no attentions. else: return None, outputs[0], outputs[1:(1+self.buckets[bucket_id][1])], outputs[(1+self.buckets[bucket_id][1]):] # No gradient norm, loss, outputs, attentions.
def train_g(images, z, counter, sess=sess): outputs = [loss, G_dec, t_sum, t_optim] outs = sess.run(outputs, feed_dict={Img: images, K.learning_phase(): 1}) gl, samples, sums = outs[:3] writer.add_summary(sums, counter) images = images.reshape((-1, 80, 160, 3))[:64] samples = samples.reshape((-1, 80, 160, 3))[:64] return gl, samples, images
def detect_image(self, image): start = timer() if self.model_image_size != (None, None): assert self.model_image_size[ 0] % 32 == 0, 'Multiples of 32 required' assert self.model_image_size[ 1] % 32 == 0, 'Multiples of 32 required' boxed_image = letterbox_image( image, tuple(reversed(self.model_image_size))) else: new_image_size = (image.width - (image.width % 32), image.height - (image.height % 32)) boxed_image = letterbox_image(image, new_image_size) image_data = np.array(boxed_image, dtype='float32') print(image_data.shape) image_data /= 255. image_data = np.expand_dims(image_data, 0) # Add batch dimension. out_boxes, out_scores, out_classes = self.sess.run( [self.boxes, self.scores, self.classes], feed_dict={ self.yolo_model.input: image_data, self.input_image_shape: [image.size[1], image.size[0]], K.learning_phase(): 0 }) print('Found {} boxes for {}'.format(len(out_boxes), 'img')) font = ImageFont.truetype(font='arial.ttf', size=np.floor(3e-2 * image.size[1] + 0.5).astype('int32')) thickness = (image.size[0] + image.size[1]) // 300 if os.path.isfile("좌표.txt"): os.remove("좌표.txt") print('이건 있어야 하는 코드야!') for i, c in reversed(list(enumerate(out_classes))): predicted_class = self.class_names[c] box = out_boxes[i] score = out_scores[i] label = '{} {:.2f}'.format(predicted_class, score) draw = ImageDraw.Draw(image) label_size = draw.textsize(label, font) top, left, bottom, right = box top = max(0, np.floor(top + 0.5).astype('int32')) left = max(0, np.floor(left + 0.5).astype('int32')) bottom = min(image.size[1], np.floor(bottom + 0.5).astype('int32')) right = min(image.size[0], np.floor(right + 0.5).astype('int32')) print("Label = ", label, "좌표(left,top) ,(right,bottom)", (left, top), (right, bottom)) sliced_label = label[6:10] print(sliced_label) with open("keras_yolo3/result/좌표.txt", "w+") as f: f.write('%d' % left) f.write(' %d' % top) f.write(' %d' % right) f.write(' %d' % bottom) f.write('\n') print('result폴더에 텍스트 파일 생성 성공') if top - label_size[1] >= 0: text_origin = np.array([left, top - label_size[1]]) else: text_origin = np.array([left, top + 1]) # My kingdom for a good redistributable image drawing library. for i in range(thickness): draw.rectangle([left + i, top + i, right - i, bottom - i], outline=self.colors[c]) draw.rectangle( [tuple(text_origin), tuple(text_origin + label_size)], fill=self.colors[c]) draw.text(text_origin, label, fill=(0, 0, 0), font=font) del draw end = timer() print(end - start) # # return image
def get_output_mask(model, layer, X_batch): get_output_mask = K.function( [model.layers[0].input, K.learning_phase()], model.layers[layer].output_mask) output_mask = get_output_mask([X_batch, 0]) return output_mask
def detect_image(self, image, show_stats=True): start = timer() global vehicleId if self.model_image_size != (None, None): assert self.model_image_size[ 0] % 32 == 0, "Multiples of 32 required" assert self.model_image_size[ 1] % 32 == 0, "Multiples of 32 required" boxed_image = letterbox_image( image, tuple(reversed(self.model_image_size))) else: new_image_size = ( image.width - (image.width % 32), image.height - (image.height % 32), ) boxed_image = letterbox_image(image, new_image_size) image_data = np.array(boxed_image, dtype="float32") if show_stats: print(image_data.shape) image_data /= 255.0 image_data = np.expand_dims(image_data, 0) # Add batch dimension. out_boxes, out_scores, out_classes = self.sess.run( [self.boxes, self.scores, self.classes], feed_dict={ self.yolo_model.input: image_data, self.input_image_shape: [image.size[1], image.size[0]], K.learning_phase(): 0, }, ) if show_stats: print("Found {} boxes for {}".format(len(out_boxes), "img")) out_prediction = [] font_path = os.path.join(os.path.dirname(__file__), "font/FiraMono-Medium.otf") font = ImageFont.truetype(font=font_path, size=np.floor(3e-2 * image.size[1] + 0.5).astype("int32")) thickness = (image.size[0] + image.size[1]) // 300 for i, c in reversed(list(enumerate(out_classes))): if self.class_names[c] == 'car' or self.class_names[ c] == 'motorbike' or self.class_names[ c] == 'truck' or self.class_names[c] == 'bus': predicted_class = self.class_names[c] box = out_boxes[i] score = out_scores[i] if score > 0.8: label = "{} {:.2f}".format(predicted_class, score) draw = ImageDraw.Draw(image) label_size = draw.textsize(label, font) top, left, bottom, right = box top = max(0, np.floor(top + 0.5).astype("int32")) left = max(0, np.floor(left + 0.5).astype("int32")) bottom = min(image.size[1], np.floor(bottom + 0.5).astype("int32")) right = min(image.size[0], np.floor(right + 0.5).astype("int32")) # image was expanded to model_image_size: make sure it did not pick # up any box outside of original image (run into this bug when # lowering confidence threshold to 0.01) if top > image.size[1] or right > image.size[0]: continue if show_stats: print(label, (left, top), (right, bottom)) # Predicting Plate out_prediction.append([left, top, right, bottom, c, score]) if top - label_size[1] >= 0: text_origin = np.array([left, top - label_size[1]]) else: text_origin = np.array([left, bottom]) # My kingdom for a good redistributable image drawing library. for i in range(thickness): draw.rectangle( [left + i, top + i, right - i, bottom - i], outline=self.colors[c]) draw.rectangle( [tuple(text_origin), tuple(text_origin + label_size)], fill=self.colors[c], ) draw.text(text_origin, label, fill=(0, 0, 0), font=font) del draw else: continue end = timer() if show_stats: print("Time spent: {:.3f}sec".format(end - start)) return out_prediction, image
def predict(sess, image_file): image, image_data = preprocess_image("images/" + image_file, model_image_size = (608, 608)) out_scores, out_boxes, out_classes = sess.run([scores, boxes, classes], feed_dict={yolo_model.input: image_data, K.learning_phase(): 0}) print('Found {} boxes for {}'.format(len(out_boxes), image_file)) # Generate colors for drawing bounding boxes. colors = generate_colors(class_names) # Draw bounding boxes on the image file draw_boxes(image, out_scores, out_boxes, out_classes, class_names, colors) # Save the predicted bounding box on the image image.save(os.path.join("out", image_file), quality=90) # Display the results in the notebook output_image = scipy.misc.imread(os.path.join("out", image_file)) plt.figure(figsize=(12,12)) imshow(output_image) return out_scores, out_boxes, out_classes
x_test = x_test.astype('float32') x_test /= 255 fx_test = x_test.reshape(y_test.shape[0], img_rows * img_cols) y_test = keras.utils.to_categorical(y_test, 10) score = model.evaluate(fx_test, y_test, verbose=1) #################################################################################################################### ######################################################################################################################## inp = model.input #outputs = [model.layers[3].output] outputs = [model.layers[4].output] functor = K.function([inp] + [K.learning_phase()], outputs) layer_outs = functor([fx_test, 0.]) #################################################################################################################### print('Test loss:', score[0]) print('Test accuracy:', score[1]) # plotting the metrics model.summary() model.save('dffnn10class_modified.h5') fig = plt.figure() plt.subplot(2, 1, 1) plt.plot(model_log.history['acc']) plt.plot(model_log.history['val_acc']) plt.title('model accuracy') plt.ylabel('accuracy')
def infer(queries, db): # Query 개수: 195 # Reference(DB) 개수: 1,127 # Total (query + reference): 1,322 queries, query_img, references, reference_img = preprocess(queries, db) print( 'test data load queries {} query_img {} references {} reference_img {}' .format(len(queries), len(query_img), len(references), len(reference_img))) queries = np.asarray(queries) query_img = np.asarray(query_img) references = np.asarray(references) reference_img = np.asarray(reference_img) query_img = query_img.astype('float32') query_img /= 255 reference_img = reference_img.astype('float32') reference_img /= 255 get_feature_layer = K.function([model.layers[0].input] + [K.learning_phase()], [model.layers[-1].output]) print('inference start: ', opt.infer_dist) # inference query_vecs = get_feature_layer([query_img, 0])[0] n_epoch = reference_img.shape[0] // batch_size reference_vecs = np.zeros([0, opt.embd_dim]) offset = 0 for i in range(n_epoch): reference_img_batch = reference_img[offset:offset + batch_size] reference_vecs = np.concatenate([ reference_vecs, get_feature_layer([reference_img_batch, 0])[0] ]) offset += batch_size # l2 normalization query_vecs = l2_normalize(query_vecs) reference_vecs = l2_normalize(reference_vecs) if opt.infer_dist == 'l2': # l2 distance sim_matrix = [] for query_vec in query_vecs: dist = np.sum(np.absolute(query_vec - reference_vecs), axis=1) sim_matrix.append(dist) sim_matrix = np.array(sim_matrix) else: # Calculate cosine similarity sim_matrix = np.dot(query_vecs, reference_vecs.T) retrieval_results = {} for (i, query) in enumerate(queries): query = query.split('/')[-1].split('.')[0] sim_list = zip(references, sim_matrix[i].tolist()) sorted_sim_list = sorted(sim_list, key=lambda x: x[1], reverse=True) ranked_list = [ k.split('/')[-1].split('.')[0] for (k, v) in sorted_sim_list ] # ranked list retrieval_results[query] = ranked_list print('done') return list( zip(range(len(retrieval_results)), retrieval_results.items()))
def detect_image(self, image, out_img=False): """ :param out_img 是否输出图片 """ start = timer() if self.model_image_size != (None, None): assert self.model_image_size[ 0] % 32 == 0, 'Multiples of 32 required' assert self.model_image_size[ 1] % 32 == 0, 'Multiples of 32 required' boxed_image = letterbox_image( image, tuple(reversed(self.model_image_size))) else: new_image_size = (image.width - (image.width % 32), image.height - (image.height % 32)) boxed_image = letterbox_image(image, new_image_size) image_data = np.array(boxed_image, dtype='float32') debug_print(image_data.shape) image_data /= 255. image_data = np.expand_dims(image_data, 0) # Add batch dimension. out_bboxes, out_scores, out_classes = None, None, None with graph.as_default(): out_bboxes, out_scores, out_classes = self.sess.run( [self.bboxes, self.scores, self.classes], feed_dict={ self.yolo_model.input: image_data, self.input_image_shape: [image.size[1], image.size[0]], K.learning_phase(): 0 }) res_data = { 'bboxes': out_bboxes, 'scores': out_scores, 'classes': out_classes, } if out_img is False: return None, res_data debug_print('Found {} bboxes for {}'.format(len(out_bboxes), 'img')) font = ImageFont.truetype(font='font/FiraMono-Medium.otf', size=np.floor(3e-2 * image.size[1] + 0.5).astype('int32')) thickness = (image.size[0] + image.size[1]) // 300 for i, c in reversed(list(enumerate(out_classes))): predicted_class = self.class_names[c] box = out_bboxes[i] score = out_scores[i] label = '{} {:.2f}'.format(predicted_class, score) draw = ImageDraw.Draw(image) label_size = draw.textsize(label, font) top, left, bottom, right = box top = max(0, np.floor(top + 0.5).astype('int32')) left = max(0, np.floor(left + 0.5).astype('int32')) bottom = min(image.size[1], np.floor(bottom + 0.5).astype('int32')) right = min(image.size[0], np.floor(right + 0.5).astype('int32')) debug_print(label, (left, top), (right, bottom)) if top - label_size[1] >= 0: text_origin = np.array([left, top - label_size[1]]) else: text_origin = np.array([left, top + 1]) # My kingdom for a good redistributable image drawing library. for i in range(thickness): draw.rectangle([left + i, top + i, right - i, bottom - i], outline=self.colors[c]) draw.rectangle( [tuple(text_origin), tuple(text_origin + label_size)], fill=self.colors[c]) draw.text(text_origin, label, fill=(0, 0, 0), font=font) del draw end = timer() debug_print(end - start) return image, res_data
# load the model model = load_model('data/model-' + 'imap-mail-embedonly' + '.h5') model.summary() pred = model.predict(all_words) layer_list = [] for layer in model.layers: layer_list.append(layer) print(layer.name) # retrieve the embedding-layer l = model.get_layer('embedding_1') extended_input = [K.learning_phase()] + [l.input] layer_f = K.function(extended_input, [l.output]) input_tensor = [0] + [all_words] l_out = layer_f(input_tensor) idx = 0 plt.title("Layer: %s" % l.name) plt.imshow(l_out[0][0][0:400]) plt.show() vocab_vec = l_out[0].reshape(400, 30) pca = PCA(n_components=6) result = pca.fit_transform(vocab_vec) plt.scatter(result[:, 0], result[:, 1]) plt.show()
def _average_gradient_norm(model, X_train, y_train, batch_size): # just checking if the model was already compiled if not hasattr(model, "train_function"): raise RuntimeError("You must compile your model before using it.") weights = model.trainable_weights # weight tensors get_gradients = model.optimizer.get_gradients(model.total_loss, weights) # gradient tensors input_tensors = [ # input data model.inputs[0], # how much to weight each sample by model.sample_weights[0], # labels model.targets[0], # train or test mode K.learning_phase() ] grad_fct = K.function(inputs=input_tensors, outputs=get_gradients) steps = 0 total_norm = 0 s_w = None nb_steps = X_train.shape[0] // batch_size if X_train.shape[0] % batch_size == 0: pad_last = False else: pad_last = True def generator(X_train, y_train, pad_last): for i in range(nb_steps): X = X_train[i * batch_size:(i + 1) * batch_size, ...] y = y_train[i * batch_size:(i + 1) * batch_size, ...] yield (X, y) if pad_last: X = X_train[nb_steps * batch_size:, ...] y = y_train[nb_steps * batch_size:, ...] yield (X, y) datagen = generator(X_train, y_train, pad_last) while steps < nb_steps: X, y = next(datagen) # set sample weights to one # for every input if s_w is None: s_w = np.ones(X.shape[0]) gradients = grad_fct([X, s_w, y, 0]) total_norm += np.sqrt(np.sum([np.sum(np.square(g)) for g in gradients])) steps += 1 if pad_last: X, y = next(datagen) # set sample weights to one # for every input if s_w is None: s_w = np.ones(X.shape[0]) gradients = grad_fct([X, s_w, y, 0]) total_norm += np.sqrt(np.sum([np.sum(np.square(g)) for g in gradients])) steps += 1 return total_norm / float(steps)
yolo_outputs = yolo_head(yolo_model.output, anchors, len(class_names)) #Now yolo_eval function selects the best boxes using filtering and non-max suppression techniques. # If you want to dive in more to see how this works, refer keras_yolo.py file in yad2k/models boxes, scores, classes = yolo_eval(yolo_outputs, image_shape) # Initiate a session sess = K.get_session() #Preprocess the input image before feeding into the convolutional network image, image_data = preprocess_image("images/" + input_image_name, model_image_size=(608, 608)) #Run the session out_scores, out_boxes, out_classes = sess.run([scores, boxes, classes], feed_dict={ yolo_model.input: image_data, K.learning_phase(): 0 }) #Print the results print('Found {} boxes for {}'.format(len(out_boxes), input_image_name)) #Produce the colors for the bounding boxs colors = generate_colors(class_names) #Draw the bounding boxes draw_boxes(image, out_scores, out_boxes, out_classes, class_names, colors) #Apply the predicted bounding boxes to the image and save it image.save(os.path.join("out", input_image_name), quality=90) output_image = scipy.misc.imread(os.path.join("out", input_image_name)) imshow(output_image)
def build_model(self): # Build a critic (Q value) network that maps (state, action) pairs -> Q-values. # Define input layers # states = layers.Input(shape=(self.state_size,), name='states') states = layers.Input( shape=self.state_size, name='states') # initial (635, 800, 3) resize(89, 120,3) actions = layers.Input(shape=(self.action_size, ), name='actions') ### Add hidden layer(s) for state pathway ### # Normalisation norma = layers.Lambda(lambda img: img / 255.0)(states) conv1 = layers.Conv2D(filters=32, kernel_size=8, strides=4, activation='relu')(norma) # hidd1 state pool1 = layers.MaxPooling2D(pool_size=2)(conv1) b_norm1 = layers.BatchNormalization()(pool1) conv2 = layers.Conv2D(filters=64, kernel_size=4, strides=2, padding='same', activation='relu')(b_norm1) # hidd2 state pool2 = layers.MaxPooling2D(pool_size=2)(conv2) b_norm2 = layers.BatchNormalization()(pool2) # drop = layers.Dropout(0.3) conv3 = layers.Conv2D(filters=128, kernel_size=3, strides=1, padding='same', activation='relu')(b_norm2) # hidd3 state pool3 = layers.MaxPooling2D(pool_size=2)(conv3) b_norm3 = layers.BatchNormalization()(pool3) flat = layers.Flatten()(b_norm3) # (conv3) net_states = layers.Dense(256, activation='relu')( flat) # could be 'relu' activation # hidd4 state ### Add hidden layer(s) for action pathway ### net_actions = layers.Dense(units=50, activation='relu')(actions) # hidd1 action net_actions = layers.BatchNormalization()(net_actions) # drop = layers.Dropout(0.3) net_actions = layers.Dense(units=128, activation='relu')( net_actions) # hidd2 action net_actions = layers.BatchNormalization()(net_actions) net_actions = layers.Dense(units=128, activation='relu')( net_actions) # hidd3 action net_actions = layers.BatchNormalization()(net_actions) net_actions = layers.Dense(units=256, activation='relu')( net_actions) # hidd4 action net_actions = layers.BatchNormalization()(net_actions) # Combine state and action pathways net = layers.Add()([net_states, net_actions]) net = layers.Activation('relu')(net) # Add more layers to the combined network if needed net = layers.Dense(units=32, activation='relu')(net) # Add final output layer to produce action values (Q values) Q_values = layers.Dense(units=1, name='q_values')(net) net = layers.BatchNormalization()(net) # Create Keras model self.model = models.Model(inputs=[states, actions], outputs=Q_values) # Print summary of cnn model # print("Summary of Critic model:\n", self.model.summary()) # plot graph # plot_model(self.model, to_file='critic_neural_network.png') # Define optimizer and compile model for training with built-in loss function optimizer = optimizers.Adam(0.0001) # 0.013 self.model.compile(optimizer=optimizer, loss='mse') # Compute action gradients (derivative of Q values w.r.t. to actions) action_gradients = K.gradients(Q_values, actions) # Define an additional function to fetch action gradients (to be used by actor model) self.get_action_gradients = K.function( inputs=[*self.model.input, K.learning_phase()], outputs=action_gradients)
def build_model(self): """Build an actor (policy) network that maps states -> actions.""" # Define input layer (states) states = layers.Input(shape=(self.state_size, ), name='states') # Add hidden layers net = layers.Dense(units=128, activation='relu', kernel_initializer='glorot_uniform', kernel_regularizer=regularizers.l2(0.01))(states) net = layers.BatchNormalization()(net) net = layers.Dropout(0.01)(net) # Try different layer sizes, activations, add batch normalization, regularizers, etc. net = layers.Dense(units=256, activation='relu', kernel_initializer='glorot_uniform', kernel_regularizer=regularizers.l2(0.01))(net) net = layers.BatchNormalization()(net) net = layers.Dropout(0.01)(net) net = layers.Dense(units=256, activation='relu', kernel_initializer='glorot_uniform', kernel_regularizer=regularizers.l2(0.01))(net) net = layers.BatchNormalization()(net) net = layers.Dropout(0.01)(net) net = layers.Dense(units=128, activation='relu', kernel_initializer='glorot_uniform', kernel_regularizer=regularizers.l2(0.01))(net) net = layers.BatchNormalization()(net) net = layers.Dropout(0.01)(net) # Add final output layer with sigmoid activation raw_actions = layers.Dense( units=self.action_size, activation='sigmoid', kernel_initializer='random_uniform', kernel_regularizer=regularizers.l2(0.01) # ,activity_regularizer=regularizers.l2(0.01) , name='raw_actions')(net) # Scale [0, 1] output for each action dimension to proper range actions = layers.Lambda(lambda x: (x * self.action_range) + self.action_low, name='actions')(raw_actions) # Create Keras model self.model = models.Model(inputs=states, outputs=actions) # Define loss function using action value (Q value) gradients action_gradients = layers.Input(shape=(self.action_size, )) loss = K.mean(-action_gradients * actions) # TODO: check loss function # Incorporate any additional losses here (e.g. from regularizers) # Define optimizer and training function optimizer = optimizers.Adam(lr=0.0001, clipvalue=0.5) updates_op = optimizer.get_updates(params=self.model.trainable_weights, loss=loss) self.train_fn = K.function( inputs=[self.model.input, action_gradients, K.learning_phase()], outputs=[loss], updates=updates_op)
def detect_image(self, image, time): global hang global hang1 global hang2 global hang3 global hang4 start = timer() if self.model_image_size != (None, None): assert self.model_image_size[ 0] % 32 == 0, 'Multiples of 32 required' assert self.model_image_size[ 1] % 32 == 0, 'Multiples of 32 required' boxed_image = letterbox_image( image, tuple(reversed(self.model_image_size))) else: new_image_size = (image.width - (image.width % 32), image.height - (image.height % 32)) boxed_image = letterbox_image(image, new_image_size) image_data = np.array(boxed_image, dtype='float32') print(image_data.shape) image_data /= 255. image_data = np.expand_dims(image_data, 0) # Add batch dimension. out_boxes, out_scores, out_classes = self.sess.run( [self.boxes, self.scores, self.classes], feed_dict={ self.yolo_model.input: image_data, self.input_image_shape: [image.size[1], image.size[0]], K.learning_phase(): 0 }) print('Found {} boxes for {}'.format(len(out_boxes), 'img')) font = ImageFont.truetype(font='font/FiraMono-Medium.otf', size=np.floor(3e-2 * image.size[1] + 0.5).astype('int32')) thickness = (image.size[0] + image.size[1]) // 300 for i, c in reversed(list(enumerate(out_classes))): predicted_class = self.class_names[c] box = out_boxes[i] score = out_scores[i] label = '{} {:.2f}'.format(predicted_class, score) draw = ImageDraw.Draw(image) label_size = draw.textsize(label, font) top, left, bottom, right = box top = max(0, np.floor(top + 0.5).astype('int32')) left = max(0, np.floor(left + 0.5).astype('int32')) bottom = min(image.size[1], np.floor(bottom + 0.5).astype('int32')) right = min(image.size[0], np.floor(right + 0.5).astype('int32')) print(label, (left, top), (right, bottom)) print(left) if (label.split(' ')[0] == 'basketball'): ws.write(hang, 0, label.split(' ')[0]) ws.write(hang, 1, label.split(' ')[1]) ws.write(hang, 2, int(left)) ws.write(hang, 3, int(top)) ws.write(hang, 4, int(right)) ws.write(hang, 5, int(bottom)) ws.write(hang, 6, time) #ws.write(hang,7,str((int(left)+int(right))/2) ) #ws.write(hang,8,str((1080-int(top) + 1080-int(bottom))/2)) hang = hang + 1 if (label.split(' ')[0] == 'hand'): ws2.write(hang1, 0, label.split(' ')[0]) ws2.write(hang1, 1, label.split(' ')[1]) ws2.write(hang1, 2, int(left)) ws2.write(hang1, 3, int(top)) ws2.write(hang1, 4, int(right)) ws2.write(hang1, 5, int(bottom)) ws2.write(hang1, 6, time) hang1 = hang1 + 1 if (label.split(' ')[0] == 'arm'): ws3.write(hang2, 0, label.split(' ')[0]) ws3.write(hang2, 1, label.split(' ')[1]) ws3.write(hang2, 2, int(left)) ws3.write(hang2, 3, int(top)) ws3.write(hang2, 4, int(right)) ws3.write(hang2, 5, int(bottom)) ws3.write(hang2, 6, time) hang2 = hang2 + 1 if (label.split(' ')[0] == 'hoop'): ws4.write(hang3, 0, label.split(' ')[0]) ws4.write(hang3, 1, label.split(' ')[1]) ws4.write(hang3, 2, int(left)) ws4.write(hang3, 3, int(top)) ws4.write(hang3, 4, int(right)) ws4.write(hang3, 5, int(bottom)) ws4.write(hang3, 6, time) #ws4.write(hang, 7, str((int(left) + int(right))/2)) #ws4.write(hang, 8, str((1080 - int(top) + 1080 - int(bottom))/2)) hang3 = hang3 + 1 if (label.split(' ')[0] == 'head'): ws5.write(hang4, 0, label.split(' ')[0]) ws5.write(hang4, 1, label.split(' ')[1]) ws5.write(hang4, 2, int(left)) ws5.write(hang4, 3, int(top)) ws5.write(hang4, 4, int(right)) ws5.write(hang4, 5, int(bottom)) ws5.write(hang4, 6, time) hang4 = hang4 + 1 if top - label_size[1] >= 0: text_origin = np.array([left, top - label_size[1]]) else: text_origin = np.array([left, top + 1]) # My kingdom for a good redistributable image drawing library. for i in range(thickness): draw.rectangle([left + i, top + i, right - i, bottom - i], outline=self.colors[c]) draw.rectangle( [tuple(text_origin), tuple(text_origin + label_size)], fill=self.colors[c]) draw.text(text_origin, label, fill=(0, 0, 0), font=font) del draw end = timer() print(end - start) return image
def get_input(model, layer, X_batch): get_input = K.function( [model.layers[0].input, K.learning_phase()], model.layers[layer].input) _input = get_input([X_batch, 0]) return _input
def detect_image(self, image, image_name="#NA#", lbl_box="", output_detections=False): start = timer() if self.model_image_size != (None, None): assert self.model_image_size[ 0] % 32 == 0, 'Multiples of 32 required' assert self.model_image_size[ 1] % 32 == 0, 'Multiples of 32 required' boxed_image = letterbox_image( image, tuple(reversed(self.model_image_size))) else: new_image_size = (image.width - (image.width % 32), image.height - (image.height % 32)) boxed_image = letterbox_image(image, new_image_size) image_data = np.array(boxed_image, dtype='float32') print(image_data.shape) image_data /= 255. image_data = np.expand_dims(image_data, 0) # Add batch dimension. out_boxes, out_scores, out_classes = self.sess.run( [self.boxes, self.scores, self.classes], feed_dict={ self.yolo_model.input: image_data, self.input_image_shape: [image.size[1], image.size[0]], K.learning_phase(): 0 }) print('Found {} boxes for {}'.format(len(out_boxes), image_name)) font = ImageFont.truetype(font='font/FiraMono-Medium.otf', size=np.floor(3e-2 * image.size[1] + 0.5).astype('int32')) thickness = (image.size[0] + image.size[1]) // 300 dets = [] for i, c in reversed(list(enumerate(out_classes))): predicted_class = self.class_names[c] box = out_boxes[i] score = out_scores[i] label = '{} {:.2f}'.format(predicted_class, score) draw = ImageDraw.Draw(image) label_size = draw.textsize(label, font) top, left, bottom, right = box top = max(0, np.floor(top + 0.5).astype('int32')) left = max(0, np.floor(left + 0.5).astype('int32')) bottom = min(image.size[1], np.floor(bottom + 0.5).astype('int32')) right = min(image.size[0], np.floor(right + 0.5).astype('int32')) # print(label, (left, top), (right, bottom)) # for reuse in labeling # print(f'###EXPORT###{image_name}|ORIG_LABEL:{lbl_box}|{left},{top},{right},{bottom},{label}') print( f'###EXPORT###{{"filename":"{image_name}","orig_label":"{lbl_box}","yolo_detection":"{left},{top},{right},{bottom}","yolo_label":"{" ".join(label.split(" ")[0:-1])}","yolo_score":"{label.split(" ")[-1]}"}}' ) dets.append( f'{{"filename":"{image_name}","orig_label":"{lbl_box}","yolo_detection":"{left},{top},{right},{bottom}","yolo_label":"{" ".join(label.split(" ")[0:-1])}","yolo_score":"{label.split(" ")[-1]}"}}' ) if top - label_size[1] >= 0: text_origin = np.array([left, top - label_size[1]]) else: text_origin = np.array([left, top + 1]) # My kingdom for a good redistributable image drawing library. for i in range(thickness): draw.rectangle([left + i, top + i, right - i, bottom - i], outline=self.colors[c]) draw.rectangle( [tuple(text_origin), tuple(text_origin + label_size)], fill=self.colors[c]) draw.text(text_origin, label, fill=(0, 0, 0), font=font) del draw end = timer() print(end - start) if (output_detections): return {"image": image, "detections": dets} return image
def build_model(self): """Build a critic (value) network that maps (state, action) pairs -> Q-values.""" # Define input layers states = layers.Input(shape=(self.state_size,), name='states') actions = layers.Input(shape=(self.action_size,), name='actions') # Parameters for critic network ki = initializers.glorot_uniform(); kr = None if self.model_params["use_l2"] > 0: l2 = self.model_params["use_l2"] kr = regularizers.l2(l2) dropout_rate = self.model_params["dropout_rate"] use_bias = True use_bn = self.model_params["use_bn"] if use_bn: use_bias=False act_fn = self.model_params["act_fn"] n1 = self.model_params["layer1_n"] n2 = self.model_params["layer2_n"] # The critic network s_fc1 = layers.Dense(units=n1, kernel_initializer=ki, kernel_regularizer=kr, use_bias=use_bias, name='s_fc1')(states) if use_bn: s_fc1 = layers.BatchNormalization()(s_fc1) if (dropout_rate > 0): s_fc1 = layers.Dropout(dropout_rate)(s_fc1) s_fc1 = layers.Activation(act_fn)(s_fc1) a_fc1 = layers.Dense(units=n1, kernel_initializer=ki, kernel_regularizer=kr, use_bias=use_bias, name='a_fc1')(actions) if use_bn: a_fc1 = layers.BatchNormalization()(a_fc1) if (dropout_rate > 0): a_fc1 = layers.Dropout(dropout_rate)(a_fc1) a_fc1 = layers.Activation(act_fn)(a_fc1) # Combine state and action pathways net = layers.Add()([s_fc1, a_fc1]) net = layers.Dense(units=n2, kernel_initializer=ki, kernel_regularizer=kr, use_bias=use_bias, name='fc2')(net) if use_bn: net = layers.BatchNormalization()(net) if (dropout_rate > 0): net = layers.Dropout(dropout_rate)(net) net = layers.Activation(act_fn)(net) # Add final output layer to prduce action values (Q values) ki = initializers.RandomUniform(minval=-3e-3, maxval=3e-3) Q_values = layers.Dense(units=1, kernel_initializer=ki, kernel_regularizer=kr, activation='linear', name='q_values')(net) # Create Keras model self.model = models.Model(inputs=[states, actions], outputs=Q_values) # Define optimizer and compile model for training with built-in loss function optimizer = optimizers.Adam(lr=0.001) self.model.compile(optimizer=optimizer, loss='mse') # Compute action gradients (derivative of Q values w.r.t. to actions) action_gradients = K.gradients(Q_values, actions) # Define an additional function to fetch action gradients (to be used by actor model) self.get_action_gradients = K.function( inputs=[*self.model.input, K.learning_phase()], outputs=action_gradients)
def predict(sess, image_file, is_show_info=True, is_plot=True): """ 运行存储在sess的计算图以预测image_file的边界框,打印出预测的图与信息。 参数: sess - 包含了YOLO计算图的TensorFlow/Keras的会话。 image_file - 存储在images文件夹下的图片名称 返回: out_scores - tensor类型,维度为(None,),锚框的预测的可能值。 out_boxes - tensor类型,维度为(None,4),包含了锚框位置信息。 out_classes - tensor类型,维度为(None,),锚框的预测的分类索引。 """ #图像预处理 image, image_data = yolo_utils.preprocess_image(r"E:\深度学习\第四课第三周编程作业\Car detection for Autonomous Driving\images\\" + image_file, model_image_size = (608, 608)) #运行会话并在feed_dict中选择正确的占位符. out_scores, out_boxes, out_classes = sess.run([scores, boxes, classes], feed_dict = {yolo_model.input:image_data, K.learning_phase(): 0}) #打印预测信息 if is_show_info: print("在" + str(image_file) + "中找到了" + str(len(out_boxes)) + "个锚框。") #指定要绘制的边界框的颜色 colors = yolo_utils.generate_colors(class_names) #在图中绘制边界框 yolo_utils.draw_boxes(image, out_scores, out_boxes, out_classes, class_names, colors) #保存已经绘制了边界框的图 image.save(os.path.join("out", image_file), quality=100) #打印出已经绘制了边界框的图 if is_plot: output_image = scipy.misc.imread(os.path.join("out", image_file)) plt.imshow(output_image) return out_scores, out_boxes, out_classes
def loadCNN(wf_index): global get_output model = Sequential() model.add(Conv2D(nb_filters, (nb_conv, nb_conv), padding='valid', input_shape=(img_channels, img_rows, img_cols))) convout1 = Activation('relu') model.add(convout1) model.add(Conv2D(nb_filters, (nb_conv, nb_conv))) convout2 = Activation('relu') model.add(convout2) model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool))) model.add(Dropout(0.5)) model.add(Flatten()) model.add(Dense(128)) model.add(Activation('relu')) model.add(Dropout(0.5)) model.add(Dense(nb_classes)) model.add(Activation('softmax')) ''' model.add(ZeroPadding2D((1,1),input_shape=(img_channels, img_rows, img_cols))) model.add(Conv2D(nb_filters , (nb_conv, nb_conv), activation='relu')) #model.add(ZeroPadding2D((1,1))) #model.add(Conv2D(nb_filters , (nb_conv, nb_conv), activation='relu')) model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool))) model.add(Dropout(0.2)) #model.add(ZeroPadding2D((1,1))) model.add(Conv2D(nb_filters , (nb_conv, nb_conv), activation='relu')) #model.add(ZeroPadding2D((1,1))) model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool))) ## #model.add(Conv2D(nb_filters , (nb_conv, nb_conv), activation='relu')) #model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool), strides=(2,2))) model.add(Dropout(0.3)) model.add(Flatten()) ### #model.add(Dense(128)) #model.add(Activation('relu')) #model.add(Dropout(0.5)) model.add(Dense(256)) model.add(Activation('relu')) model.add(Dropout(0.5)) model.add(Dense(nb_classes)) model.add(Activation('softmax')) ''' #sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True) model.compile(loss='categorical_crossentropy', optimizer='adadelta', metrics=['accuracy']) # Model summary model.summary() # Model conig details model.get_config() #from keras.utils import plot_model #plot_model(model, to_file='new_model.png', show_shapes = True) if wf_index >= 0: #Load pretrained weights fname = WeightFileName[int(wf_index)] print("loading ", fname) model.load_weights(fname) layer = model.layers[11] get_output = K.function([model.layers[0].input, K.learning_phase()], [layer.output,]) return model
args.add_argument('--pause', type=int, default=0, help='model 을 load 할때 1로 설정됩니다.') config = args.parse_args() # training parameters nb_epoch = config.epochs batch_size = config.batch_size num_classes = 1000 input_shape = (224, 224, 3) # input image shape """ Model """ model, base_model = get_model('triplet', 224, num_classes, opt.base_model) bind_model(base_model, config.batch_size) get_feature_layer = K.function([base_model.layers[0].input] + [K.learning_phase()], [base_model.layers[-1].output]) if config.pause: nsml.paused(scope=locals()) bTrainmode = False if config.mode == 'train': bTrainmode = True """ Load data """ print('dataset path', DATASET_PATH) output_path = ['./img_list.pkl', './label_list.pkl'] train_dataset_path = os.path.join(DATASET_PATH, 'train/train_data') if nsml.IS_ON_NSML: # Caching file
def infer(queries, db): # Query 개수: 195 # Reference(DB) 개수: 1,127 # Total (query + reference): 1,322 queries, query_img, references, reference_img = preprocess(queries, db) print( 'test data load queries {} query_img {} references {} reference_img {}' .format(len(queries), len(query_img), len(references), len(reference_img))) queries = np.asarray(queries) query_img = np.asarray(query_img) references = np.asarray(references) reference_img = np.asarray(reference_img) query_img = query_img.astype('float32') query_img /= 255 reference_img = reference_img.astype('float32') reference_img /= 255 get_feature_layer = K.function([model.layers[0].input] + [K.learning_phase()], [model.layers[-2].output]) print('inference start') # inference query_vecs = get_feature_layer([query_img, 0])[0] # caching db output, db inference db_output = './db_infer.pkl' if os.path.exists(db_output): with open(db_output, 'rb') as f: reference_vecs = pickle.load(f) else: reference_vecs = get_feature_layer([reference_img, 0])[0] with open(db_output, 'wb') as f: pickle.dump(reference_vecs, f) # l2 normalization query_vecs = l2_normalize(query_vecs) reference_vecs = l2_normalize(reference_vecs) # Calculate cosine similarity sim_matrix = np.dot(query_vecs, reference_vecs.T) retrieval_results = {} for (i, query) in enumerate(queries): query = query.split('/')[-1].split('.')[0] sim_list = zip(references, sim_matrix[i].tolist()) sorted_sim_list = sorted(sim_list, key=lambda x: x[1], reverse=True) ranked_list = [ k.split('/')[-1].split('.')[0] for (k, v) in sorted_sim_list ] # ranked list retrieval_results[query] = ranked_list print('done') return list( zip(range(len(retrieval_results)), retrieval_results.items()))
regions.append(' '.join(tokens[:filter_sizes[0]])) for i in range(filter_sizes[0], len(tokens)): regions.append(' '.join(tokens[(i-filter_sizes[0]+1):(i+1)])) my_review = np.array([my_review]) reg_emb = get_region_embedding([my_review,0])[0][0,:,:] ### fill the gap ### ### compute the norms of the region embeddings ### ### you may use np.linalg.norm() https://docs.scipy.org/doc/numpy/reference/generated/numpy.linalg.norm.html ### norms = np.linalg.norm(reg_emb, axis=1) print([list(zip(regions,norms))[idx] for idx in np.argsort(-norms).tolist()]) # = = = = = saliency map = = = = = input_tensors = [model.input, K.learning_phase()] ### fill the gap (extract the rows of the embedding matrix from the model) ### saliency_input = model.layers[1].output ### fill the gap (get the probability distribution over classes from the model) ### saliency_output = model.output[0][0] gradients = model.optimizer.get_gradients(saliency_output,saliency_input) compute_gradients = K.function(inputs=input_tensors,outputs=gradients) matrix = compute_gradients([my_review,0])[0][0,:,:] ### fill the gap (compute the magnitude of the partial derivatives) ### ### you may use np.absolute https://docs.scipy.org/doc/numpy/reference/generated/numpy.absolute.html ### to_plot = np.absolute(matrix)
def grad_cam_batch(model, images, classes, layer_name, X, Y, Z, all_features, non_image_mean, non_image_std, non_images, input_layer_1, input_layer_2, dense_layer): """ GradCAM method to process multiple images in one run. INPUT: model - the model for which the gradcams should be computed images - the batch of images for which the gradcams should be computed classes - an array indicating the classes corresponding to the image batch X, Y, Z - the input shapes of the images OUTPUT cam_mean - the average gradcam of the image batch cam_var - the variation of the mean gradcam of the image batch """ # get loss, output and gradients loss = tf.gather_nd(model.output, np.dstack([range(images.shape[0]), classes])[0]) first_input = model.get_layer(input_layer_1).input second_input = model.get_layer(input_layer_2).input layer_output = model.get_layer(layer_name).output dense_layer_output = model.get_layer(dense_layer).output grads = K.gradients(loss, layer_output)[0] dense_grads = K.gradients(loss, dense_layer_output)[0] # calculate class activation maps for image batch if non_image_features == False: # create gradient function gradient_fn = K.function([first_input, K.learning_phase()], [layer_output, grads]) conv_output, grads_val = gradient_fn([images, learning_phase]) else: # create gradient function gradient_fn = K.function([first_input, second_input, K.learning_phase()], [layer_output, grads]) conv_output, grads_val = gradient_fn([images, non_images, learning_phase]) # create gradient function for dense layer dense_gradient_fn = K.function([first_input, second_input, K.learning_phase()], [dense_layer_output, dense_grads]) dense_conv_output, dense_grads_val = dense_gradient_fn([images, non_images, learning_phase]) weights = np.mean(grads_val, axis=(1, 2, 3)) cams = np.einsum('ijklm,im->ijkl', conv_output, weights) #dense_weights = np.mean(dense_grads_val, axis=(1)) dense_cams = np.einsum('ij,ij->i', dense_conv_output, dense_grads_val) # process CAMs new_cams = np.empty((images.shape[0], X, Y, Z)) for i in range(new_cams.shape[0]): cam_i = cams[i] - cams[i].mean() cam_i = (cam_i + 1e-10) / (np.linalg.norm(cam_i) + 1e-10) new_cams[i] = resize(cam_i, (X, Y, Z), order=1, mode='constant', cval=0, anti_aliasing=False) new_cams[i] = np.maximum(new_cams[i], 0) new_cams[i] = new_cams[i] / new_cams[i].max() # process CAMs for dense layer new_cams_dense = np.empty((images.shape[0], X, Y, Z)) for i in range(new_cams.shape[0]): cam_i = cams[i] - cams[i].mean() cam_i = (cam_i + 1e-10) / (np.linalg.norm(cam_i) + 1e-10) new_cams[i] = resize(cam_i, (X, Y, Z), order=1, mode='constant', cval=0, anti_aliasing=False) new_cams[i] = np.maximum(new_cams[i], 0) new_cams[i] = new_cams[i] / new_cams[i].max() # calculate mean and variation cam_mean = np.mean(new_cams, axis=0) cam_var = np.var(new_cams, axis=0) return cam_mean, cam_var
def test_yolo(image, image_file_name): if is_fixed_size: # TODO: When resizing we can use minibatch input. resized_image = image.resize(tuple(reversed(model_image_size)), Image.BICUBIC) image_data = np.array(resized_image, dtype='float32') else: # Due to skip connection + max pooling in YOLO_v2, inputs must have # width and height as multiples of 32. new_image_size = (image.width - (image.width % 32), image.height - (image.height % 32)) resized_image = image.resize(new_image_size, Image.BICUBIC) image_data = np.array(resized_image, dtype='float32') print(image_data.shape) image_data /= 255. image_data = np.expand_dims(image_data, 0) # Add batch dimension. out_boxes, out_scores, out_classes = sess.run( [boxes, scores, classes], feed_dict={ yolo_model.input: image_data, input_image_shape: [image.size[1], image.size[0]], K.learning_phase(): 0 }) print('Found {} boxes for {}'.format(len(out_boxes), image_file_name)) # Write data to a JSON file located within the 'output/' directory. # This ASSUMES that the game comes from a spectated video starting at 0:00 # Else, data will not be alligned! font = ImageFont.truetype(font='font/FiraMono-Medium.otf', size=np.floor(3e-2 * image.size[1] + 0.5).astype('int32')) thickness = (image.size[0] + image.size[1]) // 300 data = {} data['timestamp'] = '0:00' data['champs'] = {} for i, c in reversed(list(enumerate(out_classes))): predicted_class = class_names[c] box = out_boxes[i] score = out_scores[i] if user_did_specify_champs and predicted_class not in champs_in_game: continue label = '{} {:.2f}'.format(predicted_class, score) draw = ImageDraw.Draw(image) label_size = draw.textsize(label, font) top, left, bottom, right = box top = max(0, np.floor(top + 0.5).astype('int32')) left = max(0, np.floor(left + 0.5).astype('int32')) bottom = min(image.size[1], np.floor(bottom + 0.5).astype('int32')) right = min(image.size[0], np.floor(right + 0.5).astype('int32')) print(label, (left, top), (right, bottom)) # Save important data to JSON. data['champs'][predicted_class] = score if top - label_size[1] >= 0: text_origin = np.array([left, top - label_size[1]]) else: text_origin = np.array([left, top + 1]) for i in range(thickness): draw.rectangle([left + i, top + i, right - i, bottom - i], outline=colors[c]) draw.rectangle([tuple(text_origin), tuple(text_origin + label_size)], fill=colors[c]) draw.text(text_origin, label, fill=(0, 0, 0), font=font) del draw image.save(os.path.join(output_path, image_file_name), quality=90)
def __init__(self, model): self.f = K.function( [model.layers[0].input, K.learning_phase()], [model.layers[-1].output])
def add_method(self, model): action_gradients = K.gradients(model.output[0], model.input[1]) self.get_action_gradients = K.function( inputs=[*model.input, K.learning_phase()], outputs=action_gradients)
def compile(self, optimizer, metrics=[]): metrics += [mean_q] if type(optimizer) in (list, tuple): if len(optimizer) != 2: raise ValueError( 'More than two optimizers provided. Please only provide a maximum of two optimizers, the first one for the actor and the second one for the critic.' ) actor_optimizer, critic_optimizer = optimizer else: actor_optimizer = optimizer critic_optimizer = clone_optimizer(optimizer) if type(actor_optimizer) is str: actor_optimizer = optimizers.get(actor_optimizer) if type(critic_optimizer) is str: critic_optimizer = optimizers.get(critic_optimizer) assert actor_optimizer != critic_optimizer if len(metrics) == 2 and hasattr(metrics[0], '__len__') and hasattr( metrics[1], '__len__'): actor_metrics, critic_metrics = metrics else: actor_metrics = critic_metrics = metrics def clipped_error(y_true, y_pred): return K.mean(huber_loss(y_true, y_pred, self.delta_clip), axis=-1) # Compile target networks. We only use them in feed-forward mode, hence we can pass any # optimizer and loss since we never use it anyway. self.target_actor = clone_model(self.actor, self.custom_model_objects) self.target_actor.compile(optimizer='sgd', loss='mse') self.target_critic = clone_model(self.critic, self.custom_model_objects) self.target_critic.compile(optimizer='sgd', loss='mse') # We also compile the actor. We never optimize the actor using Keras but instead compute # the policy gradient ourselves. However, we need the actor in feed-forward mode, hence # we also compile it with any optimzer and self.actor.compile(optimizer='sgd', loss='mse') # Compile the critic. if self.target_model_update < 1.: # We use the `AdditionalUpdatesOptimizer` to efficiently soft-update the target model. critic_updates = get_soft_target_model_updates( self.target_critic, self.critic, self.target_model_update) critic_optimizer = AdditionalUpdatesOptimizer( critic_optimizer, critic_updates) self.critic.compile(optimizer=critic_optimizer, loss=clipped_error, metrics=critic_metrics) # Combine actor and critic so that we can get the policy gradient. # Assuming critic's state inputs are the same as actor's. combined_inputs = [] state_inputs = [] for i in self.critic.input: if i == self.critic_action_input: combined_inputs.append([]) else: combined_inputs.append(i) state_inputs.append(i) combined_inputs[self.critic_action_input_idx] = self.actor( state_inputs) combined_output = self.critic(combined_inputs) updates = actor_optimizer.get_updates( params=self.actor.trainable_weights, loss=-K.mean(combined_output)) if self.target_model_update < 1.: # Include soft target model updates. updates += get_soft_target_model_updates(self.target_actor, self.actor, self.target_model_update) updates += self.actor.updates # include other updates of the actor, e.g. for BN # Finally, combine it all into a callable function. if K.backend() == 'tensorflow': self.actor_train_fn = K.function(state_inputs + [K.learning_phase()], [self.actor(state_inputs)], updates=updates) else: if self.uses_learning_phase: state_inputs += [K.learning_phase()] self.actor_train_fn = K.function(state_inputs, [self.actor(state_inputs)], updates=updates) self.actor_optimizer = actor_optimizer self.compiled = True
#y_pred = (y_pred > 0.5) # Making the Confusion Matrix #from sklearn.metrics import confusion_matrix #cm = confusion_matrix(y_test, y_pred) #from keras.models import Model from keras import backend as Ke inp = classifier.input #inp=X_test[0] # input placeholder outputs = [layer.output for layer in classifier.layers] # all layer outputs functors = [ Ke.function([inp] + [Ke.learning_phase()], [out]) for out in outputs ] # evaluation functions # Testing input_shape = 1 test = data #np.random.random(input_shape)[np.newaxis,...] layer_outs = [func([test, 1.]) for func in functors] #np.array(layer_outs[0]).shape recomposed_image = recompose_image( np.reshape(np.array(layer_outs[1]), (256, 12)), 64, 48, 4, 3) recomposed_image = recompose_image(y_pred, 64, 64, 4, 4) #recomposed_image = recompose_image(np.reshape(np.array(layer_outs[1]),(16384,12)),512,384,4,3) #recomposed_image = recompose_image(y_pred,512,512,4,4) cv2.imshow('Recomposed Image', recomposed_image) cv2.waitKey(0) cv2.imwrite('01.png', recomposed_image * 255)
def main(): # Prepare args args = parse_args() num_labeled_train = args.num_labeled_train num_test = args.num_test ramp_up_period = args.ramp_up_period ramp_down_period = args.ramp_down_period num_class = args.num_class num_epoch = args.num_epoch batch_size = args.batch_size weight_max = args.weight_max learning_rate = args.learning_rate weight_norm_flag = args.weight_norm_flag augmentation_flag = args.augmentation_flag whitening_flag = args.whitening_flag trans_range = args.trans_range # Data Preparation train_x, train_y, test_x, test_y = load_data(args.data_path) ret_dic = split_supervised_train(train_x, train_y, num_labeled_train) ret_dic['test_x'] = test_x ret_dic['test_y'] = test_y ret_dic = make_train_test_dataset(ret_dic, num_class) unsupervised_target = ret_dic['unsupervised_target'] supervised_label = ret_dic['supervised_label'] supervised_flag = ret_dic['train_sup_flag'] unsupervised_weight = ret_dic['unsupervised_weight'] test_y = ret_dic['test_y'] train_x, test_x = normalize_images(ret_dic['train_x'], ret_dic['test_x']) # pre-process if whitening_flag: train_x, test_x = whiten_zca(train_x, test_x) if augmentation_flag: train_x = np.pad(train_x, ((0, 0), (trans_range, trans_range), (trans_range, trans_range), (0, 0)), 'reflect') # make the whole data and labels for training # x = [train_x, supervised_label, supervised_flag, unsupervised_weight] y = np.concatenate((unsupervised_target, supervised_label, supervised_flag, unsupervised_weight), axis=1) num_train_data = train_x.shape[0] # Build Model if weight_norm_flag: from lib.model_WN import build_model from lib.weight_norm import AdamWithWeightnorm optimizer = AdamWithWeightnorm(lr=learning_rate, beta_1=0.9, beta_2=0.999) else: from lib.model_BN import build_model optimizer = Adam(lr=learning_rate, beta_1=0.9, beta_2=0.999) model = build_model(num_class=num_class) model.compile(optimizer=optimizer, loss=semi_supervised_loss(num_class)) model.metrics_tensors += model.outputs model.summary() get_prediction = K.function( inputs=[model.layers[0].input, K.learning_phase()], outputs=[model.get_layer('activation_1').output]) # prepare weights and arrays for updates gen_weight = ramp_up_weight( ramp_up_period, weight_max * (num_labeled_train / num_train_data)) gen_lr_weight = ramp_down_weight(ramp_down_period) idx_list = [v for v in range(num_train_data)] # Training for epoch in range(num_epoch): print('epoch: ', epoch) idx_list = shuffle(idx_list) if epoch > num_epoch - ramp_down_period: weight_down = next(gen_lr_weight) K.set_value(model.optimizer.lr, weight_down * learning_rate) K.set_value(model.optimizer.beta_1, 0.4 * weight_down + 0.5) ave_loss = 0 for i in range(0, num_train_data, batch_size): target_idx = idx_list[i:i + batch_size] if augmentation_flag: x1 = data_augmentation_tempen(train_x[target_idx], trans_range) else: x1 = train_x[target_idx] x2 = supervised_label[target_idx] x3 = supervised_flag[target_idx] x4 = unsupervised_weight[target_idx] y_t = y[target_idx] # get the first prediction y_t[:, 0:num_class] = get_prediction(inputs=[x1, 1])[0] x_t = [x1, x2, x3, x4] tr_loss, output = model.train_on_batch(x=x_t, y=y_t) ave_loss += tr_loss print('Training Loss: ', (ave_loss * batch_size) / num_train_data, flush=True) # Update phase next_weight = next(gen_weight) y, unsupervised_weight = update_weight(y, unsupervised_weight, next_weight) # Evaluation if epoch % 5 == 0: print('Evaluate epoch : ', epoch, flush=True) evaluate(model, num_class, num_test, test_x, test_y)
def get_activations(model, layer, X_batch): get_activations = K.function( [model.layers[0].input, K.learning_phase()], model.layers[layer].output) activations = get_activations([X_batch, 0]) return activations
def build_model(self): """Build an actor (policy) network that maps states -> actions.""" # Define input layer (states) states = layers.Input(shape=(self.state_size,), name='states') # Parameters for actor network ki = initializers.glorot_uniform(); kr = None if self.model_params["use_l2"] > 0: l2 = self.model_params["use_l2"] kr = regularizers.l2(l2) dropout_rate = self.model_params["dropout_rate"] use_bias = True use_bn = self.model_params["use_bn"] if use_bn: use_bias=False act_fn = self.model_params["act_fn"] n1 = self.model_params["layer1_n"] n2 = self.model_params["layer2_n"] # the actor network fc1 = layers.Dense(units=n1, kernel_initializer=ki, kernel_regularizer=kr, use_bias=use_bias, name='fc1')(states) if use_bn: fc1 = layers.BatchNormalization()(fc1) if (dropout_rate > 0): fc1 = layers.Dropout(dropout_rate)(fc1) fc1 = layers.Activation(act_fn)(fc1) fc2 = layers.Dense(units=n2, kernel_initializer=ki, kernel_regularizer=kr, use_bias=use_bias, name='fc2')(fc1) if use_bn: fc2 = layers.BatchNormalization()(fc2) if (dropout_rate > 0): fc2 = layers.Dropout(dropout_rate)(fc2) fc2 = layers.Activation(act_fn)(fc2) # Add final output layer ki = initializers.RandomUniform(minval=-3e-3, maxval=3e-3) raw_actions = layers.Dense(units=self.action_size, activation='tanh', kernel_initializer=ki, kernel_regularizer=kr, use_bias=False, name='raw_actions')(fc2) # Scale [-1, 1] output for each action dimension to proper range actions = layers.Lambda(lambda x: (0.5*(x + 1.0)*self.action_range + self.action_low), name='actions')(raw_actions) # Create Keras model self.model = models.Model(inputs=states, outputs=actions) # Define loss function using action value (Q value) gradients action_gradients = layers.Input(shape=(self.action_size,)) loss = K.mean(-action_gradients * actions) if (kr is not None): loss = loss + l2*K.sum(K.square(self.model.get_layer('fc1').get_weights()[0])) \ + l2*K.sum(K.square(self.model.get_layer('fc2').get_weights()[0])) \ + l2*K.sum(K.square(self.model.get_layer('raw_actions').get_weights()[0])) # Define optimizer and training function optimizer = optimizers.Adam(lr=0.0001) updates_op = optimizer.get_updates(params=self.model.trainable_weights, loss=loss) self.train_fn = K.function( inputs=[self.model.input, action_gradients, K.learning_phase()], outputs=[], updates=updates_op)