def setUp(self):
self.inp = (np.random.randn(10*10*51*51)
.reshape(10,10,51,51))
self.keras_model = keras.models.Sequential()
conv_layer = keras.layers.convolutional.Conv2D(
filters=2, kernel_size=(4,4), strides=(2,2),
activation="relu", input_shape=(10,51,51),
data_format="channels_first")
self.keras_model.add(conv_layer)
self.keras_model.add(keras.layers.pooling.MaxPooling2D(
pool_size=(4,4), strides=(2,2),
data_format="channels_first"))
self.keras_model.add(keras.layers.pooling.AveragePooling2D(
pool_size=(4,4), strides=(2,2),
data_format="channels_first"))
self.keras_model.add(keras.layers.Flatten())
self.keras_model.add(keras.layers.Dense(output_dim=1))
self.keras_model.add(keras.layers.core.Activation("sigmoid"))
self.keras_model.compile(loss="mse", optimizer="sgd")
self.keras_output_fprop_func = compile_func(
[self.keras_model.layers[0].input,
K.learning_phase()],
self.keras_model.layers[-1].output)
grad = tf.gradients(tf.reduce_sum(
self.keras_model.layers[-2].output[:,0]),
[self.keras_model.layers[0].input])[0]
self.grad_func = compile_func(
[self.keras_model.layers[0].input,
K.learning_phase()], grad)
self.saved_file_path = "conv2model_channelsfirst.h5"
if (os.path.isfile(self.saved_file_path)):
os.remove(self.saved_file_path)
self.keras_model.save(self.saved_file_path)
def main():
args = get_args(__doc__)
with get_session() as sess, get_device():
tf.set_random_seed(1234)
np.random.seed(1234)
model = load_model(args.modelconf)
learn_opts = load_learn_opts(args.learnconf)
mnist_data = load_mnist()
train_feeder = Feeder({
model.view.x_bhh: FeedArray(mnist_data.x_train.reshape((-1, 28, 28))),
model.view.y_b: FeedArray(mnist_data.y_train),
K.learning_phase(): FeedRandomStream(lambda b: 1)
},
batch_size=learn_opts.batch_size
)
valid_feeder = Feeder({
model.view.x_bhh: FeedArray(mnist_data.x_valid.reshape((-1, 28, 28))),
model.view.y_b: FeedArray(mnist_data.y_valid),
K.learning_phase(): FeedRandomStream(lambda b: 0)
},
batch_size=learn_opts.batch_size
)
optimizer = tf.train.AdamOptimizer(learning_rate=1e-4)
callbacks = [DefaultLogger(['loss', 'err']),
DeepDashboardLogger(
learn_opts.exp_id,
[RawLogger('out.log', 'Log', ['loss', 'err']),
CSVLogger('loss.csv', 'Loss', ['loss']),
CSVLogger('err.csv', 'Classification Error', ['err'])]
),
EarlyStopping('loss', learn_opts.patience),
ModelCheckpoint(os.path.join(os.getenv('MYSHKIN_CHECKPOINTDIR', '.'),
learn_opts.exp_id),
'loss',
verbose=True)]
if not args.nodash:
call(["open", "http://localhost/deep-dashboard/?id={:s}".format(learn_opts.exp_id)])
else:
print "experiment id: {:s}".format(learn_opts.exp_id)
fit(model,
optimizer,
train_feeder,
valid_feeder,
sess,
callbacks=callbacks,
n_epochs=learn_opts.n_epochs)
def __init__(self, replay_filename, group_name, model_filename=''):
# Set learning phase to TEST
self.learning_phase = TEST_MODE
# If not informed, defaults to '_model' suffix
if model_filename == '':
model_filename = '{}_model.h5'.format(group_name)
# Loads Keras model
self.model = load_model(model_filename)
# Loads ReplayData file
self.replay_data = h5py.File('{}'.format(replay_filename), 'r')
self.group_name = group_name
self.group = self.replay_data[self.group_name]
# Retrieves some basic information from the replay data
self.inputs = self.group['inputs'][:]
self.targets = self.group['targets'][:]
self.n_epochs = self.group.attrs['n_epochs']
self.n_layers = self.group.attrs['n_layers']
# Retrieves weights as a list, each element being one epoch
self.weights = self._retrieve_weights()
# Gets Tensors for the weights in the same order as the layers
# Keras' model.weights returns the Tensors in a different order!
self._model_weights = [w for layer in self.model.layers for w in layer.weights]
### Functions
# Keras function to get the outputs, given inputs and weights
self._get_output = K.function(inputs=[K.learning_phase()] + self.model.inputs + self._model_weights,
outputs=[self.model.layers[-1].output])
# Keras function to get the loss and metrics, given inputs, targets, weights and sample weights
self._get_metrics = K.function(inputs=[K.learning_phase()] + self.model.inputs + self.model.targets +
self._model_weights + self.model.sample_weights,
outputs=[self.model.total_loss] + self.model.metrics_tensors)
# Keras function to compute the binary cross entropy, given inputs, targets, weights and sample weights
self._get_binary_crossentropy = K.function(inputs=[K.learning_phase()] + self.model.inputs +
self.model.targets + self._model_weights +
self.model.sample_weights,
outputs=[K.binary_crossentropy(self.model.targets[0],
self.model.outputs[0])])
# Attributes for the visualizations - Data
self._feature_space_data = None
self._loss_hist_data = None
self._loss_and_metric_data = None
self._prob_hist_data = None
self._decision_boundary_data = None
# Attributes for the visualizations - Plot objects
self._feature_space_plot = None
self._loss_hist_plot = None
self._loss_and_metric_plot = None
self._prob_hist_plot = None
self._decision_boundary_plot = None
def __init__(self, state_size, num_actuators, args):
# remember parameters
self.state_size = state_size
self.num_actuators = num_actuators
self.discount_rate = args.discount_rate
self.target_rate = args.target_rate
self.noise = args.noise
self.noise_scale = args.noise_scale
x, u, m, v, q, p, a = self._createLayers(args)
# wrappers around computational graph
fmu = K.function([K.learning_phase(), x], m)
self.mu = lambda x: fmu([0, x])
fP = K.function([K.learning_phase(), x], p)
self.P = lambda x: fP([0, x])
fA = K.function([K.learning_phase(), x, u], a)
self.A = lambda x, u: fA([0, x, u])
fQ = K.function([K.learning_phase(), x, u], q)
self.Q = lambda x, u: fQ([0, x, u])
# main model
self.model = Model(input=[x,u], output=q)
self.model.summary()
if args.optimizer == 'adam':
optimizer = Adam(args.learning_rate)
elif args.optimizer == 'rmsprop':
optimizer = RMSprop(args.learning_rate)
else:
assert False
self.model.compile(optimizer=optimizer, loss='mse')
if args.optimizer == 'adam':
optimizer = Adam(args.learning_rate)
elif args.optimizer == 'rmsprop':
optimizer = RMSprop(args.learning_rate)
else:
assert False
self.model.compile(optimizer=optimizer, loss='mse')
# another set of layers for target model
x, u, m, v, q, p, a = self._createLayers(args)
# V() function uses target model weights
fV = K.function([K.learning_phase(), x], v)
self.V = lambda x: fV([0, x])
# target model is initialized from main model
self.target_model = Model(input=[x,u], output=q)
self.target_model.set_weights(self.model.get_weights())
def _make_loss_pred_function(self):
if not hasattr(self, 'loss_pred_function'):
raise RuntimeError('You must compile your model before using it.')
if self.loss_pred_function is None:
inputs = self._feed_inputs + self._feed_targets + self._feed_sample_weights
if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
inputs += [K.learning_phase()]
# Return loss and metrics, no gradient updates.
# Does update the network states.
self.loss_pred_function = K.function(inputs,
[self.total_loss] + self.outputs,
updates=self.state_updates,
**self._function_kwargs)
def _make_test_function(self):
if not hasattr(self, 'test_function'):
raise Exception('You must compile your model before using it.')
if self.test_function is None:
inputs = self.inputs + self.targets + self.sample_weights
if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
inputs += [K.learning_phase()]
outputs = [self.total_loss]
outputs += list(itertools.chain.from_iterable(
[model.total_loss] + model.metrics_tensors
for model in self.layers))
self.test_function = K.function(inputs,
outputs,
updates=self.state_updates,
**self._function_kwargs)
def visualize(model, datas):
for img_name in range(len(datas)):
data = [datas[img_name]]
layers = ['input', 'conv1', 'pool1', 'conv2', 'pool2', 'conv3', 'pool3', 'conv4', 'pool4']
save_dir = 'visual_pic/%d/'%(img_name+1)
if os.path.exists(save_dir) == False:
os.mkdir(save_dir)
for layer_name in layers:
layer = K.function([model.input, K.learning_phase()], [model.get_layer(layer_name).output])
layer_out = layer([data, 0])[0][0]
nb_filter = layer_out.shape[0]
layer_dir = save_dir +layer_name
for k in range(nb_filter):
img_data = layer_out[k]
width, height = img_data.shape[0], img_data.shape[1]
max_val = 1e-9
for i in range(width):
for j in range(height):
max_val = max(max_val, img_data[i][j])
im = Image.new('L', img_data.shape)
for i in range(width):
for j in range(height):
val = int(img_data[i][j] / max_val * 255)# scale pixel value back to 255
im.putpixel((i,j), val)
if os.path.exists(layer_dir) == False:
os.mkdir(layer_dir)
save_path = layer_dir+'/%d.jpg'%k
im.save(save_path)
print layer_dir
def get_activations(model, model_inputs, print_shape_only=False, layer_name=None):
import keras.backend as K
print('----- activations -----')
activations = []
inp = model.input
model_multi_inputs_cond = True
if not isinstance(inp, list):
# only one input! let's wrap it in a list.
inp = [inp]
model_multi_inputs_cond = False
outputs = [layer.output for layer in model.layers if
layer.name == layer_name or layer_name is None] # all layer outputs
funcs = [K.function(inp + [K.learning_phase()], [out]) for out in outputs] # evaluation functions
if model_multi_inputs_cond:
list_inputs = []
list_inputs.extend(model_inputs)
list_inputs.append(1.)
else:
list_inputs = [model_inputs, 1.]
# Learning phase. 1 = Test mode (no dropout or batch normalization)
# layer_outputs = [func([model_inputs, 1.])[0] for func in funcs]
layer_outputs = [func(list_inputs)[0] for func in funcs]
for layer_activations in layer_outputs:
activations.append(layer_activations)
if print_shape_only:
print(layer_activations.shape)
else:
print(layer_activations)
return activations
def get_activations1(model, layer, X_batch):
if layer == -1:
return [X_batch]
else:
get_activations = K.function([model.layers[0].input, K.learning_phase()], [model.layers[layer].output])
activations = get_activations([X_batch, 0])
return activations
def on_epoch_end(self, epoch, logs={}):
import tensorflow as tf
if self.model.validation_data and self.histogram_freq:
if epoch % self.histogram_freq == 0:
# TODO: implement batched calls to sess.run
# (current call will likely go OOM on GPU)
if self.model.uses_learning_phase:
cut_v_data = len(self.model.inputs)
val_data = self.model.validation_data[:cut_v_data] + [0]
tensors = self.model.inputs + [K.learning_phase()]
else:
val_data = self.model.validation_data
tensors = self.model.inputs
feed_dict = dict(zip(tensors, val_data))
result = self.sess.run([self.merged], feed_dict=feed_dict)
summary_str = result[0]
self.writer.add_summary(summary_str, epoch)
for name, value in logs.items():
if name in ['batch', 'size']:
continue
summary = tf.Summary()
summary_value = summary.value.add()
summary_value.simple_value = value.item()
summary_value.tag = name
self.writer.add_summary(summary, epoch)
self.writer.flush()
def ANN_supervisor(predictor, descriptors, descriptor_names, debug=False):
print('ANN activated for ' + str(predictor))
# _start = time.time()
## form the excitation in the corrrect order/variables
excitation = tf_ANN_excitation_prepare(predictor, descriptors, descriptor_names)
if debug:
print('excitation is ' + str(excitation.shape))
print('fetching non-dimensionalization data... ')
train_mean_x, train_mean_y, train_var_x, train_var_y = load_normalization_data(predictor)
if debug:
print('rescaling input excitation...')
excitation = data_normalize(excitation, train_mean_x, train_var_x)
## fetch ANN
# print('This is the predictor......',predictor)
loaded_model = load_keras_ann(predictor)
result = data_rescale(loaded_model.predict(excitation), train_mean_y, train_var_y)
if not "clf" in predictor:
if debug:
print('LOADED MODEL HAS ' + str(
len(loaded_model.layers)) + ' layers, so latent space measure will be from first ' + str(
len(loaded_model.layers) - 1) + ' layers')
get_outputs = K.function([loaded_model.layers[0].input, K.learning_phase()],
[loaded_model.layers[len(loaded_model.layers) - 2].output])
latent_space_vector = get_outputs([excitation, 0]) # Using test phase.
if debug:
print('calling ANN model...')
else:
latent_space_vector = find_clf_lse(predictor, excitation, loaded_model=loaded_model, ensemble=False,
modelname=False)
# print("Finished in %f s" % (time.time() - _start))
return result, latent_space_vector
def main(_):
cluster,server,job_name,task_index,num_workers = get_mpi_cluster_server_jobname(num_ps = 4, num_workers = 5)
MY_GPU = task_index % NUM_GPUS
if job_name == "ps":
server.join()
elif job_name == "worker":
is_chief = (task_index == 0)
# Assigns ops to the local worker by default.
with tf.device(tf.train.replica_device_setter(\
worker_device='/job:worker/task:{}/gpu:{}'.format(task_index,MY_GPU),
cluster=cluster)):
loss,accuracy,input_tensor,true_output_tensor = get_loss_accuracy_ops()
global_step = tf.Variable(0,trainable=False)
optimizer = tf.train.AdagradOptimizer(0.01)
if sync_mode:
optimizer = tf.train.SyncReplicasOptimizer(optimizer,replicas_to_aggregate=num_workers,
replica_id=task_index,total_num_replicas=num_workers)
train_op = optimizer.minimize(loss, global_step=global_step)
if sync_mode and is_chief:
# Initial token and chief queue runners required by the sync_replicas mode
chief_queue_runner = optimizer.get_chief_queue_runner()
init_tokens_op = optimizer.get_init_tokens_op()
saver = tf.train.Saver()
summary_op = tf.merge_all_summaries()
init_op = tf.initialize_all_variables()
# Create a "supervisor", which oversees the training process.
sv = tf.train.Supervisor(is_chief=is_chief,logdir="/tmp/train_logs",init_op=init_op,summary_op=summary_op,
saver=saver,global_step=global_step,save_model_secs=600)
mnist = input_data.read_data_sets(data_dir, one_hot=True)
# The supervisor takes care of session initialization, restoring from
# a checkpoint, and closing when done or an error occurs.
config = tf.ConfigProto(allow_soft_placement=True)
with sv.prepare_or_wait_for_session(server.target,config=config) as sess:
if sync_mode and is_chief:
sv.start_queue_runners(sess,[chief_queue_runner])
sess.run(init_tokens_op)
step = 0
start = time.time()
while not sv.should_stop() and step < 1000:
batch_xs, batch_ys = mnist.train.next_batch(batch_size)
train_feed = {input_tensor: batch_xs, true_output_tensor: batch_ys,K.learning_phase(): 1}
_, step, curr_loss, curr_accuracy = sess.run([train_op, global_step, loss, accuracy], feed_dict=train_feed)
sys.stdout.write('\rWorker {}, step: {}, loss: {}, accuracy: {}'.format(task_index,step,curr_loss,curr_accuracy))
sys.stdout.flush()
# Ask for all the services to stop.
sv.stop()
print('Elapsed: {}'.format(time.time() - start))
def test_attention_layer_by_run(self):
input_shape = (7, 8, 5, 6, 9)
# record, document,section,sentence,word
input_feature_dims = (20, 10, 50, 60, 30)
# document, section, sentence, word
attention_output_dims = (45, 35, 25, 65)
# document, section, sentence, word
attention_weight_vector_dims = (82, 72, 62, 52)
# embedding
embedding_rows = 200
embedding_dim = 50
# classifier
initial_embedding = np.random.random((embedding_rows, embedding_dim))
inputs = HierarchicalAttention.build_inputs(input_shape, input_feature_dims)
hierarchical_attention = HierarchicalAttention(input_feature_dims[0], attention_output_dims, attention_weight_vector_dims, embedding_rows, embedding_dim, initial_embedding, use_sequence_to_vector_encoder=False)
output = hierarchical_attention(inputs)
total = 2
output_dim = 10
timesteps = input_shape[0]
x_train, _ = faked_dataset(inputs, total, timesteps, embedding_rows, output_dim)
# build feed_dic
feed_dict = {}
for i in range(len(inputs)):
feed_dict[inputs[i]] = x_train[i]
feed_dict[K.learning_phase()] = 1
# tf.initialize_all_variables()
# y_out is fine 2,7, 110
y_out = K.get_session().run(output, feed_dict=feed_dict)
self.assertEquals(y_out.shape , (2, 7, 110), "y_out")
def build_train_fn(model):
# cost
lr = T.scalar()
labels = K.placeholder(ndim=2, dtype='int32')
ob_input = model.inputs[0]
raw_softmax_outputs = model.outputs[0]
softmax_outputs = raw_softmax_outputs.dimshuffle((2,0,1))
softmax_outputs = softmax_outputs.reshape((softmax_outputs.shape[0], softmax_outputs.shape[1]*softmax_outputs.shape[2]))
softmax_outputs = softmax_outputs.dimshuffle((1,0))
cost = categorical_crossentropy(softmax_outputs, labels).mean()
# gradients
trainable_vars = model.trainable_weights
grads = K.gradients(cost, trainable_vars)
grads = lasagne.updates.total_norm_constraint(grads, 100)
updates = lasagne.updates.nesterov_momentum(grads, trainable_vars, lr, 0.99)
for key, val in model.updates:
updates[key] = val
# train_fn
train_fn = K.function([ob_input, labels, K.learning_phase(), lr],
[softmax_outputs, cost],
updates=updates)
return train_fn
def build_model(self):
"""Build a Critic (Value) model that maps (state, action)> Q_values."""
# Input layers
states = layers.Input(shape=(self.state_size,), name="states")
actions = layers.Input(shape=(self.action_size,), name="actions")
# Add some hidden layers to state pathway
net_states = layers.Dense(units=32, activation="relu")(states)
net_states = layers.Dense(units=64, activation="relu")(net_states)
# Add some hidden layers to action pathway
net_actions = layers.Dense(units=32, activation="relu")(actions)
net_actions = layers.Dense(units=64, activation="relu")(net_actions)
# Combine both pathways
net = layers.Add()([net_states, net_actions])
net = layers.Activation('relu')(net)
Q_values = layers.Dense(units=1, name='q_values')(net)
self.model = models.Model(inputs=[states, actions], outputs=Q_values)
optimizer = optimizers.Adam()
self.model.compile(optimizer=optimizer, loss='mse')
# Compute action gradients (derivative of Q values w.r.t. to actions)
action_gradients = K.gradients(Q_values, actions)
# Define an additional function to fetch action gradients (to be used
# by actor model)
self.get_action_gradients = K.function(
inputs=[*self.model.inputs, K.learning_phase()],
outputs=action_gradients)
def main():
args = get_args(__doc__)
with get_session() as sess, get_device():
tf.set_random_seed(1234)
np.random.seed(1234)
model = load_model(os.path.join(args.checkpointdir, 'model_conf.yaml'))
mnist_data = load_mnist()
test_feeder = Feeder({
model.view.x_bk: FeedArray(mnist_data.x_test),
model.view.y_b: FeedArray(mnist_data.y_test),
K.learning_phase(): FeedRandomStream(lambda b: 0)
},
batch_size=128
)
saver = tf.train.Saver()
sess.run(tf.initialize_all_variables())
# restore model parameters
checkpoint_file = os.path.join(args.checkpointdir, 'checkpoint')
with open(checkpoint_file, 'r') as f:
param_file = os.path.join(args.checkpointdir,
yaml.load(f)['model_checkpoint_path'])
saver.restore(sess, param_file)
test_fields = ['loss', 'err']
test_stats = evaluate(sess, model, test_feeder, test_fields)
print ", ".join(["test {:s} = {:0.8f}".format(field, test_stats[field])
for field in test_fields])
def get_mc_predictions(model, X, nb_iter=50, batch_size=256):
"""
TODO
:param model:
:param X:
:param nb_iter:
:param batch_size:
:return:
"""
output_dim = model.layers[-1].output.shape[-1].value
get_output = K.function(
[model.layers[0].input, K.learning_phase()],
[model.layers[-1].output]
)
def predict():
n_batches = int(np.ceil(X.shape[0] / float(batch_size)))
output = np.zeros(shape=(len(X), output_dim))
for i in range(n_batches):
output[i * batch_size:(i + 1) * batch_size] = \
get_output([X[i * batch_size:(i + 1) * batch_size], 1])[0]
return output
preds_mc = []
for i in tqdm(range(nb_iter)):
preds_mc.append(predict())
return np.asarray(preds_mc)
def train():
global_step = tf.Variable(0, trainable=False)
img = tf.placeholder(tf.float32, shape=(None, 120, 160, 3))
lbs = tf.placeholder(tf.float32, shape=(None, 10))
preds = cnn_model.load_model(img)
loss = tf.reduce_mean(categorical_crossentropy(lbs, preds))
tf.scalar_summary('loss', loss)
lr = tf.train.exponential_decay(INITIAL_LEARNING_RATE,
global_step,
cnn_input.decay_steps,
LEARNING_RATE_DECAY_FACTOR,
staircase=True)
tf.scalar_summary('learning_rate', lr)
train_step = tf.train.GradientDescentOptimizer(lr).minimize(loss, global_step=global_step)
sess = tf.Session()
K.set_session(sess)
init = tf.initialize_all_variables()
sess.run(init)
summary_writer = tf.train.SummaryWriter(FLAGS.train_dir, sess.graph)
with sess.as_default():
train_data = cnn_input.load_train_data()
for epoch in cnn_input.epochs(train_data):
for batch in cnn_input.batches(epoch):
train_step.run(feed_dict={img: batch[0],
lbs: batch[1],
K.learning_phase(): 1})
def compile_saliency_function(model, activation_layer='block5_conv3'):
input_img = model.input
layer_dict = dict([(layer.name, layer) for layer in model.layers[1:]])
layer_output = layer_dict[activation_layer].output
max_output = K.max(layer_output, axis=3)
saliency = K.gradients(K.sum(max_output), input_img)[0]
return K.function([input_img, K.learning_phase()], [saliency])
def __init__(self, model, outchannels=[], verbose=1):
# Bacnend: either tensorflow or theano)
self.backend = K.backend()
# load model supports keras.Model and keras.Sequential
if isinstance(model, Sequential):
self.model = model.model
elif isinstance(model, Model):
self.model = model
else:
print("Invalid input model")
return -1
# load input tensors
self.input_tensors = []
for i in self.model.inputs:
self.input_tensors.append(i)
# The learning phase flag is a bool tensor (0 = test, 1 = train)
# to be passed as input to any Keras function that uses
# a different behavior at train time and test time.
self.input_tensors.append(K.learning_phase())
# If outputchanel is specified, use it.
# Otherwise evalueate all outputs.
self.outchannels = outchannels
if len(self.outchannels) == 0:
if verbose: print("Evaluated output channel (0-based index): All")
if K.backend() == "tensorflow":
self.outchannels = range(self.model.output.shape[1]._value)
elif K.backend() == "theano":
self.outchannels = range(model1.output._keras_shape[1])
else:
if verbose:
print("Evaluated output channels (0-based index):")
print(','.join([str(i) for i in self.outchannels]))
# Build gradient functions for desired output channels.
self.get_gradients = {}
if verbose: print("Building gradient functions")
# Evaluate over all channels.
for c in self.outchannels:
# Get tensor that calcuates gradient
if K.backend() == "tensorflow":
gradients = self.model.optimizer.get_gradients(self.model.output[:, c], self.model.input)
if K.backend() == "theano":
gradients = self.model.optimizer.get_gradients(self.model.output[:, c].sum(), self.model.input)
# Build computational graph that calculates the tensfor given inputs
self.get_gradients[c] = K.function(inputs=self.input_tensors, outputs=gradients)
# This takes a lot of time for a big model with many tasks.
# So lets pring the progress.
if verbose:
sys.stdout.write('\r')
sys.stdout.write("Progress: " + str(int((c + 1) * 1.0 / len(self.outchannels) * 1000) * 1.0 / 10) + "%")
sys.stdout.flush()
# Done
if verbose: print("\nDone.")
def on_batch_end(self, batch, logs=None):
logs = logs or {}
if batch % self.histogram_freq == 0:
if self.model.uses_learning_phase:
try:
cut_v_data = self.model.input.shape[0]
val_data = self.X_test[:cut_v_data] + [0]
tensors = self.model.input + [K.learning_phase()]
except:
print "Tensor batch size not defined. Skipping"
return
else:
val_data = self.X_test
tensors = self.model.input
feed_dict = {tensors: val_data}
result = self.sess.run([self.merged], feed_dict=feed_dict)
summary_str = result[0]
self.writer.add_summary(summary_str, self.global_step)
for name, value in logs.items():
if name in ['batch', 'size']:
continue
summary = tf.Summary()
summary_value = summary.value.add()
summary_value.simple_value = value.item()
#print(name)
#print(summary_value.simple_value)
summary_value.tag = name
self.writer.add_summary(summary, self.global_step)
self.writer.flush()
self.global_step = self.global_step +1
def grad_cam(input_model, model_x, orig_x, category_index, layer_name, class_names):
output = input_model.output
final_layer = Lambda(lambda x: target_category_loss(x, category_index, len(class_names)))
output = final_layer(output)
model = Model(inputs=input_model.input, outputs=output)
loss = K.sum(model.layers[-1].output)
conv_output = model.get_layer(layer_name).output
grads = normalize(K.gradients(loss, conv_output)[0])
gradient_function = K.function([model.layers[0].input, K.learning_phase()], [conv_output, grads])
output, grads_val = gradient_function([model_x, 0])
output, grads_val = output[0, :], grads_val[0, :, :, :]
weights = np.mean(grads_val, axis=(0, 1))
cam = np.zeros(output.shape[0: 2], dtype=np.float32)
for i, w in enumerate(weights):
cam += w * output[:, :, i]
cam = np.maximum(cam, np.zeros(output.shape[0: 2], dtype=np.float32))
cam = cam.squeeze()
cam = cv2.applyColorMap(np.uint8(255 * cam / np.max(cam)), cv2.COLORMAP_JET)
cam = cv2.resize(cam, (np.shape(orig_x)[0], np.shape(orig_x)[1]))
cam = 0.4 * cam + 0.6 * orig_x
return np.uint8(cam)
def test_hierarchical_attention_layer(self):
input_shape = (3, 2, 4, 2, 6)
# record, document,section,sentence,word
input_feature_dims = (2, 3, 4, 5, 6)
# document, section, sentence, word
attention_output_dims = (3, 4, 5, 8)
# document, section, sentence, word
attention_weight_vector_dims = (3, 4, 5, 6)
# embedding
embedding_rows = 10
embedding_dim = 2
# classifier
initial_embedding = np.random.random((embedding_rows, embedding_dim))
inputs = HierarchicalAttention.build_inputs(input_shape, input_feature_dims)
hierarchical_attention = HierarchicalAttention(input_shape,input_feature_dims[0], attention_output_dims, attention_weight_vector_dims, embedding_rows, embedding_dim, initial_embedding, use_sequence_to_vector_encoder = False)
output = hierarchical_attention(inputs)
self.assertEqual(shape(output), (None, 3, 8), "y")
total = 2
output_dim = 10
timesteps = input_shape[0]
x_train, _ = faked_dataset(inputs, total, timesteps, embedding_rows, output_dim)
# build feed_dic
feed_dict = {}
for i in range(len(inputs)):
feed_dict[inputs[i]] = x_train[i]
feed_dict[K.learning_phase()] = 1
y_out = run(output, feed_dict = feed_dict)
self.assertEquals(y_out.shape , (2, 3, 8), "y_out")
def build_model_policy_value(self, model, max_cache_size=100000):
from collections import OrderedDict
cache = OrderedDict()
from tensorflow.python.keras import backend as K
get_output = K.function([model.input, K.learning_phase()], [model.output[0], model.output[1]])
def model_policy_value(input_array):
key = input_array.tobytes()
if key in cache:
cache.move_to_end(key, last=True)
return cache[key]
input_array = input_array.reshape((-1, 54, 6))
#input_array = np.rollaxis(input_array, 2, 1)
policy, value = model.predict(input_array)
#policy, value = get_output([input_array, 0])
policy = policy.reshape((12,))
value = value[0, 0]
cache[key] = (policy, value)
if len(cache) > max_cache_size:
cache.popitem(last=False)
return policy, value
return model_policy_value
def test_hierarchical_attention_with_classifier_layer(self):
input_shape = (3, 2, 4, 2, 6)
# record, document,section,sentence,word
input_feature_dims = (2, 3, 4, 5, 6)
# document, section, sentence, word
attention_output_dims = (3, 4, 5, 8)
# document, section, sentence, word
attention_weight_vector_dims = (3, 4, 5, 6)
# embedding
embedding_rows = 10
embedding_dim = 2
initial_embedding = np.random.random((embedding_rows, embedding_dim))
# classifier
output_dim = 5
hidden_unit_numbers = (5, 20) # 5--> first hidden layer, 20 --> second hidden layer
hidden_unit_activation_functions = ("relu", "relu")
use_sequence_to_vector_encoder = False
inputs = HierarchicalAttention.build_inputs(input_shape, input_feature_dims)
classifier = ClassifierWithHierarchicalAttention(input_shape,input_feature_dims[0], attention_output_dims, attention_weight_vector_dims,
embedding_rows, embedding_dim, initial_embedding, use_sequence_to_vector_encoder,
output_dim, hidden_unit_numbers, hidden_unit_activation_functions)
output = classifier(inputs)
total = 2
timesteps = input_shape[0]
x_train, _ = faked_dataset(inputs, total, timesteps, embedding_rows, output_dim)
# build feed_dic
feed_dict = {}
for i in range(len(inputs)):
feed_dict[inputs[i]] = x_train[i]
feed_dict[K.learning_phase()] = 1
y_out = run(output, feed_dict = feed_dict)
self.assertEqual(y_out.shape , (total , timesteps, output_dim), "y_out")
def get_network_layer_output(model, dataInput, layerNum, **kwargs):
"""
:param model:
:param dataInput:
:param layerNum:
:param kwargs:
:return:
"""
get_output = K.function([model.layers[0].input, K.learning_phase()],
[model.layers[layerNum].output])
phase = kwargs.get('phase', None)
if phase is None or phase == 'test':
# output in test mode = 0
layer_output = get_output([dataInput, 0])[0]
elif phase == 'train':
# output in train mode = 1
layer_output = get_output([dataInput, 1])[0]
else:
raise RuntimeError("invalid phase passed to get_network_layer_output")
return layer_output
def get_training_function(self, x, y):
model = self.model
cost, grads = self.get_cost_grads()
outs = [cost]
if type(grads) in {list, tuple}:
outs += grads
else:
outs.append(grads)
fn = K.function(
inputs=[],
outputs=outs,
givens={
model.inputs[0]: x,
model.targets[0]: y,
model.sample_weights[0]: np.ones(
(x.shape[0],
),
dtype=np.float32),
K.learning_phase(): np.uint8(1)})
def train_fn(theta):
self.set_model_params(theta)
cost_grads = fn([])
cost = np.asarray(cost_grads[0], dtype=np.float64)
grads = np.asarray(self.flatten_grads(cost_grads[1:]), dtype=np.float64)
return cost, grads
return train_fn
def extract(matlabfile,outfile):
res_ims = []
with open(matlabfile) as matfile:
mat = sio.loadmat(matfile)
# ys = mat['labels']
ims = mat['ims'][0]
for im in ims:
im = cv2.resize(im, (224, 224)).astype(np.float32)
im[:,:,0] -= 103.939
im[:,:,1] -= 116.779
im[:,:,2] -= 123.68
im = im.transpose((2,0,1))
im = np.expand_dims(im, axis=0)
res_ims.append(im)
#%%
layer_outputs = []
model = vgg.VGG_19('vgg19_weights.h5')
sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True) #not used since the weights are pre-trained
model.compile(optimizer=sgd, loss='categorical_crossentropy')
#get_37th_layer_output = K.function([model.layers[0].input], [model.layers[38].output]) #37 = flatten
get_layer_output = K.function([model.layers[0].input, K.learning_phase()], [model.layers[41].output]) #41 = last layer-1
i = 0
l = '('+str(len(res_ims))+')'
for im in res_ims:
print 'progress',str(i)+l
i += 1
# layer_output = get_37th_layer_output([im])[0][0]
layer_output = get_layer_output([im, 1])[0][0]
layer_outputs.append(layer_output)
Xtrain = np.array(layer_outputs).T
np.save(outfile,Xtrain)
def step(self, encoder_masks, img_data, zero_paddings, decoder_inputs, target_weights,
bucket_id, forward_only):
# Check if the sizes match.
encoder_size, decoder_size = self.buckets[bucket_id]
if len(decoder_inputs) != decoder_size:
raise ValueError("Decoder length must be equal to the one in bucket,"
" %d != %d." % (len(decoder_inputs), decoder_size))
if len(target_weights) != decoder_size:
raise ValueError("Weights length must be equal to the one in bucket,"
" %d != %d." % (len(target_weights), decoder_size))
# Input feed: encoder inputs, decoder inputs, target_weights, as provided.
input_feed = {}
if not forward_only:
input_feed[K.learning_phase()] = 0
else:
input_feed[K.learning_phase()] = 0
input_feed[self.img_data.name] = img_data
input_feed[self.zero_paddings.name] = zero_paddings
for l in xrange(decoder_size):
input_feed[self.decoder_inputs[l].name] = decoder_inputs[l]
input_feed[self.target_weights[l].name] = target_weights[l]
for l in xrange(encoder_size):
input_feed[self.encoder_masks[l].name] = encoder_masks[l]
# Since our targets are decoder inputs shifted by one, we need one more.
last_target = self.decoder_inputs[decoder_size].name
input_feed[last_target] = np.zeros([self.batch_size], dtype=np.int32)
# Output feed: depends on whether we do a backward step or not.
if not forward_only:
output_feed = [self.updates[bucket_id], # Update Op that does SGD.
#self.gradient_norms[bucket_id], # Gradient norm.
self.attention_decoder_model.losses[bucket_id]]
output_feed += self.keras_updates
else:
output_feed = [self.attention_decoder_model.losses[bucket_id]] # Loss for this batch.
for l in xrange(decoder_size): # Output logits.
output_feed.append(self.attention_decoder_model.outputs[bucket_id][l])
if self.visualize:
output_feed += self.attention_decoder_model.attention_weights_histories[bucket_id]
outputs = self.sess.run(output_feed, input_feed)
if not forward_only:
return None, outputs[1], None, None # Gradient norm, loss, no outputs, no attentions.
else:
return None, outputs[0], outputs[1:(1+self.buckets[bucket_id][1])], outputs[(1+self.buckets[bucket_id][1]):] # No gradient norm, loss, outputs, attentions.
def train_g(images, z, counter, sess=sess):
outputs = [loss, G_dec, t_sum, t_optim]
outs = sess.run(outputs, feed_dict={Img: images, K.learning_phase(): 1})
gl, samples, sums = outs[:3]
writer.add_summary(sums, counter)
images = images.reshape((-1, 80, 160, 3))[:64]
samples = samples.reshape((-1, 80, 160, 3))[:64]
return gl, samples, images
def detect_image(self, image):
start = timer()
if self.model_image_size != (None, None):
assert self.model_image_size[
0] % 32 == 0, 'Multiples of 32 required'
assert self.model_image_size[
1] % 32 == 0, 'Multiples of 32 required'
boxed_image = letterbox_image(
image, tuple(reversed(self.model_image_size)))
else:
new_image_size = (image.width - (image.width % 32),
image.height - (image.height % 32))
boxed_image = letterbox_image(image, new_image_size)
image_data = np.array(boxed_image, dtype='float32')
print(image_data.shape)
image_data /= 255.
image_data = np.expand_dims(image_data, 0) # Add batch dimension.
out_boxes, out_scores, out_classes = self.sess.run(
[self.boxes, self.scores, self.classes],
feed_dict={
self.yolo_model.input: image_data,
self.input_image_shape: [image.size[1], image.size[0]],
K.learning_phase(): 0
})
print('Found {} boxes for {}'.format(len(out_boxes), 'img'))
font = ImageFont.truetype(font='arial.ttf',
size=np.floor(3e-2 * image.size[1] +
0.5).astype('int32'))
thickness = (image.size[0] + image.size[1]) // 300
if os.path.isfile("좌표.txt"):
os.remove("좌표.txt")
print('이건 있어야 하는 코드야!')
for i, c in reversed(list(enumerate(out_classes))):
predicted_class = self.class_names[c]
box = out_boxes[i]
score = out_scores[i]
label = '{} {:.2f}'.format(predicted_class, score)
draw = ImageDraw.Draw(image)
label_size = draw.textsize(label, font)
top, left, bottom, right = box
top = max(0, np.floor(top + 0.5).astype('int32'))
left = max(0, np.floor(left + 0.5).astype('int32'))
bottom = min(image.size[1], np.floor(bottom + 0.5).astype('int32'))
right = min(image.size[0], np.floor(right + 0.5).astype('int32'))
print("Label = ", label, "좌표(left,top) ,(right,bottom)",
(left, top), (right, bottom))
sliced_label = label[6:10]
print(sliced_label)
with open("keras_yolo3/result/좌표.txt", "w+") as f:
f.write('%d' % left)
f.write(' %d' % top)
f.write(' %d' % right)
f.write(' %d' % bottom)
f.write('\n')
print('result폴더에 텍스트 파일 생성 성공')
if top - label_size[1] >= 0:
text_origin = np.array([left, top - label_size[1]])
else:
text_origin = np.array([left, top + 1])
# My kingdom for a good redistributable image drawing library.
for i in range(thickness):
draw.rectangle([left + i, top + i, right - i, bottom - i],
outline=self.colors[c])
draw.rectangle(
[tuple(text_origin),
tuple(text_origin + label_size)],
fill=self.colors[c])
draw.text(text_origin, label, fill=(0, 0, 0), font=font)
del draw
end = timer()
print(end - start)
#
#
return image
def get_output_mask(model, layer, X_batch):
get_output_mask = K.function(
[model.layers[0].input, K.learning_phase()],
model.layers[layer].output_mask)
output_mask = get_output_mask([X_batch, 0])
return output_mask
def detect_image(self, image, show_stats=True):
start = timer()
global vehicleId
if self.model_image_size != (None, None):
assert self.model_image_size[
0] % 32 == 0, "Multiples of 32 required"
assert self.model_image_size[
1] % 32 == 0, "Multiples of 32 required"
boxed_image = letterbox_image(
image, tuple(reversed(self.model_image_size)))
else:
new_image_size = (
image.width - (image.width % 32),
image.height - (image.height % 32),
)
boxed_image = letterbox_image(image, new_image_size)
image_data = np.array(boxed_image, dtype="float32")
if show_stats:
print(image_data.shape)
image_data /= 255.0
image_data = np.expand_dims(image_data, 0) # Add batch dimension.
out_boxes, out_scores, out_classes = self.sess.run(
[self.boxes, self.scores, self.classes],
feed_dict={
self.yolo_model.input: image_data,
self.input_image_shape: [image.size[1], image.size[0]],
K.learning_phase(): 0,
},
)
if show_stats:
print("Found {} boxes for {}".format(len(out_boxes), "img"))
out_prediction = []
font_path = os.path.join(os.path.dirname(__file__),
"font/FiraMono-Medium.otf")
font = ImageFont.truetype(font=font_path,
size=np.floor(3e-2 * image.size[1] +
0.5).astype("int32"))
thickness = (image.size[0] + image.size[1]) // 300
for i, c in reversed(list(enumerate(out_classes))):
if self.class_names[c] == 'car' or self.class_names[
c] == 'motorbike' or self.class_names[
c] == 'truck' or self.class_names[c] == 'bus':
predicted_class = self.class_names[c]
box = out_boxes[i]
score = out_scores[i]
if score > 0.8:
label = "{} {:.2f}".format(predicted_class, score)
draw = ImageDraw.Draw(image)
label_size = draw.textsize(label, font)
top, left, bottom, right = box
top = max(0, np.floor(top + 0.5).astype("int32"))
left = max(0, np.floor(left + 0.5).astype("int32"))
bottom = min(image.size[1],
np.floor(bottom + 0.5).astype("int32"))
right = min(image.size[0],
np.floor(right + 0.5).astype("int32"))
# image was expanded to model_image_size: make sure it did not pick
# up any box outside of original image (run into this bug when
# lowering confidence threshold to 0.01)
if top > image.size[1] or right > image.size[0]:
continue
if show_stats:
print(label, (left, top), (right, bottom))
# Predicting Plate
out_prediction.append([left, top, right, bottom, c, score])
if top - label_size[1] >= 0:
text_origin = np.array([left, top - label_size[1]])
else:
text_origin = np.array([left, bottom])
# My kingdom for a good redistributable image drawing library.
for i in range(thickness):
draw.rectangle(
[left + i, top + i, right - i, bottom - i],
outline=self.colors[c])
draw.rectangle(
[tuple(text_origin),
tuple(text_origin + label_size)],
fill=self.colors[c],
)
draw.text(text_origin, label, fill=(0, 0, 0), font=font)
del draw
else:
continue
end = timer()
if show_stats:
print("Time spent: {:.3f}sec".format(end - start))
return out_prediction, image
def predict(sess, image_file):
image, image_data = preprocess_image("images/" + image_file, model_image_size = (608, 608))
out_scores, out_boxes, out_classes = sess.run([scores, boxes, classes], feed_dict={yolo_model.input: image_data, K.learning_phase(): 0})
print('Found {} boxes for {}'.format(len(out_boxes), image_file))
# Generate colors for drawing bounding boxes.
colors = generate_colors(class_names)
# Draw bounding boxes on the image file
draw_boxes(image, out_scores, out_boxes, out_classes, class_names, colors)
# Save the predicted bounding box on the image
image.save(os.path.join("out", image_file), quality=90)
# Display the results in the notebook
output_image = scipy.misc.imread(os.path.join("out", image_file))
plt.figure(figsize=(12,12))
imshow(output_image)
return out_scores, out_boxes, out_classes
x_test = x_test.astype('float32')
x_test /= 255
fx_test = x_test.reshape(y_test.shape[0], img_rows * img_cols)
y_test = keras.utils.to_categorical(y_test, 10)
score = model.evaluate(fx_test, y_test, verbose=1)
####################################################################################################################
########################################################################################################################
inp = model.input
#outputs = [model.layers[3].output]
outputs = [model.layers[4].output]
functor = K.function([inp] + [K.learning_phase()], outputs)
layer_outs = functor([fx_test, 0.])
####################################################################################################################
print('Test loss:', score[0])
print('Test accuracy:', score[1]) # plotting the metrics
model.summary()
model.save('dffnn10class_modified.h5')
fig = plt.figure()
plt.subplot(2, 1, 1)
plt.plot(model_log.history['acc'])
plt.plot(model_log.history['val_acc'])
plt.title('model accuracy')
plt.ylabel('accuracy')
def infer(queries, db):
# Query 개수: 195
# Reference(DB) 개수: 1,127
# Total (query + reference): 1,322
queries, query_img, references, reference_img = preprocess(queries, db)
print(
'test data load queries {} query_img {} references {} reference_img {}'
.format(len(queries), len(query_img), len(references),
len(reference_img)))
queries = np.asarray(queries)
query_img = np.asarray(query_img)
references = np.asarray(references)
reference_img = np.asarray(reference_img)
query_img = query_img.astype('float32')
query_img /= 255
reference_img = reference_img.astype('float32')
reference_img /= 255
get_feature_layer = K.function([model.layers[0].input] +
[K.learning_phase()],
[model.layers[-1].output])
print('inference start: ', opt.infer_dist)
# inference
query_vecs = get_feature_layer([query_img, 0])[0]
n_epoch = reference_img.shape[0] // batch_size
reference_vecs = np.zeros([0, opt.embd_dim])
offset = 0
for i in range(n_epoch):
reference_img_batch = reference_img[offset:offset + batch_size]
reference_vecs = np.concatenate([
reference_vecs,
get_feature_layer([reference_img_batch, 0])[0]
])
offset += batch_size
# l2 normalization
query_vecs = l2_normalize(query_vecs)
reference_vecs = l2_normalize(reference_vecs)
if opt.infer_dist == 'l2':
# l2 distance
sim_matrix = []
for query_vec in query_vecs:
dist = np.sum(np.absolute(query_vec - reference_vecs), axis=1)
sim_matrix.append(dist)
sim_matrix = np.array(sim_matrix)
else:
# Calculate cosine similarity
sim_matrix = np.dot(query_vecs, reference_vecs.T)
retrieval_results = {}
for (i, query) in enumerate(queries):
query = query.split('/')[-1].split('.')[0]
sim_list = zip(references, sim_matrix[i].tolist())
sorted_sim_list = sorted(sim_list,
key=lambda x: x[1],
reverse=True)
ranked_list = [
k.split('/')[-1].split('.')[0] for (k, v) in sorted_sim_list
] # ranked list
retrieval_results[query] = ranked_list
print('done')
return list(
zip(range(len(retrieval_results)), retrieval_results.items()))
def detect_image(self, image, out_img=False):
"""
:param out_img 是否输出图片
"""
start = timer()
if self.model_image_size != (None, None):
assert self.model_image_size[
0] % 32 == 0, 'Multiples of 32 required'
assert self.model_image_size[
1] % 32 == 0, 'Multiples of 32 required'
boxed_image = letterbox_image(
image, tuple(reversed(self.model_image_size)))
else:
new_image_size = (image.width - (image.width % 32),
image.height - (image.height % 32))
boxed_image = letterbox_image(image, new_image_size)
image_data = np.array(boxed_image, dtype='float32')
debug_print(image_data.shape)
image_data /= 255.
image_data = np.expand_dims(image_data, 0) # Add batch dimension.
out_bboxes, out_scores, out_classes = None, None, None
with graph.as_default():
out_bboxes, out_scores, out_classes = self.sess.run(
[self.bboxes, self.scores, self.classes],
feed_dict={
self.yolo_model.input: image_data,
self.input_image_shape: [image.size[1], image.size[0]],
K.learning_phase(): 0
})
res_data = {
'bboxes': out_bboxes,
'scores': out_scores,
'classes': out_classes,
}
if out_img is False:
return None, res_data
debug_print('Found {} bboxes for {}'.format(len(out_bboxes), 'img'))
font = ImageFont.truetype(font='font/FiraMono-Medium.otf',
size=np.floor(3e-2 * image.size[1] +
0.5).astype('int32'))
thickness = (image.size[0] + image.size[1]) // 300
for i, c in reversed(list(enumerate(out_classes))):
predicted_class = self.class_names[c]
box = out_bboxes[i]
score = out_scores[i]
label = '{} {:.2f}'.format(predicted_class, score)
draw = ImageDraw.Draw(image)
label_size = draw.textsize(label, font)
top, left, bottom, right = box
top = max(0, np.floor(top + 0.5).astype('int32'))
left = max(0, np.floor(left + 0.5).astype('int32'))
bottom = min(image.size[1], np.floor(bottom + 0.5).astype('int32'))
right = min(image.size[0], np.floor(right + 0.5).astype('int32'))
debug_print(label, (left, top), (right, bottom))
if top - label_size[1] >= 0:
text_origin = np.array([left, top - label_size[1]])
else:
text_origin = np.array([left, top + 1])
# My kingdom for a good redistributable image drawing library.
for i in range(thickness):
draw.rectangle([left + i, top + i, right - i, bottom - i],
outline=self.colors[c])
draw.rectangle(
[tuple(text_origin),
tuple(text_origin + label_size)],
fill=self.colors[c])
draw.text(text_origin, label, fill=(0, 0, 0), font=font)
del draw
end = timer()
debug_print(end - start)
return image, res_data
# load the model
model = load_model('data/model-' + 'imap-mail-embedonly' + '.h5')
model.summary()
pred = model.predict(all_words)
layer_list = []
for layer in model.layers:
layer_list.append(layer)
print(layer.name)
# retrieve the embedding-layer
l = model.get_layer('embedding_1')
extended_input = [K.learning_phase()] + [l.input]
layer_f = K.function(extended_input, [l.output])
input_tensor = [0] + [all_words]
l_out = layer_f(input_tensor)
idx = 0
plt.title("Layer: %s" % l.name)
plt.imshow(l_out[0][0][0:400])
plt.show()
vocab_vec = l_out[0].reshape(400, 30)
pca = PCA(n_components=6)
result = pca.fit_transform(vocab_vec)
plt.scatter(result[:, 0], result[:, 1])
plt.show()
def _average_gradient_norm(model, X_train, y_train, batch_size):
# just checking if the model was already compiled
if not hasattr(model, "train_function"):
raise RuntimeError("You must compile your model before using it.")
weights = model.trainable_weights # weight tensors
get_gradients = model.optimizer.get_gradients(model.total_loss,
weights) # gradient tensors
input_tensors = [
# input data
model.inputs[0],
# how much to weight each sample by
model.sample_weights[0],
# labels
model.targets[0],
# train or test mode
K.learning_phase()
]
grad_fct = K.function(inputs=input_tensors, outputs=get_gradients)
steps = 0
total_norm = 0
s_w = None
nb_steps = X_train.shape[0] // batch_size
if X_train.shape[0] % batch_size == 0:
pad_last = False
else:
pad_last = True
def generator(X_train, y_train, pad_last):
for i in range(nb_steps):
X = X_train[i * batch_size:(i + 1) * batch_size, ...]
y = y_train[i * batch_size:(i + 1) * batch_size, ...]
yield (X, y)
if pad_last:
X = X_train[nb_steps * batch_size:, ...]
y = y_train[nb_steps * batch_size:, ...]
yield (X, y)
datagen = generator(X_train, y_train, pad_last)
while steps < nb_steps:
X, y = next(datagen)
# set sample weights to one
# for every input
if s_w is None:
s_w = np.ones(X.shape[0])
gradients = grad_fct([X, s_w, y, 0])
total_norm += np.sqrt(np.sum([np.sum(np.square(g))
for g in gradients]))
steps += 1
if pad_last:
X, y = next(datagen)
# set sample weights to one
# for every input
if s_w is None:
s_w = np.ones(X.shape[0])
gradients = grad_fct([X, s_w, y, 0])
total_norm += np.sqrt(np.sum([np.sum(np.square(g))
for g in gradients]))
steps += 1
return total_norm / float(steps)
yolo_outputs = yolo_head(yolo_model.output, anchors, len(class_names))
#Now yolo_eval function selects the best boxes using filtering and non-max suppression techniques.
# If you want to dive in more to see how this works, refer keras_yolo.py file in yad2k/models
boxes, scores, classes = yolo_eval(yolo_outputs, image_shape)
# Initiate a session
sess = K.get_session()
#Preprocess the input image before feeding into the convolutional network
image, image_data = preprocess_image("images/" + input_image_name,
model_image_size=(608, 608))
#Run the session
out_scores, out_boxes, out_classes = sess.run([scores, boxes, classes],
feed_dict={
yolo_model.input: image_data,
K.learning_phase(): 0
})
#Print the results
print('Found {} boxes for {}'.format(len(out_boxes), input_image_name))
#Produce the colors for the bounding boxs
colors = generate_colors(class_names)
#Draw the bounding boxes
draw_boxes(image, out_scores, out_boxes, out_classes, class_names, colors)
#Apply the predicted bounding boxes to the image and save it
image.save(os.path.join("out", input_image_name), quality=90)
output_image = scipy.misc.imread(os.path.join("out", input_image_name))
imshow(output_image)
def build_model(self):
# Build a critic (Q value) network that maps (state, action) pairs -> Q-values.
# Define input layers
# states = layers.Input(shape=(self.state_size,), name='states')
states = layers.Input(
shape=self.state_size,
name='states') # initial (635, 800, 3) resize(89, 120,3)
actions = layers.Input(shape=(self.action_size, ), name='actions')
### Add hidden layer(s) for state pathway ###
# Normalisation
norma = layers.Lambda(lambda img: img / 255.0)(states)
conv1 = layers.Conv2D(filters=32,
kernel_size=8,
strides=4,
activation='relu')(norma) # hidd1 state
pool1 = layers.MaxPooling2D(pool_size=2)(conv1)
b_norm1 = layers.BatchNormalization()(pool1)
conv2 = layers.Conv2D(filters=64,
kernel_size=4,
strides=2,
padding='same',
activation='relu')(b_norm1) # hidd2 state
pool2 = layers.MaxPooling2D(pool_size=2)(conv2)
b_norm2 = layers.BatchNormalization()(pool2)
# drop = layers.Dropout(0.3)
conv3 = layers.Conv2D(filters=128,
kernel_size=3,
strides=1,
padding='same',
activation='relu')(b_norm2) # hidd3 state
pool3 = layers.MaxPooling2D(pool_size=2)(conv3)
b_norm3 = layers.BatchNormalization()(pool3)
flat = layers.Flatten()(b_norm3) # (conv3)
net_states = layers.Dense(256, activation='relu')(
flat) # could be 'relu' activation # hidd4 state
### Add hidden layer(s) for action pathway ###
net_actions = layers.Dense(units=50,
activation='relu')(actions) # hidd1 action
net_actions = layers.BatchNormalization()(net_actions)
# drop = layers.Dropout(0.3)
net_actions = layers.Dense(units=128, activation='relu')(
net_actions) # hidd2 action
net_actions = layers.BatchNormalization()(net_actions)
net_actions = layers.Dense(units=128, activation='relu')(
net_actions) # hidd3 action
net_actions = layers.BatchNormalization()(net_actions)
net_actions = layers.Dense(units=256, activation='relu')(
net_actions) # hidd4 action
net_actions = layers.BatchNormalization()(net_actions)
# Combine state and action pathways
net = layers.Add()([net_states, net_actions])
net = layers.Activation('relu')(net)
# Add more layers to the combined network if needed
net = layers.Dense(units=32, activation='relu')(net)
# Add final output layer to produce action values (Q values)
Q_values = layers.Dense(units=1, name='q_values')(net)
net = layers.BatchNormalization()(net)
# Create Keras model
self.model = models.Model(inputs=[states, actions], outputs=Q_values)
# Print summary of cnn model
# print("Summary of Critic model:\n", self.model.summary())
# plot graph
# plot_model(self.model, to_file='critic_neural_network.png')
# Define optimizer and compile model for training with built-in loss function
optimizer = optimizers.Adam(0.0001) # 0.013
self.model.compile(optimizer=optimizer, loss='mse')
# Compute action gradients (derivative of Q values w.r.t. to actions)
action_gradients = K.gradients(Q_values, actions)
# Define an additional function to fetch action gradients (to be used by actor model)
self.get_action_gradients = K.function(
inputs=[*self.model.input, K.learning_phase()],
outputs=action_gradients)
def build_model(self):
"""Build an actor (policy) network that maps states -> actions."""
# Define input layer (states)
states = layers.Input(shape=(self.state_size, ), name='states')
# Add hidden layers
net = layers.Dense(units=128,
activation='relu',
kernel_initializer='glorot_uniform',
kernel_regularizer=regularizers.l2(0.01))(states)
net = layers.BatchNormalization()(net)
net = layers.Dropout(0.01)(net)
# Try different layer sizes, activations, add batch normalization, regularizers, etc.
net = layers.Dense(units=256,
activation='relu',
kernel_initializer='glorot_uniform',
kernel_regularizer=regularizers.l2(0.01))(net)
net = layers.BatchNormalization()(net)
net = layers.Dropout(0.01)(net)
net = layers.Dense(units=256,
activation='relu',
kernel_initializer='glorot_uniform',
kernel_regularizer=regularizers.l2(0.01))(net)
net = layers.BatchNormalization()(net)
net = layers.Dropout(0.01)(net)
net = layers.Dense(units=128,
activation='relu',
kernel_initializer='glorot_uniform',
kernel_regularizer=regularizers.l2(0.01))(net)
net = layers.BatchNormalization()(net)
net = layers.Dropout(0.01)(net)
# Add final output layer with sigmoid activation
raw_actions = layers.Dense(
units=self.action_size,
activation='sigmoid',
kernel_initializer='random_uniform',
kernel_regularizer=regularizers.l2(0.01)
# ,activity_regularizer=regularizers.l2(0.01)
,
name='raw_actions')(net)
# Scale [0, 1] output for each action dimension to proper range
actions = layers.Lambda(lambda x:
(x * self.action_range) + self.action_low,
name='actions')(raw_actions)
# Create Keras model
self.model = models.Model(inputs=states, outputs=actions)
# Define loss function using action value (Q value) gradients
action_gradients = layers.Input(shape=(self.action_size, ))
loss = K.mean(-action_gradients * actions)
# TODO: check loss function
# Incorporate any additional losses here (e.g. from regularizers)
# Define optimizer and training function
optimizer = optimizers.Adam(lr=0.0001, clipvalue=0.5)
updates_op = optimizer.get_updates(params=self.model.trainable_weights,
loss=loss)
self.train_fn = K.function(
inputs=[self.model.input, action_gradients,
K.learning_phase()],
outputs=[loss],
updates=updates_op)
def detect_image(self, image, time):
global hang
global hang1
global hang2
global hang3
global hang4
start = timer()
if self.model_image_size != (None, None):
assert self.model_image_size[
0] % 32 == 0, 'Multiples of 32 required'
assert self.model_image_size[
1] % 32 == 0, 'Multiples of 32 required'
boxed_image = letterbox_image(
image, tuple(reversed(self.model_image_size)))
else:
new_image_size = (image.width - (image.width % 32),
image.height - (image.height % 32))
boxed_image = letterbox_image(image, new_image_size)
image_data = np.array(boxed_image, dtype='float32')
print(image_data.shape)
image_data /= 255.
image_data = np.expand_dims(image_data, 0) # Add batch dimension.
out_boxes, out_scores, out_classes = self.sess.run(
[self.boxes, self.scores, self.classes],
feed_dict={
self.yolo_model.input: image_data,
self.input_image_shape: [image.size[1], image.size[0]],
K.learning_phase(): 0
})
print('Found {} boxes for {}'.format(len(out_boxes), 'img'))
font = ImageFont.truetype(font='font/FiraMono-Medium.otf',
size=np.floor(3e-2 * image.size[1] +
0.5).astype('int32'))
thickness = (image.size[0] + image.size[1]) // 300
for i, c in reversed(list(enumerate(out_classes))):
predicted_class = self.class_names[c]
box = out_boxes[i]
score = out_scores[i]
label = '{} {:.2f}'.format(predicted_class, score)
draw = ImageDraw.Draw(image)
label_size = draw.textsize(label, font)
top, left, bottom, right = box
top = max(0, np.floor(top + 0.5).astype('int32'))
left = max(0, np.floor(left + 0.5).astype('int32'))
bottom = min(image.size[1], np.floor(bottom + 0.5).astype('int32'))
right = min(image.size[0], np.floor(right + 0.5).astype('int32'))
print(label, (left, top), (right, bottom))
print(left)
if (label.split(' ')[0] == 'basketball'):
ws.write(hang, 0, label.split(' ')[0])
ws.write(hang, 1, label.split(' ')[1])
ws.write(hang, 2, int(left))
ws.write(hang, 3, int(top))
ws.write(hang, 4, int(right))
ws.write(hang, 5, int(bottom))
ws.write(hang, 6, time)
#ws.write(hang,7,str((int(left)+int(right))/2) )
#ws.write(hang,8,str((1080-int(top) + 1080-int(bottom))/2))
hang = hang + 1
if (label.split(' ')[0] == 'hand'):
ws2.write(hang1, 0, label.split(' ')[0])
ws2.write(hang1, 1, label.split(' ')[1])
ws2.write(hang1, 2, int(left))
ws2.write(hang1, 3, int(top))
ws2.write(hang1, 4, int(right))
ws2.write(hang1, 5, int(bottom))
ws2.write(hang1, 6, time)
hang1 = hang1 + 1
if (label.split(' ')[0] == 'arm'):
ws3.write(hang2, 0, label.split(' ')[0])
ws3.write(hang2, 1, label.split(' ')[1])
ws3.write(hang2, 2, int(left))
ws3.write(hang2, 3, int(top))
ws3.write(hang2, 4, int(right))
ws3.write(hang2, 5, int(bottom))
ws3.write(hang2, 6, time)
hang2 = hang2 + 1
if (label.split(' ')[0] == 'hoop'):
ws4.write(hang3, 0, label.split(' ')[0])
ws4.write(hang3, 1, label.split(' ')[1])
ws4.write(hang3, 2, int(left))
ws4.write(hang3, 3, int(top))
ws4.write(hang3, 4, int(right))
ws4.write(hang3, 5, int(bottom))
ws4.write(hang3, 6, time)
#ws4.write(hang, 7, str((int(left) + int(right))/2))
#ws4.write(hang, 8, str((1080 - int(top) + 1080 - int(bottom))/2))
hang3 = hang3 + 1
if (label.split(' ')[0] == 'head'):
ws5.write(hang4, 0, label.split(' ')[0])
ws5.write(hang4, 1, label.split(' ')[1])
ws5.write(hang4, 2, int(left))
ws5.write(hang4, 3, int(top))
ws5.write(hang4, 4, int(right))
ws5.write(hang4, 5, int(bottom))
ws5.write(hang4, 6, time)
hang4 = hang4 + 1
if top - label_size[1] >= 0:
text_origin = np.array([left, top - label_size[1]])
else:
text_origin = np.array([left, top + 1])
# My kingdom for a good redistributable image drawing library.
for i in range(thickness):
draw.rectangle([left + i, top + i, right - i, bottom - i],
outline=self.colors[c])
draw.rectangle(
[tuple(text_origin),
tuple(text_origin + label_size)],
fill=self.colors[c])
draw.text(text_origin, label, fill=(0, 0, 0), font=font)
del draw
end = timer()
print(end - start)
return image
def get_input(model, layer, X_batch):
get_input = K.function(
[model.layers[0].input, K.learning_phase()],
model.layers[layer].input)
_input = get_input([X_batch, 0])
return _input
def detect_image(self,
image,
image_name="#NA#",
lbl_box="",
output_detections=False):
start = timer()
if self.model_image_size != (None, None):
assert self.model_image_size[
0] % 32 == 0, 'Multiples of 32 required'
assert self.model_image_size[
1] % 32 == 0, 'Multiples of 32 required'
boxed_image = letterbox_image(
image, tuple(reversed(self.model_image_size)))
else:
new_image_size = (image.width - (image.width % 32),
image.height - (image.height % 32))
boxed_image = letterbox_image(image, new_image_size)
image_data = np.array(boxed_image, dtype='float32')
print(image_data.shape)
image_data /= 255.
image_data = np.expand_dims(image_data, 0) # Add batch dimension.
out_boxes, out_scores, out_classes = self.sess.run(
[self.boxes, self.scores, self.classes],
feed_dict={
self.yolo_model.input: image_data,
self.input_image_shape: [image.size[1], image.size[0]],
K.learning_phase(): 0
})
print('Found {} boxes for {}'.format(len(out_boxes), image_name))
font = ImageFont.truetype(font='font/FiraMono-Medium.otf',
size=np.floor(3e-2 * image.size[1] +
0.5).astype('int32'))
thickness = (image.size[0] + image.size[1]) // 300
dets = []
for i, c in reversed(list(enumerate(out_classes))):
predicted_class = self.class_names[c]
box = out_boxes[i]
score = out_scores[i]
label = '{} {:.2f}'.format(predicted_class, score)
draw = ImageDraw.Draw(image)
label_size = draw.textsize(label, font)
top, left, bottom, right = box
top = max(0, np.floor(top + 0.5).astype('int32'))
left = max(0, np.floor(left + 0.5).astype('int32'))
bottom = min(image.size[1], np.floor(bottom + 0.5).astype('int32'))
right = min(image.size[0], np.floor(right + 0.5).astype('int32'))
# print(label, (left, top), (right, bottom))
# for reuse in labeling
# print(f'###EXPORT###{image_name}|ORIG_LABEL:{lbl_box}|{left},{top},{right},{bottom},{label}')
print(
f'###EXPORT###{{"filename":"{image_name}","orig_label":"{lbl_box}","yolo_detection":"{left},{top},{right},{bottom}","yolo_label":"{" ".join(label.split(" ")[0:-1])}","yolo_score":"{label.split(" ")[-1]}"}}'
)
dets.append(
f'{{"filename":"{image_name}","orig_label":"{lbl_box}","yolo_detection":"{left},{top},{right},{bottom}","yolo_label":"{" ".join(label.split(" ")[0:-1])}","yolo_score":"{label.split(" ")[-1]}"}}'
)
if top - label_size[1] >= 0:
text_origin = np.array([left, top - label_size[1]])
else:
text_origin = np.array([left, top + 1])
# My kingdom for a good redistributable image drawing library.
for i in range(thickness):
draw.rectangle([left + i, top + i, right - i, bottom - i],
outline=self.colors[c])
draw.rectangle(
[tuple(text_origin),
tuple(text_origin + label_size)],
fill=self.colors[c])
draw.text(text_origin, label, fill=(0, 0, 0), font=font)
del draw
end = timer()
print(end - start)
if (output_detections):
return {"image": image, "detections": dets}
return image
def build_model(self):
"""Build a critic (value) network that maps (state, action) pairs -> Q-values."""
# Define input layers
states = layers.Input(shape=(self.state_size,), name='states')
actions = layers.Input(shape=(self.action_size,), name='actions')
# Parameters for critic network
ki = initializers.glorot_uniform();
kr = None
if self.model_params["use_l2"] > 0:
l2 = self.model_params["use_l2"]
kr = regularizers.l2(l2)
dropout_rate = self.model_params["dropout_rate"]
use_bias = True
use_bn = self.model_params["use_bn"]
if use_bn:
use_bias=False
act_fn = self.model_params["act_fn"]
n1 = self.model_params["layer1_n"]
n2 = self.model_params["layer2_n"]
# The critic network
s_fc1 = layers.Dense(units=n1, kernel_initializer=ki, kernel_regularizer=kr, use_bias=use_bias, name='s_fc1')(states)
if use_bn:
s_fc1 = layers.BatchNormalization()(s_fc1)
if (dropout_rate > 0):
s_fc1 = layers.Dropout(dropout_rate)(s_fc1)
s_fc1 = layers.Activation(act_fn)(s_fc1)
a_fc1 = layers.Dense(units=n1, kernel_initializer=ki, kernel_regularizer=kr, use_bias=use_bias, name='a_fc1')(actions)
if use_bn:
a_fc1 = layers.BatchNormalization()(a_fc1)
if (dropout_rate > 0):
a_fc1 = layers.Dropout(dropout_rate)(a_fc1)
a_fc1 = layers.Activation(act_fn)(a_fc1)
# Combine state and action pathways
net = layers.Add()([s_fc1, a_fc1])
net = layers.Dense(units=n2, kernel_initializer=ki, kernel_regularizer=kr, use_bias=use_bias, name='fc2')(net)
if use_bn:
net = layers.BatchNormalization()(net)
if (dropout_rate > 0):
net = layers.Dropout(dropout_rate)(net)
net = layers.Activation(act_fn)(net)
# Add final output layer to prduce action values (Q values)
ki = initializers.RandomUniform(minval=-3e-3, maxval=3e-3)
Q_values = layers.Dense(units=1, kernel_initializer=ki, kernel_regularizer=kr,
activation='linear', name='q_values')(net)
# Create Keras model
self.model = models.Model(inputs=[states, actions], outputs=Q_values)
# Define optimizer and compile model for training with built-in loss function
optimizer = optimizers.Adam(lr=0.001)
self.model.compile(optimizer=optimizer, loss='mse')
# Compute action gradients (derivative of Q values w.r.t. to actions)
action_gradients = K.gradients(Q_values, actions)
# Define an additional function to fetch action gradients (to be used by actor model)
self.get_action_gradients = K.function(
inputs=[*self.model.input, K.learning_phase()],
outputs=action_gradients)
def predict(sess, image_file, is_show_info=True, is_plot=True):
"""
运行存储在sess的计算图以预测image_file的边界框，打印出预测的图与信息。
参数：
sess - 包含了YOLO计算图的TensorFlow/Keras的会话。
image_file - 存储在images文件夹下的图片名称
返回：
out_scores - tensor类型，维度为(None,)，锚框的预测的可能值。
out_boxes - tensor类型，维度为(None,4)，包含了锚框位置信息。
out_classes - tensor类型，维度为(None,)，锚框的预测的分类索引。
"""
#图像预处理
image, image_data = yolo_utils.preprocess_image(r"E:\深度学习\第四课第三周编程作业\Car detection for Autonomous Driving\images\\" + image_file, model_image_size = (608, 608))
#运行会话并在feed_dict中选择正确的占位符.
out_scores, out_boxes, out_classes = sess.run([scores, boxes, classes], feed_dict = {yolo_model.input:image_data, K.learning_phase(): 0})
#打印预测信息
if is_show_info:
print("在" + str(image_file) + "中找到了" + str(len(out_boxes)) + "个锚框。")
#指定要绘制的边界框的颜色
colors = yolo_utils.generate_colors(class_names)
#在图中绘制边界框
yolo_utils.draw_boxes(image, out_scores, out_boxes, out_classes, class_names, colors)
#保存已经绘制了边界框的图
image.save(os.path.join("out", image_file), quality=100)
#打印出已经绘制了边界框的图
if is_plot:
output_image = scipy.misc.imread(os.path.join("out", image_file))
plt.imshow(output_image)
return out_scores, out_boxes, out_classes
def loadCNN(wf_index):
global get_output
model = Sequential()
model.add(Conv2D(nb_filters, (nb_conv, nb_conv),
padding='valid',
input_shape=(img_channels, img_rows, img_cols)))
convout1 = Activation('relu')
model.add(convout1)
model.add(Conv2D(nb_filters, (nb_conv, nb_conv)))
convout2 = Activation('relu')
model.add(convout2)
model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool)))
model.add(Dropout(0.5))
model.add(Flatten())
model.add(Dense(128))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))
'''
model.add(ZeroPadding2D((1,1),input_shape=(img_channels, img_rows, img_cols)))
model.add(Conv2D(nb_filters , (nb_conv, nb_conv), activation='relu'))
#model.add(ZeroPadding2D((1,1)))
#model.add(Conv2D(nb_filters , (nb_conv, nb_conv), activation='relu'))
model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool)))
model.add(Dropout(0.2))
#model.add(ZeroPadding2D((1,1)))
model.add(Conv2D(nb_filters , (nb_conv, nb_conv), activation='relu'))
#model.add(ZeroPadding2D((1,1)))
model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool)))
##
#model.add(Conv2D(nb_filters , (nb_conv, nb_conv), activation='relu'))
#model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool), strides=(2,2)))
model.add(Dropout(0.3))
model.add(Flatten())
###
#model.add(Dense(128))
#model.add(Activation('relu'))
#model.add(Dropout(0.5))
model.add(Dense(256))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))
'''
#sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy', optimizer='adadelta', metrics=['accuracy'])
# Model summary
model.summary()
# Model conig details
model.get_config()
#from keras.utils import plot_model
#plot_model(model, to_file='new_model.png', show_shapes = True)
if wf_index >= 0:
#Load pretrained weights
fname = WeightFileName[int(wf_index)]
print("loading ", fname)
model.load_weights(fname)
layer = model.layers[11]
get_output = K.function([model.layers[0].input, K.learning_phase()], [layer.output,])
return model
args.add_argument('--pause',
type=int,
default=0,
help='model 을 load 할때 1로 설정됩니다.')
config = args.parse_args()
# training parameters
nb_epoch = config.epochs
batch_size = config.batch_size
num_classes = 1000
input_shape = (224, 224, 3) # input image shape
""" Model """
model, base_model = get_model('triplet', 224, num_classes, opt.base_model)
bind_model(base_model, config.batch_size)
get_feature_layer = K.function([base_model.layers[0].input] +
[K.learning_phase()],
[base_model.layers[-1].output])
if config.pause:
nsml.paused(scope=locals())
bTrainmode = False
if config.mode == 'train':
bTrainmode = True
""" Load data """
print('dataset path', DATASET_PATH)
output_path = ['./img_list.pkl', './label_list.pkl']
train_dataset_path = os.path.join(DATASET_PATH, 'train/train_data')
if nsml.IS_ON_NSML:
# Caching file
def infer(queries, db):
# Query 개수: 195
# Reference(DB) 개수: 1,127
# Total (query + reference): 1,322
queries, query_img, references, reference_img = preprocess(queries, db)
print(
'test data load queries {} query_img {} references {} reference_img {}'
.format(len(queries), len(query_img), len(references),
len(reference_img)))
queries = np.asarray(queries)
query_img = np.asarray(query_img)
references = np.asarray(references)
reference_img = np.asarray(reference_img)
query_img = query_img.astype('float32')
query_img /= 255
reference_img = reference_img.astype('float32')
reference_img /= 255
get_feature_layer = K.function([model.layers[0].input] +
[K.learning_phase()],
[model.layers[-2].output])
print('inference start')
# inference
query_vecs = get_feature_layer([query_img, 0])[0]
# caching db output, db inference
db_output = './db_infer.pkl'
if os.path.exists(db_output):
with open(db_output, 'rb') as f:
reference_vecs = pickle.load(f)
else:
reference_vecs = get_feature_layer([reference_img, 0])[0]
with open(db_output, 'wb') as f:
pickle.dump(reference_vecs, f)
# l2 normalization
query_vecs = l2_normalize(query_vecs)
reference_vecs = l2_normalize(reference_vecs)
# Calculate cosine similarity
sim_matrix = np.dot(query_vecs, reference_vecs.T)
retrieval_results = {}
for (i, query) in enumerate(queries):
query = query.split('/')[-1].split('.')[0]
sim_list = zip(references, sim_matrix[i].tolist())
sorted_sim_list = sorted(sim_list,
key=lambda x: x[1],
reverse=True)
ranked_list = [
k.split('/')[-1].split('.')[0] for (k, v) in sorted_sim_list
] # ranked list
retrieval_results[query] = ranked_list
print('done')
return list(
zip(range(len(retrieval_results)), retrieval_results.items()))
regions.append(' '.join(tokens[:filter_sizes[0]]))
for i in range(filter_sizes[0], len(tokens)):
regions.append(' '.join(tokens[(i-filter_sizes[0]+1):(i+1)]))
my_review = np.array([my_review])
reg_emb = get_region_embedding([my_review,0])[0][0,:,:]
### fill the gap ###
### compute the norms of the region embeddings ###
### you may use np.linalg.norm() https://docs.scipy.org/doc/numpy/reference/generated/numpy.linalg.norm.html ###
norms = np.linalg.norm(reg_emb, axis=1)
print([list(zip(regions,norms))[idx] for idx in np.argsort(-norms).tolist()])
# = = = = = saliency map = = = = =
input_tensors = [model.input, K.learning_phase()]
### fill the gap (extract the rows of the embedding matrix from the model) ###
saliency_input = model.layers[1].output
### fill the gap (get the probability distribution over classes from the model) ###
saliency_output = model.output[0][0]
gradients = model.optimizer.get_gradients(saliency_output,saliency_input)
compute_gradients = K.function(inputs=input_tensors,outputs=gradients)
matrix = compute_gradients([my_review,0])[0][0,:,:]
### fill the gap (compute the magnitude of the partial derivatives) ###
### you may use np.absolute https://docs.scipy.org/doc/numpy/reference/generated/numpy.absolute.html ###
to_plot = np.absolute(matrix)
def grad_cam_batch(model, images, classes, layer_name, X, Y, Z, all_features, non_image_mean, non_image_std, non_images, input_layer_1, input_layer_2, dense_layer):
"""
GradCAM method to process multiple images in one run.
INPUT:
model - the model for which the gradcams should be computed
images - the batch of images for which the gradcams should be computed
classes - an array indicating the classes corresponding to the image batch
X, Y, Z - the input shapes of the images
OUTPUT
cam_mean - the average gradcam of the image batch
cam_var - the variation of the mean gradcam of the image batch
"""
# get loss, output and gradients
loss = tf.gather_nd(model.output, np.dstack([range(images.shape[0]), classes])[0])
first_input = model.get_layer(input_layer_1).input
second_input = model.get_layer(input_layer_2).input
layer_output = model.get_layer(layer_name).output
dense_layer_output = model.get_layer(dense_layer).output
grads = K.gradients(loss, layer_output)[0]
dense_grads = K.gradients(loss, dense_layer_output)[0]
# calculate class activation maps for image batch
if non_image_features == False:
# create gradient function
gradient_fn = K.function([first_input, K.learning_phase()], [layer_output, grads])
conv_output, grads_val = gradient_fn([images, learning_phase])
else:
# create gradient function
gradient_fn = K.function([first_input, second_input, K.learning_phase()], [layer_output, grads])
conv_output, grads_val = gradient_fn([images, non_images, learning_phase])
# create gradient function for dense layer
dense_gradient_fn = K.function([first_input, second_input, K.learning_phase()], [dense_layer_output, dense_grads])
dense_conv_output, dense_grads_val = dense_gradient_fn([images, non_images, learning_phase])
weights = np.mean(grads_val, axis=(1, 2, 3))
cams = np.einsum('ijklm,im->ijkl', conv_output, weights)
#dense_weights = np.mean(dense_grads_val, axis=(1))
dense_cams = np.einsum('ij,ij->i', dense_conv_output, dense_grads_val)
# process CAMs
new_cams = np.empty((images.shape[0], X, Y, Z))
for i in range(new_cams.shape[0]):
cam_i = cams[i] - cams[i].mean()
cam_i = (cam_i + 1e-10) / (np.linalg.norm(cam_i) + 1e-10)
new_cams[i] = resize(cam_i, (X, Y, Z), order=1, mode='constant', cval=0, anti_aliasing=False)
new_cams[i] = np.maximum(new_cams[i], 0)
new_cams[i] = new_cams[i] / new_cams[i].max()
# process CAMs for dense layer
new_cams_dense = np.empty((images.shape[0], X, Y, Z))
for i in range(new_cams.shape[0]):
cam_i = cams[i] - cams[i].mean()
cam_i = (cam_i + 1e-10) / (np.linalg.norm(cam_i) + 1e-10)
new_cams[i] = resize(cam_i, (X, Y, Z), order=1, mode='constant', cval=0, anti_aliasing=False)
new_cams[i] = np.maximum(new_cams[i], 0)
new_cams[i] = new_cams[i] / new_cams[i].max()
# calculate mean and variation
cam_mean = np.mean(new_cams, axis=0)
cam_var = np.var(new_cams, axis=0)
return cam_mean, cam_var
def test_yolo(image, image_file_name):
if is_fixed_size: # TODO: When resizing we can use minibatch input.
resized_image = image.resize(tuple(reversed(model_image_size)),
Image.BICUBIC)
image_data = np.array(resized_image, dtype='float32')
else:
# Due to skip connection + max pooling in YOLO_v2, inputs must have
# width and height as multiples of 32.
new_image_size = (image.width - (image.width % 32),
image.height - (image.height % 32))
resized_image = image.resize(new_image_size, Image.BICUBIC)
image_data = np.array(resized_image, dtype='float32')
print(image_data.shape)
image_data /= 255.
image_data = np.expand_dims(image_data, 0) # Add batch dimension.
out_boxes, out_scores, out_classes = sess.run(
[boxes, scores, classes],
feed_dict={
yolo_model.input: image_data,
input_image_shape: [image.size[1], image.size[0]],
K.learning_phase(): 0
})
print('Found {} boxes for {}'.format(len(out_boxes), image_file_name))
# Write data to a JSON file located within the 'output/' directory.
# This ASSUMES that the game comes from a spectated video starting at 0:00
# Else, data will not be alligned!
font = ImageFont.truetype(font='font/FiraMono-Medium.otf',
size=np.floor(3e-2 * image.size[1] +
0.5).astype('int32'))
thickness = (image.size[0] + image.size[1]) // 300
data = {}
data['timestamp'] = '0:00'
data['champs'] = {}
for i, c in reversed(list(enumerate(out_classes))):
predicted_class = class_names[c]
box = out_boxes[i]
score = out_scores[i]
if user_did_specify_champs and predicted_class not in champs_in_game:
continue
label = '{} {:.2f}'.format(predicted_class, score)
draw = ImageDraw.Draw(image)
label_size = draw.textsize(label, font)
top, left, bottom, right = box
top = max(0, np.floor(top + 0.5).astype('int32'))
left = max(0, np.floor(left + 0.5).astype('int32'))
bottom = min(image.size[1], np.floor(bottom + 0.5).astype('int32'))
right = min(image.size[0], np.floor(right + 0.5).astype('int32'))
print(label, (left, top), (right, bottom))
# Save important data to JSON.
data['champs'][predicted_class] = score
if top - label_size[1] >= 0:
text_origin = np.array([left, top - label_size[1]])
else:
text_origin = np.array([left, top + 1])
for i in range(thickness):
draw.rectangle([left + i, top + i, right - i, bottom - i],
outline=colors[c])
draw.rectangle([tuple(text_origin),
tuple(text_origin + label_size)],
fill=colors[c])
draw.text(text_origin, label, fill=(0, 0, 0), font=font)
del draw
image.save(os.path.join(output_path, image_file_name), quality=90)
def __init__(self, model):
self.f = K.function(
[model.layers[0].input, K.learning_phase()],
[model.layers[-1].output])
def add_method(self, model):
action_gradients = K.gradients(model.output[0], model.input[1])
self.get_action_gradients = K.function(
inputs=[*model.input, K.learning_phase()],
outputs=action_gradients)
def compile(self, optimizer, metrics=[]):
metrics += [mean_q]
if type(optimizer) in (list, tuple):
if len(optimizer) != 2:
raise ValueError(
'More than two optimizers provided. Please only provide a maximum of two optimizers, the first one for the actor and the second one for the critic.'
)
actor_optimizer, critic_optimizer = optimizer
else:
actor_optimizer = optimizer
critic_optimizer = clone_optimizer(optimizer)
if type(actor_optimizer) is str:
actor_optimizer = optimizers.get(actor_optimizer)
if type(critic_optimizer) is str:
critic_optimizer = optimizers.get(critic_optimizer)
assert actor_optimizer != critic_optimizer
if len(metrics) == 2 and hasattr(metrics[0], '__len__') and hasattr(
metrics[1], '__len__'):
actor_metrics, critic_metrics = metrics
else:
actor_metrics = critic_metrics = metrics
def clipped_error(y_true, y_pred):
return K.mean(huber_loss(y_true, y_pred, self.delta_clip), axis=-1)
# Compile target networks. We only use them in feed-forward mode, hence we can pass any
# optimizer and loss since we never use it anyway.
self.target_actor = clone_model(self.actor, self.custom_model_objects)
self.target_actor.compile(optimizer='sgd', loss='mse')
self.target_critic = clone_model(self.critic,
self.custom_model_objects)
self.target_critic.compile(optimizer='sgd', loss='mse')
# We also compile the actor. We never optimize the actor using Keras but instead compute
# the policy gradient ourselves. However, we need the actor in feed-forward mode, hence
# we also compile it with any optimzer and
self.actor.compile(optimizer='sgd', loss='mse')
# Compile the critic.
if self.target_model_update < 1.:
# We use the `AdditionalUpdatesOptimizer` to efficiently soft-update the target model.
critic_updates = get_soft_target_model_updates(
self.target_critic, self.critic, self.target_model_update)
critic_optimizer = AdditionalUpdatesOptimizer(
critic_optimizer, critic_updates)
self.critic.compile(optimizer=critic_optimizer,
loss=clipped_error,
metrics=critic_metrics)
# Combine actor and critic so that we can get the policy gradient.
# Assuming critic's state inputs are the same as actor's.
combined_inputs = []
state_inputs = []
for i in self.critic.input:
if i == self.critic_action_input:
combined_inputs.append([])
else:
combined_inputs.append(i)
state_inputs.append(i)
combined_inputs[self.critic_action_input_idx] = self.actor(
state_inputs)
combined_output = self.critic(combined_inputs)
updates = actor_optimizer.get_updates(
params=self.actor.trainable_weights, loss=-K.mean(combined_output))
if self.target_model_update < 1.:
# Include soft target model updates.
updates += get_soft_target_model_updates(self.target_actor,
self.actor,
self.target_model_update)
updates += self.actor.updates # include other updates of the actor, e.g. for BN
# Finally, combine it all into a callable function.
if K.backend() == 'tensorflow':
self.actor_train_fn = K.function(state_inputs +
[K.learning_phase()],
[self.actor(state_inputs)],
updates=updates)
else:
if self.uses_learning_phase:
state_inputs += [K.learning_phase()]
self.actor_train_fn = K.function(state_inputs,
[self.actor(state_inputs)],
updates=updates)
self.actor_optimizer = actor_optimizer
self.compiled = True
#y_pred = (y_pred > 0.5)
# Making the Confusion Matrix
#from sklearn.metrics import confusion_matrix
#cm = confusion_matrix(y_test, y_pred)
#from keras.models import Model
from keras import backend as Ke
inp = classifier.input
#inp=X_test[0] # input placeholder
outputs = [layer.output for layer in classifier.layers] # all layer outputs
functors = [
Ke.function([inp] + [Ke.learning_phase()], [out]) for out in outputs
] # evaluation functions
# Testing
input_shape = 1
test = data #np.random.random(input_shape)[np.newaxis,...]
layer_outs = [func([test, 1.]) for func in functors]
#np.array(layer_outs[0]).shape
recomposed_image = recompose_image(
np.reshape(np.array(layer_outs[1]), (256, 12)), 64, 48, 4, 3)
recomposed_image = recompose_image(y_pred, 64, 64, 4, 4)
#recomposed_image = recompose_image(np.reshape(np.array(layer_outs[1]),(16384,12)),512,384,4,3)
#recomposed_image = recompose_image(y_pred,512,512,4,4)
cv2.imshow('Recomposed Image', recomposed_image)
cv2.waitKey(0)
cv2.imwrite('01.png', recomposed_image * 255)
def main():
# Prepare args
args = parse_args()
num_labeled_train = args.num_labeled_train
num_test = args.num_test
ramp_up_period = args.ramp_up_period
ramp_down_period = args.ramp_down_period
num_class = args.num_class
num_epoch = args.num_epoch
batch_size = args.batch_size
weight_max = args.weight_max
learning_rate = args.learning_rate
weight_norm_flag = args.weight_norm_flag
augmentation_flag = args.augmentation_flag
whitening_flag = args.whitening_flag
trans_range = args.trans_range
# Data Preparation
train_x, train_y, test_x, test_y = load_data(args.data_path)
ret_dic = split_supervised_train(train_x, train_y, num_labeled_train)
ret_dic['test_x'] = test_x
ret_dic['test_y'] = test_y
ret_dic = make_train_test_dataset(ret_dic, num_class)
unsupervised_target = ret_dic['unsupervised_target']
supervised_label = ret_dic['supervised_label']
supervised_flag = ret_dic['train_sup_flag']
unsupervised_weight = ret_dic['unsupervised_weight']
test_y = ret_dic['test_y']
train_x, test_x = normalize_images(ret_dic['train_x'], ret_dic['test_x'])
# pre-process
if whitening_flag:
train_x, test_x = whiten_zca(train_x, test_x)
if augmentation_flag:
train_x = np.pad(train_x, ((0, 0), (trans_range, trans_range),
(trans_range, trans_range), (0, 0)),
'reflect')
# make the whole data and labels for training
# x = [train_x, supervised_label, supervised_flag, unsupervised_weight]
y = np.concatenate((unsupervised_target, supervised_label, supervised_flag,
unsupervised_weight),
axis=1)
num_train_data = train_x.shape[0]
# Build Model
if weight_norm_flag:
from lib.model_WN import build_model
from lib.weight_norm import AdamWithWeightnorm
optimizer = AdamWithWeightnorm(lr=learning_rate,
beta_1=0.9,
beta_2=0.999)
else:
from lib.model_BN import build_model
optimizer = Adam(lr=learning_rate, beta_1=0.9, beta_2=0.999)
model = build_model(num_class=num_class)
model.compile(optimizer=optimizer, loss=semi_supervised_loss(num_class))
model.metrics_tensors += model.outputs
model.summary()
get_prediction = K.function(
inputs=[model.layers[0].input,
K.learning_phase()],
outputs=[model.get_layer('activation_1').output])
# prepare weights and arrays for updates
gen_weight = ramp_up_weight(
ramp_up_period, weight_max * (num_labeled_train / num_train_data))
gen_lr_weight = ramp_down_weight(ramp_down_period)
idx_list = [v for v in range(num_train_data)]
# Training
for epoch in range(num_epoch):
print('epoch: ', epoch)
idx_list = shuffle(idx_list)
if epoch > num_epoch - ramp_down_period:
weight_down = next(gen_lr_weight)
K.set_value(model.optimizer.lr, weight_down * learning_rate)
K.set_value(model.optimizer.beta_1, 0.4 * weight_down + 0.5)
ave_loss = 0
for i in range(0, num_train_data, batch_size):
target_idx = idx_list[i:i + batch_size]
if augmentation_flag:
x1 = data_augmentation_tempen(train_x[target_idx], trans_range)
else:
x1 = train_x[target_idx]
x2 = supervised_label[target_idx]
x3 = supervised_flag[target_idx]
x4 = unsupervised_weight[target_idx]
y_t = y[target_idx]
# get the first prediction
y_t[:, 0:num_class] = get_prediction(inputs=[x1, 1])[0]
x_t = [x1, x2, x3, x4]
tr_loss, output = model.train_on_batch(x=x_t, y=y_t)
ave_loss += tr_loss
print('Training Loss: ', (ave_loss * batch_size) / num_train_data,
flush=True)
# Update phase
next_weight = next(gen_weight)
y, unsupervised_weight = update_weight(y, unsupervised_weight,
next_weight)
# Evaluation
if epoch % 5 == 0:
print('Evaluate epoch : ', epoch, flush=True)
evaluate(model, num_class, num_test, test_x, test_y)
def get_activations(model, layer, X_batch):
get_activations = K.function(
[model.layers[0].input, K.learning_phase()],
model.layers[layer].output)
activations = get_activations([X_batch, 0])
return activations
def build_model(self):
"""Build an actor (policy) network that maps states -> actions."""
# Define input layer (states)
states = layers.Input(shape=(self.state_size,), name='states')
# Parameters for actor network
ki = initializers.glorot_uniform();
kr = None
if self.model_params["use_l2"] > 0:
l2 = self.model_params["use_l2"]
kr = regularizers.l2(l2)
dropout_rate = self.model_params["dropout_rate"]
use_bias = True
use_bn = self.model_params["use_bn"]
if use_bn:
use_bias=False
act_fn = self.model_params["act_fn"]
n1 = self.model_params["layer1_n"]
n2 = self.model_params["layer2_n"]
# the actor network
fc1 = layers.Dense(units=n1, kernel_initializer=ki, kernel_regularizer=kr, use_bias=use_bias, name='fc1')(states)
if use_bn:
fc1 = layers.BatchNormalization()(fc1)
if (dropout_rate > 0):
fc1 = layers.Dropout(dropout_rate)(fc1)
fc1 = layers.Activation(act_fn)(fc1)
fc2 = layers.Dense(units=n2, kernel_initializer=ki, kernel_regularizer=kr, use_bias=use_bias, name='fc2')(fc1)
if use_bn:
fc2 = layers.BatchNormalization()(fc2)
if (dropout_rate > 0):
fc2 = layers.Dropout(dropout_rate)(fc2)
fc2 = layers.Activation(act_fn)(fc2)
# Add final output layer
ki = initializers.RandomUniform(minval=-3e-3, maxval=3e-3)
raw_actions = layers.Dense(units=self.action_size, activation='tanh',
kernel_initializer=ki, kernel_regularizer=kr,
use_bias=False, name='raw_actions')(fc2)
# Scale [-1, 1] output for each action dimension to proper range
actions = layers.Lambda(lambda x: (0.5*(x + 1.0)*self.action_range + self.action_low),
name='actions')(raw_actions)
# Create Keras model
self.model = models.Model(inputs=states, outputs=actions)
# Define loss function using action value (Q value) gradients
action_gradients = layers.Input(shape=(self.action_size,))
loss = K.mean(-action_gradients * actions)
if (kr is not None):
loss = loss + l2*K.sum(K.square(self.model.get_layer('fc1').get_weights()[0])) \
+ l2*K.sum(K.square(self.model.get_layer('fc2').get_weights()[0])) \
+ l2*K.sum(K.square(self.model.get_layer('raw_actions').get_weights()[0]))
# Define optimizer and training function
optimizer = optimizers.Adam(lr=0.0001)
updates_op = optimizer.get_updates(params=self.model.trainable_weights, loss=loss)
self.train_fn = K.function(
inputs=[self.model.input, action_gradients, K.learning_phase()],
outputs=[],
updates=updates_op)
