def test_binary_accuracy_with_threshold_(self):
y_true = Input((2,))
y_pred = Input((2,))
threshold = K.placeholder((2,))
acc = binary_accuracy_with_threshold(y_true,y_pred,threshold)
self.assertEqual(K.ndim(acc), 0)
binary_accuracy_with_threshold_func = K.function(inputs=[y_true,y_pred,threshold], outputs=[acc])
acc_val=binary_accuracy_with_threshold_func([np.array([[0,1],[1,0]]),np.array([[0.2,0.6],[0.3,0.1]]),np.array([0.25,0.4])])[0]
self.assertEqual(round(acc_val,2), 1.00,"acc_val")
#works on a single threshold
threshold = K.placeholder(ndim=0)
acc = binary_accuracy_with_threshold(y_true, y_pred, threshold)
binary_accuracy_with_threshold_func = K.function(inputs=[y_true, y_pred, threshold], outputs=[acc])
acc_val = binary_accuracy_with_threshold_func(
[np.array([[0, 1], [1, 0]]), np.array([[0.2, 0.6], [0.3, 0.1]]), 0.5])[0]
self.assertEqual(round(acc_val, 2), 0.75, "acc_val")
# works on 3 dimension inputs
y_true = Input((None,2))
y_pred = Input((None,2))
threshold = K.placeholder((2,))
acc = binary_accuracy_with_threshold(y_true,y_pred,threshold)
self.assertEqual(K.ndim(acc), 0)
binary_accuracy_with_threshold_func = K.function(inputs=[y_true,y_pred,threshold], outputs=[acc])
acc_val=binary_accuracy_with_threshold_func([np.array([[[0,1]],[[1,0]]]),np.array([[[0.2,0.6]],[[0.3,0.1]]]),np.array([0.25,0.4])])[0]
self.assertEqual(round(acc_val,2), 1.00,"acc_val")
def test_sequential_call():
"""Test keras.models.Sequential.__call__"""
nb_samples, input_dim, output_dim = 3, 10, 5
model = Sequential()
model.add(Dense(output_dim=output_dim, input_dim=input_dim))
model.compile('sgd', 'mse')
# test flat model
X = K.placeholder(ndim=2)
Y = model(X)
f = K.function([X], [Y])
x = np.ones((nb_samples, input_dim)).astype(K.floatx())
y1 = f([x])[0].astype(K.floatx())
y2 = model.predict(x)
# results of __call__ should match model.predict
assert_allclose(y1, y2)
# test nested model
model2 = Sequential()
model2.add(model)
model2.compile('sgd', 'mse')
Y2 = model2(X)
f = K.function([X], [Y2])
y1 = f([x])[0].astype(K.floatx())
y2 = model2.predict(x)
# results of __call__ should match model.predict
assert_allclose(y1, y2)
def build_model(self, p):
S = Input(p['input_shape'], name='input_state')
A = Input((1,), name='input_action', dtype='int32')
R = Input((1,), name='input_reward')
T = Input((1,), name='input_terminate', dtype='int32')
NS = Input(p['input_shape'], name='input_next_sate')
self.Q_model = self.build_cnn_model(p)
self.Q_old_model = self.build_cnn_model(p, False) # Q hat in paper
self.Q_old_model.set_weights(self.Q_model.get_weights()) # Q' = Q
Q_S = self.Q_model(S) # batch * actions
Q_NS = disconnected_grad(self.Q_old_model(NS)) # disconnected gradient is not necessary
y = R + p['discount'] * (1-T) * K.max(Q_NS, axis=1, keepdims=True) # batch * 1
action_mask = K.equal(Tht.arange(p['num_actions']).reshape((1, -1)), A.reshape((-1, 1)))
output = K.sum(Q_S * action_mask, axis=1).reshape((-1, 1))
loss = K.sum((output - y) ** 2) # sum could also be mean()
optimizer = adam(p['learning_rate'])
params = self.Q_model.trainable_weights
update = optimizer.get_updates(params, [], loss)
self.training_func = K.function([S, A, R, T, NS], loss, updates=update)
self.Q_func = K.function([S], Q_S)
def compile(self, optimizer, loss):
self.optimizer = optimizers.get(optimizer)
self.loss = objectives.get(loss)
# input of model
self.X_train = self.get_input(train=True)
self.X_test = self.get_input(train=False)
train_loss = self.loss(self.X_train)
test_loss = self.loss(self.X_test)
train_loss.name = 'train_loss'
test_loss.name = 'test_loss'
for r in self.regularizers:
train_loss = r(train_loss)
updates = self.optimizer.get_updates(self.params, self.constraints, train_loss)
updates += self.updates
if type(self.X_train) == list:
train_ins = self.X_train
test_ins = self.X_test
else:
train_ins = [self.X_train]
test_ins = [self.X_test]
self._train = K.function(train_ins, train_loss, updates=updates)
self._test = K.function(test_ins, test_loss)
def setup(self):
distorted_A, fake_A, fake_sz64_A, mask_A, self.path_A, self.path_mask_A, self.path_abgr_A, self.path_bgr_A = self.cycle_variables(self.model.netGA)
distorted_B, fake_B, fake_sz64_B, mask_B, self.path_B, self.path_mask_B, self.path_abgr_B, self.path_bgr_B = self.cycle_variables(self.model.netGB)
real_A = Input(shape=self.model.img_shape)
real_B = Input(shape=self.model.img_shape)
if self.use_lsgan:
self.loss_fn = lambda output, target : K.mean(K.abs(K.square(output-target)))
else:
self.loss_fn = lambda output, target : -K.mean(K.log(output+1e-12)*target+K.log(1-output+1e-12)*(1-target))
# ========== Define Perceptual Loss Model==========
if self.use_perceptual_loss:
from keras.models import Model
from keras_vggface.vggface import VGGFace
vggface = VGGFace(include_top=False, model='resnet50', input_shape=(224, 224, 3))
vggface.trainable = False
out_size55 = vggface.layers[36].output
out_size28 = vggface.layers[78].output
out_size7 = vggface.layers[-2].output
vggface_feat = Model(vggface.input, [out_size55, out_size28, out_size7])
vggface_feat.trainable = False
else:
vggface_feat = None
loss_DA, loss_GA = self.define_loss(self.model.netDA, real_A, fake_A, fake_sz64_A, distorted_A, vggface_feat)
loss_DB, loss_GB = self.define_loss(self.model.netDB, real_B, fake_B, fake_sz64_B, distorted_B, vggface_feat)
if self.use_mask_refinement:
loss_GA += 1e-3 * K.mean(K.square(mask_A))
loss_GB += 1e-3 * K.mean(K.square(mask_B))
else:
loss_GA += 3e-3 * K.mean(K.abs(mask_A))
loss_GB += 3e-3 * K.mean(K.abs(mask_B))
w_fo = 0.01
loss_GA += w_fo * K.mean(self.first_order(mask_A, axis=1))
loss_GA += w_fo * K.mean(self.first_order(mask_A, axis=2))
loss_GB += w_fo * K.mean(self.first_order(mask_B, axis=1))
loss_GB += w_fo * K.mean(self.first_order(mask_B, axis=2))
weightsDA = self.model.netDA.trainable_weights
weightsGA = self.model.netGA.trainable_weights
weightsDB = self.model.netDB.trainable_weights
weightsGB = self.model.netGB.trainable_weights
# Adam(..).get_updates(...)
training_updates = Adam(lr=self.lrD, beta_1=0.5).get_updates(weightsDA,[],loss_DA)
self.netDA_train = K.function([distorted_A, real_A],[loss_DA], training_updates)
training_updates = Adam(lr=self.lrG, beta_1=0.5).get_updates(weightsGA,[], loss_GA)
self.netGA_train = K.function([distorted_A, real_A], [loss_GA], training_updates)
training_updates = Adam(lr=self.lrD, beta_1=0.5).get_updates(weightsDB,[],loss_DB)
self.netDB_train = K.function([distorted_B, real_B],[loss_DB], training_updates)
training_updates = Adam(lr=self.lrG, beta_1=0.5).get_updates(weightsGB,[], loss_GB)
self.netGB_train = K.function([distorted_B, real_B], [loss_GB], training_updates)
def compile(self, multi_optimizer):
updates = multi_optimizer.get_updates(
self.params, self.constraints, self.objectives)
ins = [self.inputs[name] for name in self.input_order]
self._train_fn = K.function(
ins, list(self.metrics.values()),
updates=updates + self.additional_updates)
if self.debug_map:
self._debug_fn = K.function(ins, list(self.debug_map.values()))
def __init__(self, replay_filename, group_name, model_filename=''):
# Set learning phase to TEST
self.learning_phase = TEST_MODE
# If not informed, defaults to '_model' suffix
if model_filename == '':
model_filename = '{}_model.h5'.format(group_name)
# Loads Keras model
self.model = load_model(model_filename)
# Loads ReplayData file
self.replay_data = h5py.File('{}'.format(replay_filename), 'r')
self.group_name = group_name
self.group = self.replay_data[self.group_name]
# Retrieves some basic information from the replay data
self.inputs = self.group['inputs'][:]
self.targets = self.group['targets'][:]
self.n_epochs = self.group.attrs['n_epochs']
self.n_layers = self.group.attrs['n_layers']
# Retrieves weights as a list, each element being one epoch
self.weights = self._retrieve_weights()
# Gets Tensors for the weights in the same order as the layers
# Keras' model.weights returns the Tensors in a different order!
self._model_weights = [w for layer in self.model.layers for w in layer.weights]
### Functions
# Keras function to get the outputs, given inputs and weights
self._get_output = K.function(inputs=[K.learning_phase()] + self.model.inputs + self._model_weights,
outputs=[self.model.layers[-1].output])
# Keras function to get the loss and metrics, given inputs, targets, weights and sample weights
self._get_metrics = K.function(inputs=[K.learning_phase()] + self.model.inputs + self.model.targets +
self._model_weights + self.model.sample_weights,
outputs=[self.model.total_loss] + self.model.metrics_tensors)
# Keras function to compute the binary cross entropy, given inputs, targets, weights and sample weights
self._get_binary_crossentropy = K.function(inputs=[K.learning_phase()] + self.model.inputs +
self.model.targets + self._model_weights +
self.model.sample_weights,
outputs=[K.binary_crossentropy(self.model.targets[0],
self.model.outputs[0])])
# Attributes for the visualizations - Data
self._feature_space_data = None
self._loss_hist_data = None
self._loss_and_metric_data = None
self._prob_hist_data = None
self._decision_boundary_data = None
# Attributes for the visualizations - Plot objects
self._feature_space_plot = None
self._loss_hist_plot = None
self._loss_and_metric_plot = None
self._prob_hist_plot = None
self._decision_boundary_plot = None
def __init__(self, state_size, num_actuators, args):
# remember parameters
self.state_size = state_size
self.num_actuators = num_actuators
self.discount_rate = args.discount_rate
self.target_rate = args.target_rate
self.noise = args.noise
self.noise_scale = args.noise_scale
x, u, m, v, q, p, a = self._createLayers(args)
# wrappers around computational graph
fmu = K.function([K.learning_phase(), x], m)
self.mu = lambda x: fmu([0, x])
fP = K.function([K.learning_phase(), x], p)
self.P = lambda x: fP([0, x])
fA = K.function([K.learning_phase(), x, u], a)
self.A = lambda x, u: fA([0, x, u])
fQ = K.function([K.learning_phase(), x, u], q)
self.Q = lambda x, u: fQ([0, x, u])
# main model
self.model = Model(input=[x,u], output=q)
self.model.summary()
if args.optimizer == 'adam':
optimizer = Adam(args.learning_rate)
elif args.optimizer == 'rmsprop':
optimizer = RMSprop(args.learning_rate)
else:
assert False
self.model.compile(optimizer=optimizer, loss='mse')
if args.optimizer == 'adam':
optimizer = Adam(args.learning_rate)
elif args.optimizer == 'rmsprop':
optimizer = RMSprop(args.learning_rate)
else:
assert False
self.model.compile(optimizer=optimizer, loss='mse')
# another set of layers for target model
x, u, m, v, q, p, a = self._createLayers(args)
# V() function uses target model weights
fV = K.function([K.learning_phase(), x], v)
self.V = lambda x: fV([0, x])
# target model is initialized from main model
self.target_model = Model(input=[x,u], output=q)
self.target_model.set_weights(self.model.get_weights())
def _deconv(self, X, lname, d_switch, feat_map=None):
o_width, o_height = self[lname].output_shape[-2:]
# Get filter size
f_width = self[lname].W_shape[2]
f_height = self[lname].W_shape[3]
# Compute padding needed
i_width, i_height = X.shape[-2:]
pad_width = (o_width - i_width + f_width - 1) / 2
pad_height = (o_height - i_height + f_height - 1) / 2
assert isinstance(
pad_width, int), "Pad width size issue at layer %s" % lname
assert isinstance(
pad_height, int), "Pad height size issue at layer %s" % lname
# Set to zero based on switch values
X[d_switch[lname]] = 0
# Get activation function
activation = self[lname].activation
X = activation(X)
if feat_map is not None:
print "Setting other feat map to zero"
for i in range(X.shape[1]):
if i != feat_map:
X[:, i, :, :] = 0
print "Setting non max activations to zero"
for i in range(X.shape[0]):
iw, ih = np.unravel_index(
X[i, feat_map, :, :].argmax(), X[i, feat_map, :, :].shape)
m = np.max(X[i, feat_map, :, :])
X[i, feat_map, :, :] = 0
X[i, feat_map, iw, ih] = m
# Get filters. No bias for now
W = self[lname].W
# Transpose filter
W = W.transpose([1, 0, 2, 3])
W = W[:, :, ::-1, ::-1]
# CUDNN for conv2d ?
conv_out = K.T.nnet.conv2d(
input=self.x, filters=W, border_mode='valid')
# Add padding to get correct size
pad = K.function([self.x], K.spatial_2d_padding(
self.x, padding=(pad_width, pad_height), dim_ordering="th"))
X_pad = pad([X])
# Get Deconv output
deconv_func = K.function([self.x], conv_out)
X_deconv = deconv_func([X_pad])
assert X_deconv.shape[-2:] == (o_width, o_height),\
"Deconv output at %s has wrong size" % lname
return X_deconv
def cycle_variables(self, netG):
distorted_input = netG.inputs[0]
fake_output = netG.outputs[0]
fake_output64 = netG.outputs[1]
alpha = Lambda(lambda x: x[:,:,:, :1])(fake_output)
rgb = Lambda(lambda x: x[:,:,:, 1:])(fake_output)
masked_fake_output = alpha * rgb + (1-alpha) * distorted_input
fn_generate = K.function([distorted_input], [masked_fake_output])
fn_mask = K.function([distorted_input], [concatenate([alpha, alpha, alpha])])
fn_abgr = K.function([distorted_input], [concatenate([alpha, rgb])])
fn_bgr = K.function([distorted_input], [rgb])
return distorted_input, fake_output, fake_output64, alpha, fn_generate, fn_mask, fn_abgr, fn_bgr
def _backward_pass(self, X, target_layer, d_switch, feat_map):
# Run deconv/maxunpooling until input pixel space
layer_index = self.lnames.index(target_layer)
# Get the output of the target_layer of interest
layer_output = K.function(
[self[self.lnames[0]].input], self[target_layer].output)
X_outl = layer_output([X])
# Special case for the starting layer where we may want
# to switchoff somes maps/ activations
print "Deconvolving %s..." % target_layer
if "maxpooling2d" in target_layer:
X_maxunp = K.pool.max_pool_2d_same_size(
self[target_layer].input, self[target_layer].pool_size)
unpool_func = K.function([self[self.lnames[0]].input], X_maxunp)
X_outl = unpool_func([X])
if feat_map is not None:
for i in range(X_outl.shape[1]):
if i != feat_map:
X_outl[:, i, :, :] = 0
for i in range(X_outl.shape[0]):
iw, ih = np.unravel_index(
X_outl[i, feat_map, :, :].argmax(), X_outl[i, feat_map, :, :].shape)
m = np.max(X_outl[i, feat_map, :, :])
X_outl[i, feat_map, :, :] = 0
X_outl[i, feat_map, iw, ih] = m
elif "convolution2d" in target_layer:
X_outl = self._deconv(X_outl, target_layer,
d_switch, feat_map=feat_map)
else:
raise ValueError(
"Invalid layer name: %s \n Can only handle maxpool and conv" % target_layer)
# Iterate over layers (deepest to shallowest)
for lname in self.lnames[:layer_index][::-1]:
print "Deconvolving %s..." % lname
# Unpool, Deconv or do nothing
if "maxpooling2d" in lname:
p1, p2 = self[lname].pool_size
uppool = K.function(
[self.x], K.resize_images(self.x, p1, p2, "th"))
X_outl = uppool([X_outl])
elif "convolution2d" in lname:
X_outl = self._deconv(X_outl, lname, d_switch)
elif "padding" in lname:
pass
else:
raise ValueError(
"Invalid layer name: %s \n Can only handle maxpool and conv" % lname)
return X_outl
def set_batch_function(self, model, input_shape, batch_size, nb_actions, gamma):
input_dim = np.prod(input_shape)
samples = K.placeholder(shape=(batch_size, input_dim * 2 + 3))
S = samples[:, 0 : input_dim]
a = samples[:, input_dim]
a = K.cast(a, '')
r = samples[:, input_dim + 1]
S_prime = samples[:, input_dim + 2 : 2 * input_dim + 2]
game_over = samples[:, 2 * input_dim + 2 : 2 * input_dim + 3]
r = K.reshape(r, (batch_size, 1))
r = K.repeat(r, nb_actions)
r = K.reshape(r, (batch_size, nb_actions))
game_over = K.repeat(game_over, nb_actions)
game_over = K.reshape(game_over, (batch_size, nb_actions))
S = K.reshape(S, (batch_size, ) + input_shape)
S_prime = K.reshape(S_prime, (batch_size, ) + input_shape)
X = K.concatenate([S, S_prime], axis=0)
Y = model(X)
Qsa = K.max(Y[batch_size:], axis=1)
Qsa = K.reshape(Qsa, (batch_size, 1))
Qsa = K.repeat(Qsa, nb_actions)
Qsa = K.reshape(Qsa, (batch_size, nb_actions))
delta = K.reshape(self.one_hot(a, nb_actions), (batch_size, nb_actions))
targets = (1 - delta) * Y[:batch_size] + delta * (r + gamma * (1 - game_over) * Qsa)
self.batch_function = K.function(inputs=[samples], outputs=[S, targets])
def Visualize(self,img_path, output_path):
original_img = cv2.imread(img_path, 1)
width, height, _ = original_img.shape
#Reshape to the network input shape (3, w, h).
#img = np.array([np.transpose(np.float32(original_img), (2, 0, 1))])
#Get the 512 input weights to the softmax.
class_weights = self.model.layers[-1].get_weights()[0]
final_conv_layer = self.get_output_layer(self.model, "conv2d_26")
get_output = K.function([self.model.layers[0].input], \
[final_conv_layer.output,
self.model.layers[-1].output])
[conv_outputs, predictions] = get_output([np.array([original_img])])
conv_outputs = conv_outputs[0, :, :, :]
print(predictions)
#Create the class activation map.
cam = np.ones(conv_outputs.shape[0 : 2], dtype = np.float32)
target_class = 1
for i, w in enumerate(class_weights[:, target_class]):
cam+= w * conv_outputs[:, :,i]
print("predictions", predictions)
cam /= np.max(cam)
cam = cv2.resize(cam, (height, width))
print(cam.shape)
cam /= np.max(cam)
cam = cv2.resize(cam, (height, width))
heatmap = cv2.applyColorMap(np.uint8(255*cam), cv2.CV_8UC1)
heatmap[np.where(cam < 0.2)] = 0
img = heatmap*0.5 + original_img
cv2.imwrite(output_path, img)
def get_features(self, x, layers):
if not layers:
return None
f = K.function([self.net_input], [self.get_layer_output(layer_name) for layer_name in layers])
feature_outputs = f([x])
features = dict(zip(layers, feature_outputs))
return features
def test_relu():
x = K.placeholder(ndim=2)
f = K.function([x], [activations.relu(x)])
test_values = get_standard_values()
result = f([test_values])[0]
assert_allclose(result, test_values, rtol=1e-05)
def __init__(self, model, outchannels=[], verbose=1):
# Bacnend: either tensorflow or theano)
self.backend = K.backend()
# load model supports keras.Model and keras.Sequential
if isinstance(model, Sequential):
self.model = model.model
elif isinstance(model, Model):
self.model = model
else:
print("Invalid input model")
return -1
# load input tensors
self.input_tensors = []
for i in self.model.inputs:
self.input_tensors.append(i)
# The learning phase flag is a bool tensor (0 = test, 1 = train)
# to be passed as input to any Keras function that uses
# a different behavior at train time and test time.
self.input_tensors.append(K.learning_phase())
# If outputchanel is specified, use it.
# Otherwise evalueate all outputs.
self.outchannels = outchannels
if len(self.outchannels) == 0:
if verbose: print("Evaluated output channel (0-based index): All")
if K.backend() == "tensorflow":
self.outchannels = range(self.model.output.shape[1]._value)
elif K.backend() == "theano":
self.outchannels = range(model1.output._keras_shape[1])
else:
if verbose:
print("Evaluated output channels (0-based index):")
print(','.join([str(i) for i in self.outchannels]))
# Build gradient functions for desired output channels.
self.get_gradients = {}
if verbose: print("Building gradient functions")
# Evaluate over all channels.
for c in self.outchannels:
# Get tensor that calcuates gradient
if K.backend() == "tensorflow":
gradients = self.model.optimizer.get_gradients(self.model.output[:, c], self.model.input)
if K.backend() == "theano":
gradients = self.model.optimizer.get_gradients(self.model.output[:, c].sum(), self.model.input)
# Build computational graph that calculates the tensfor given inputs
self.get_gradients[c] = K.function(inputs=self.input_tensors, outputs=gradients)
# This takes a lot of time for a big model with many tasks.
# So lets pring the progress.
if verbose:
sys.stdout.write('\r')
sys.stdout.write("Progress: " + str(int((c + 1) * 1.0 / len(self.outchannels) * 1000) * 1.0 / 10) + "%")
sys.stdout.flush()
# Done
if verbose: print("\nDone.")
def get_features(model, dataset, position, N, training_params, verbose, flip=False):
intermediate_outputs = K.function([model.layers[0].input], [model.layers[position].get_output(train=False)])
if N==0:
raise Exception("Ntest = 0.")
for i in range(N):
if verbose:
print "\rBatch %d over %d"%(i,N),
# Get next batch
batch,targets= dataset.get_batch()
# Eventually flip
if flip:
batch = np.fliplr(batch.transpose(1,2,3,0)).transpose(3,0,1,2)
# Preprocess
for mode in training_params.valid_preprocessing:
batch = preprocess_dataset(batch, training_params, mode)
# Predict
pred = intermediate_outputs([np.array(batch.transpose(0,3,1,2), "float32")])[0]
if pred.shape[1] != 256 and pred.shape[1] != 512:
raise Exception("not the good layer. Dim = %d"%pred.shape[1])
# Accumulate preds
if i==0:
predictions = np.copy(pred)
labels = np.copy(convert_labels(targets))
else:
predictions = np.concatenate((predictions,pred))
labels = np.concatenate((labels,convert_labels(targets)))
return predictions, labels
def get_mc_predictions(model, X, nb_iter=50, batch_size=256):
"""
TODO
:param model:
:param X:
:param nb_iter:
:param batch_size:
:return:
"""
output_dim = model.layers[-1].output.shape[-1].value
get_output = K.function(
[model.layers[0].input, K.learning_phase()],
[model.layers[-1].output]
)
def predict():
n_batches = int(np.ceil(X.shape[0] / float(batch_size)))
output = np.zeros(shape=(len(X), output_dim))
for i in range(n_batches):
output[i * batch_size:(i + 1) * batch_size] = \
get_output([X[i * batch_size:(i + 1) * batch_size], 1])[0]
return output
preds_mc = []
for i in tqdm(range(nb_iter)):
preds_mc.append(predict())
return np.asarray(preds_mc)
def build_model(self):
"""Build a Critic (Value) model that maps (state, action)> Q_values."""
# Input layers
states = layers.Input(shape=(self.state_size,), name="states")
actions = layers.Input(shape=(self.action_size,), name="actions")
# Add some hidden layers to state pathway
net_states = layers.Dense(units=32, activation="relu")(states)
net_states = layers.Dense(units=64, activation="relu")(net_states)
# Add some hidden layers to action pathway
net_actions = layers.Dense(units=32, activation="relu")(actions)
net_actions = layers.Dense(units=64, activation="relu")(net_actions)
# Combine both pathways
net = layers.Add()([net_states, net_actions])
net = layers.Activation('relu')(net)
Q_values = layers.Dense(units=1, name='q_values')(net)
self.model = models.Model(inputs=[states, actions], outputs=Q_values)
optimizer = optimizers.Adam()
self.model.compile(optimizer=optimizer, loss='mse')
# Compute action gradients (derivative of Q values w.r.t. to actions)
action_gradients = K.gradients(Q_values, actions)
# Define an additional function to fetch action gradients (to be used
# by actor model)
self.get_action_gradients = K.function(
inputs=[*self.model.inputs, K.learning_phase()],
outputs=action_gradients)
def set_f_outputs(self, combo_img, loss, grads):
outputs = [loss]
if type(grads) in {list, tuple}:
outputs += grads
else:
outputs.append(grads)
return K.function([combo_img], outputs)
def pretrain(model,data,learning_rate,layer_activations,output_activation,optimizer):
weights=model.get_weights()
weigts_dummy=weights
layer_input=data
for i in range(int(len(weights)/4)):
if i==0:
inputs=Input(shape=(np.size(weights[2*i],axis=0),))
layer_1=Dense(np.size(weights[2*i],axis=1),activation=layer_activations)(inputs)
predictions=Dense(np.size(weights[2*i],axis=0),activation=output_activation)(layer_1)
else:
inputs=Input(shape=(np.size(weights[2*i],axis=0),))
layer_1=Dense(np.size(weights[2*i],axis=1),activation=layer_activations)(inputs)
predictions=Dense(np.size(weights[2*i],axis=0),activation=layer_activations)(layer_1)
stacked_ae=Model(inputs=inputs,outputs=predictions)
stacked_ae.compile(optimizer=optimizer, loss='mean_squared_error')
print(i)
stacked_ae.fit(layer_input,layer_input,shuffle=True,epochs=10,batch_size=64)
get_h_layer_output = K.function([stacked_ae.layers[0].input],
[stacked_ae.layers[1].output])
layer_1_predictions = get_h_layer_output([layer_input, 0])[0]
w=stacked_ae.get_weights()
del layer_input,layer_1,inputs,predictions,get_h_layer_output
layer_input=layer_1_predictions
weigts_dummy[2*i]=w[0]
weigts_dummy[2*i+1]=w[1]
weigts_dummy[len(weights)-2*i-2]=w[2]
weigts_dummy[len(weights)-2*i-1]=w[3]
del w
model.set_weights(weigts_dummy)
return model
def build_train_fn(model):
# cost
lr = T.scalar()
labels = K.placeholder(ndim=2, dtype='int32')
ob_input = model.inputs[0]
raw_softmax_outputs = model.outputs[0]
softmax_outputs = raw_softmax_outputs.dimshuffle((2,0,1))
softmax_outputs = softmax_outputs.reshape((softmax_outputs.shape[0], softmax_outputs.shape[1]*softmax_outputs.shape[2]))
softmax_outputs = softmax_outputs.dimshuffle((1,0))
cost = categorical_crossentropy(softmax_outputs, labels).mean()
# gradients
trainable_vars = model.trainable_weights
grads = K.gradients(cost, trainable_vars)
grads = lasagne.updates.total_norm_constraint(grads, 100)
updates = lasagne.updates.nesterov_momentum(grads, trainable_vars, lr, 0.99)
for key, val in model.updates:
updates[key] = val
# train_fn
train_fn = K.function([ob_input, labels, K.learning_phase(), lr],
[softmax_outputs, cost],
updates=updates)
return train_fn
def plot_attention(sentence, Tx=20, Ty=25):
"""
可视化Attention层
@param sentence: 待翻译的句子，str类型
@param Tx: 输入句子的长度
@param Ty: 输出句子的长度
"""
X = np.array(text_to_int(sentence, source_vocab_to_int))
f = K.function(model.inputs, [model.layers[9].get_output_at(t) for t in range(Ty)])
s0 = np.zeros((1, n_s))
c0 = np.zeros((1, n_s))
out0 = np.zeros((1, len(target_vocab_to_int)))
r = f([X.reshape(-1, 20), s0, c0, out0])
attention_map = np.zeros((Ty, Tx))
for t in range(Ty):
for t_prime in range(Tx):
attention_map[t][t_prime] = r[t][0, t_prime, 0]
Y = make_prediction(sentence)
source_list = sentence.split()
target_list = Y.split()
f, ax = plt.subplots(figsize=(20, 15))
sns.heatmap(attention_map, xticklabels=source_list, yticklabels=target_list, cmap="YlGnBu")
ax.set_xticklabels(ax.get_xticklabels(), fontsize=15, rotation=90)
ax.set_yticklabels(ax.get_yticklabels(), fontsize=15)
def extract(matlabfile,outfile):
res_ims = []
with open(matlabfile) as matfile:
mat = sio.loadmat(matfile)
# ys = mat['labels']
ims = mat['ims'][0]
for im in ims:
im = cv2.resize(im, (224, 224)).astype(np.float32)
im[:,:,0] -= 103.939
im[:,:,1] -= 116.779
im[:,:,2] -= 123.68
im = im.transpose((2,0,1))
im = np.expand_dims(im, axis=0)
res_ims.append(im)
#%%
layer_outputs = []
model = vgg.VGG_19('vgg19_weights.h5')
sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True) #not used since the weights are pre-trained
model.compile(optimizer=sgd, loss='categorical_crossentropy')
#get_37th_layer_output = K.function([model.layers[0].input], [model.layers[38].output]) #37 = flatten
get_layer_output = K.function([model.layers[0].input, K.learning_phase()], [model.layers[41].output]) #41 = last layer-1
i = 0
l = '('+str(len(res_ims))+')'
for im in res_ims:
print 'progress',str(i)+l
i += 1
# layer_output = get_37th_layer_output([im])[0][0]
layer_output = get_layer_output([im, 1])[0][0]
layer_outputs.append(layer_output)
Xtrain = np.array(layer_outputs).T
np.save(outfile,Xtrain)
def get_training_function(self, x, y):
model = self.model
cost, grads = self.get_cost_grads()
outs = [cost]
if type(grads) in {list, tuple}:
outs += grads
else:
outs.append(grads)
fn = K.function(
inputs=[],
outputs=outs,
givens={
model.inputs[0]: x,
model.targets[0]: y,
model.sample_weights[0]: np.ones(
(x.shape[0],
),
dtype=np.float32),
K.learning_phase(): np.uint8(1)})
def train_fn(theta):
self.set_model_params(theta)
cost_grads = fn([])
cost = np.asarray(cost_grads[0], dtype=np.float64)
grads = np.asarray(self.flatten_grads(cost_grads[1:]), dtype=np.float64)
return cost, grads
return train_fn
def build_model_policy_value(self, model, max_cache_size=100000):
from collections import OrderedDict
cache = OrderedDict()
from tensorflow.python.keras import backend as K
get_output = K.function([model.input, K.learning_phase()], [model.output[0], model.output[1]])
def model_policy_value(input_array):
key = input_array.tobytes()
if key in cache:
cache.move_to_end(key, last=True)
return cache[key]
input_array = input_array.reshape((-1, 54, 6))
#input_array = np.rollaxis(input_array, 2, 1)
policy, value = model.predict(input_array)
#policy, value = get_output([input_array, 0])
policy = policy.reshape((12,))
value = value[0, 0]
cache[key] = (policy, value)
if len(cache) > max_cache_size:
cache.popitem(last=False)
return policy, value
return model_policy_value
def get_grad(gen_image_array):
'''计算损失函数的梯度'''
if gen_image_array != (1, target_width, target_height, 3):
gen_image_array = gen_image_array.reshape((1, target_width, target_height, 3))
grad_fn = K.function([gModel.input], K.gradients(get_total_loss(gModel.input), [gModel.input]))
grad = grad_fn([gen_image_array])[0].flatten().astype('float64')
return grad
def compile_saliency_function(model, activation_layer='block5_conv3'):
input_img = model.input
layer_dict = dict([(layer.name, layer) for layer in model.layers[1:]])
layer_output = layer_dict[activation_layer].output
max_output = K.max(layer_output, axis=3)
saliency = K.gradients(K.sum(max_output), input_img)[0]
return K.function([input_img, K.learning_phase()], [saliency])
def grad_cam(input_model, model_x, orig_x, category_index, layer_name, class_names):
output = input_model.output
final_layer = Lambda(lambda x: target_category_loss(x, category_index, len(class_names)))
output = final_layer(output)
model = Model(inputs=input_model.input, outputs=output)
loss = K.sum(model.layers[-1].output)
conv_output = model.get_layer(layer_name).output
grads = normalize(K.gradients(loss, conv_output)[0])
gradient_function = K.function([model.layers[0].input, K.learning_phase()], [conv_output, grads])
output, grads_val = gradient_function([model_x, 0])
output, grads_val = output[0, :], grads_val[0, :, :, :]
weights = np.mean(grads_val, axis=(0, 1))
cam = np.zeros(output.shape[0: 2], dtype=np.float32)
for i, w in enumerate(weights):
cam += w * output[:, :, i]
cam = np.maximum(cam, np.zeros(output.shape[0: 2], dtype=np.float32))
cam = cam.squeeze()
cam = cv2.applyColorMap(np.uint8(255 * cam / np.max(cam)), cv2.COLORMAP_JET)
cam = cv2.resize(cam, (np.shape(orig_x)[0], np.shape(orig_x)[1]))
cam = 0.4 * cam + 0.6 * orig_x
return np.uint8(cam)
def EvaluateJacobian(model): #theano.function( [model.layers[0].input], T.jacobian(model.layers[-1].output.flatten(), model.layers[0].input) ) X = K.placeholder(shape=(15,15)) #specify the right placeholder Y = K.sum(K.square(X)) # loss function fn = K.function([X], K.gradients(Y, [X])) #function to call the gradient
def get_model(self, pad_id):
q_tokens = Input(shape=(None, ), name='q_tokens')
qr_masks = [
Input(shape=(None, ), name='qr_masks_%s' % j)
for j in range(self.query_retrieval_number)
]
qr_tokens = [
Input(shape=(None, ), name='q_retrieval_%s' % j)
for j in range(self.query_retrieval_number)
]
pos_tokens = [
Input(shape=(None, ), name='pos_tokens_%s' % j)
for j in range(self.pos_number)
]
neg_tokens = [
Input(shape=(None, ), name='neg_fact_input_%s' % j)
for j in range(self.neg_number)
]
enc_query_output = self.__get_query_encoder(q_tokens, pad_id, 'query')
enc_qrs_output = [self.__get_query_encoder(q_retrieval, pad_id, 'qr%s'%index) for index, q_retrieval in \
enumerate(qr_tokens)]
enc_pos_facts_output = [self.__get_fact_encoder(pos_fact, pad_id, 'pos%s'%index) for index, pos_fact in \
enumerate(pos_tokens)]
enc_neg_facts_output = [self.__get_fact_encoder(neg_fact, pad_id, 'neg%s'%index) for index, neg_fact in \
enumerate(neg_tokens)]
# query convolution and maxpooling
query_emb = self.query_max(enc_query_output)
query_sem = query_emb
qrs_emb = QueryRetrievalEncoderMask(
self.query_retrieval_number)(qr_masks + enc_qrs_output)
# comment for 1 qr
qrs_emb = [self.query_max(qr_emb) for qr_emb in qrs_emb]
qrs_sem = qrs_emb
# just choose one sentence, so the shape: bs, embedding_dim
useful_sent = UsefulSentenceAutoPointer(
tau=self.tau,
batch_size=self.args.batch_size,
query_retrieval_number=self.query_retrieval_number,
use_transition=True)(qrs_sem)
pos_facts_emb = [
self.fact_max(pos_fact) for pos_fact in enc_pos_facts_output
]
neg_facts_emb = [
self.fact_max(neg_fact) for neg_fact in enc_neg_facts_output
]
pos_facts_sem = pos_facts_emb
neg_facts_sem = neg_facts_emb
# cosine similarity
#query_pos_facts_cosine = [Lambda(lambda x:K.sigmoid(x))(Dot(axes=1, normalize=True)([query_sem, pos_fact_sem])) for pos_fact_sem in pos_facts_sem]
#query_neg_facts_cosine = [Lambda(lambda x:K.sigmoid(x))(Dot(axes=1, normalize=True)([query_sem, neg_fact_sem])) for neg_fact_sem in neg_facts_sem]
query_pos_facts_cosine = [
Dot(axes=1, normalize=True)([query_sem, pos_fact_sem])
for pos_fact_sem in pos_facts_sem
]
query_neg_facts_cosine = [
Dot(axes=1, normalize=True)([query_sem, neg_fact_sem])
for neg_fact_sem in neg_facts_sem
]
auto_pos_facts_cosine = [
Dot(axes=1, normalize=True)([useful_sent, pos_fact_sem])
for pos_fact_sem in pos_facts_sem
]
auto_neg_facts_cosine = [
Dot(axes=1, normalize=True)([useful_sent, neg_fact_sem])
for neg_fact_sem in neg_facts_sem
]
query_pos_facts_cosine = [
self.cosine_merger_layer(query_pos_facts_cosine +
auto_pos_facts_cosine)
]
query_neg_facts_cosine = self.cosine_merger_layer(
query_neg_facts_cosine + auto_neg_facts_cosine)
concat_cosine = Concatenate()(query_pos_facts_cosine +
query_neg_facts_cosine)
concat_cosine = Reshape(
(self.pos_number + self.neg_number, 1))(concat_cosine)
# gamma
weight = np.asarray([1]).reshape(1, 1, 1)
with_gamma = Conv1D(1,
1,
padding='same',
input_shape=(self.pos_number + self.neg_number, 1),
activation='linear',
use_bias=False,
weights=[weight])(concat_cosine)
with_gamma = Reshape((self.pos_number + self.neg_number, ))(with_gamma)
# softmax
prob = self.output_softmax_layer(with_gamma)
model = Model(inputs=[q_tokens] + qr_masks + qr_tokens + pos_tokens +
neg_tokens,
outputs=prob)
model_pos_consine = K.function([q_tokens] + qr_masks + qr_tokens +
pos_tokens, query_pos_facts_cosine)
query_embeddings = [enc_query_output]
pos_embeddings = enc_pos_facts_output
query_word_embedding_fn = K.function([q_tokens], query_embeddings)
pos_embedding_fn = K.function(pos_tokens, pos_embeddings)
return model, model_pos_consine, query_word_embedding_fn, pos_embedding_fn
def main(self, name, opts):
logging.basicConfig(filename=opts.log_file,
format='%(levelname)s (%(asctime)s): %(message)s')
log = logging.getLogger(name)
if opts.verbose:
log.setLevel(logging.DEBUG)
else:
log.setLevel(logging.INFO)
log.debug(opts)
if opts.seed is not None:
np.random.seed(opts.seed)
if not opts.model_files:
raise ValueError('No model files provided!')
log.info('Loading model ...')
K.set_learning_phase(0)
model = mod.load_model(opts.model_files)
# Get DNA layer.
dna_layer = None
for i, name in enumerate(model.input_names):
if name == 'dna':
dna_layer = model.input_layers[i]
break
if not dna_layer:
raise ValueError('The provided model is not a DNA model!')
# Create output vector.
outputs = []
for output in model.outputs:
outputs.append(K.reshape(output, (-1, 1)))
outputs = K.concatenate(outputs, axis=1)
# Compute gradient of outputs wrt. DNA layer.
grads = []
for name in opts.targets:
if name == 'mean':
target = K.mean(outputs, axis=1)
elif name == 'var':
target = K.var(outputs, axis=1)
else:
raise ValueError('Invalid effect size "%s"!' % name)
grad = K.gradients(target, dna_layer.output)
grads.extend(grad)
grad_fun = K.function(model.inputs, grads)
log.info('Reading data ...')
nb_sample = dat.get_nb_sample(opts.data_files, opts.nb_sample)
replicate_names = dat.get_replicate_names(
opts.data_files[0],
regex=opts.replicate_names,
nb_key=opts.nb_replicate)
data_reader = mod.data_reader_from_model(
model, outputs=False, replicate_names=replicate_names)
data_reader = data_reader(opts.data_files,
nb_sample=nb_sample,
batch_size=opts.batch_size,
loop=False,
shuffle=False)
meta_reader = hdf.reader(opts.data_files, ['chromo', 'pos'],
nb_sample=nb_sample,
batch_size=opts.batch_size,
loop=False,
shuffle=False)
out_file = h5.File(opts.out_file, 'w')
out_group = out_file
def h5_dump(path, data, idx, dtype=None, compression='gzip'):
if path not in out_group:
if dtype is None:
dtype = data.dtype
out_group.create_dataset(
name=path,
shape=[nb_sample] + list(data.shape[1:]),
dtype=dtype,
compression=compression
)
out_group[path][idx:idx+len(data)] = data
log.info('Computing effects ...')
progbar = ProgressBar(nb_sample, log.info)
idx = 0
for inputs in data_reader:
if isinstance(inputs, dict):
inputs = list(inputs.values())
batch_size = len(inputs[0])
progbar.update(batch_size)
# Compute gradients.
grads = grad_fun(inputs)
# Slice window at center.
if opts.dna_wlen:
for i, grad in enumerate(grads):
delta = opts.dna_wlen // 2
ctr = grad.shape[1] // 2
grads[i] = grad[:, (ctr-delta):(ctr+delta+1)]
# Aggregate effects in window
if opts.agg_effects:
for i, grad in enumerate(grads):
if opts.agg_effects == 'mean':
grad = grad.mean(axis=1)
elif opts.agg_effects == 'wmean':
weights = linear_weights(grad.shape[1])
grad = np.average(grad, axis=1, weights=weights)
elif opts.agg_effects == 'max':
grad = grad.max(axis=1)
else:
tmp = 'Invalid function "%s"!' % (opts.agg_effects)
raise ValueError(tmp)
grads[i] = grad
# Write computed effects
for name, grad in zip(opts.targets, grads):
h5_dump(name, grad, idx)
# Store inputs
if opts.store_inputs:
for name, value in zip(model.input_names, inputs):
h5_dump(name, value, idx)
# Store positions
for name, value in next(meta_reader).items():
h5_dump(name, value, idx)
idx += batch_size
progbar.close()
out_file.close()
log.info('Done!')
return 0
# 'kernel_constraint': None,
# 'kernel_initializer': {'class_name': 'VarianceScaling',
# 'config': {'distribution': 'uniform',
# 'mode': 'fan_avg',
# 'scale': 1.0,
# 'seed': None}},
# 'kernel_regularizer': {'class_name': 'L1L2',
# 'config': {'l1': 0.0, 'l2': 0.0017999999690800905}},
# 'name': 'dense_3',
# 'trainable': True,
# 'units': 1,
# 'use_bias': True}
#-------------------------
# get action potential matrix of layer 0, output = activation(dot(input, kernel) + bias), https://keras.io/layers/core/#dense
get_0th_layer_output = K.function([new_model.layers[0].input],
[new_model.layers[0].output])
zerolayer_output = get_0th_layer_output([aa_train_x
])[0] # this is a list with len of 1
zerolayer_output.shape
#(903, 180)
# get weights matrix from layer 0
len(new_model.layers[0].get_weights()[0]) # number of rows in weight matrix
zerolayer_weights = np.zeros((3092, 180), dtype=float)
for i in range(len(new_model.layers[0].get_weights()[0])):
zerolayer_weights[i, :] = new_model.layers[0].get_weights()[0][i]
# the input training matrix
aa_train_x.shape #(903, 3092)
# get pij = 1/m * sum over m (abs(wji*xi +bj)) from the paper https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=7280626
def _generate_filter_image(input_img,
layer_output,
filter_index):
"""Generates image for one particular filter.
# Arguments
input_img: The input-image Tensor.
layer_output: The output-image Tensor.
filter_index: The to be processed filter number.
Assumed to be valid.
#Returns
Either None if no image could be generated.
or a tuple of the image (array) itself and the last loss.
"""
s_time = time.time()
# we build a loss function that maximizes the activation
# of the nth filter of the layer considered
if K.image_data_format() == 'channels_first':
loss = K.mean(layer_output[:, filter_index, :, :])
else:
loss = K.mean(layer_output[:, :, :, filter_index])
# we compute the gradient of the input picture wrt this loss
grads = K.gradients(loss, input_img)[0]
# normalization trick: we normalize the gradient
grads = normalize(grads)
# this function returns the loss and grads given the input picture
iterate = K.function([input_img], [loss, grads])
# we start from a gray image with some random noise
intermediate_dim = tuple(
int(x / (upscaling_factor ** upscaling_steps)) for x in output_dim)
if K.image_data_format() == 'channels_first':
input_img_data = np.random.random(
(1, 3, intermediate_dim[0], intermediate_dim[1]))
else:
input_img_data = np.random.random(
(1, intermediate_dim[0], intermediate_dim[1], 3))
input_img_data = (input_img_data - 0.5) * 20 + 128
# Slowly upscaling towards the original size prevents
# a dominating high-frequency of the to visualized structure
# as it would occur if we directly compute the 412d-image.
# Behaves as a better starting point for each following dimension
# and therefore avoids poor local minima
for up in reversed(range(upscaling_steps)):
# we run gradient ascent for e.g. 20 steps
for _ in range(epochs):
loss_value, grads_value = iterate([input_img_data])
input_img_data += grads_value * step
# some filters get stuck to 0, we can skip them
if loss_value <= K.epsilon():
return None
# Calculate upscaled dimension
intermediate_dim = tuple(
int(x / (upscaling_factor ** up)) for x in output_dim)
# Upscale
img = deprocess_image(input_img_data[0])
img = np.array(pil_image.fromarray(img).resize(intermediate_dim,
pil_image.BICUBIC))
input_img_data = np.expand_dims(
process_image(img, input_img_data[0]), 0)
# decode the resulting input image
img = deprocess_image(input_img_data[0])
e_time = time.time()
print('Costs of filter {:3}: {:5.0f} ( {:4.2f}s )'.format(filter_index,
loss_value,
e_time - s_time))
return img, loss_value
from skimage.measure import block_reduce
from function import newtxt,newimagedata,create_plots,plot_confusion_matrix,cnn_model,cnn_model1,cnn_model2
#%%
img_path = './2012image/201208172_T-12-46-15_Dive_01_017.jpg'
img = imread(img_path)
#%%
divide_image = np.zeros([3468,30,30,3])
for i in range(51):
for j in range(68):
divide_image[(i+1)*j,:,:,:] = img[i*30:(i+1)*30,j*30:(j+1)*30]
#%%
#K.set_learning_phase(1) #set learning phase
model = cnn_model()
model.load_weights('model_weight/2012images-areas-7.5-50epoch.h5')
cnn_output = K.function([model.layers[0].input], [model.layers[19].output])
f1 = cnn_output([divide_image])[0]
#%%
kmeans = MiniBatchKMeans(n_clusters=300,
random_state=0,
batch_size=128,
max_iter=10).fit(f1)
#%%
code_book = kmeans.cluster_centers_
model.save(model_path_save)
# Layers sizes
input("Press Enter to continue.")
bot_lay_size = 2048
n_train_imgs = imgs_train.shape[0]
n_test_imgs = imgs_test.shape[0]
n_classes = lab_train_ohe.shape[1]
print("bot lay size:", bot_lay_size)
print("train_imgs", n_train_imgs)
print('test imgs', n_test_imgs)
print("classes", n_classes)
# backend function to accesss values from bottle layer
bottle_tensor_func = K.function(
[model.layers[0].input, K.learning_phase()],
[model.get_layer('fc_1_act').output])
#set up np.array to store values for all images
bottle_tensor_train = np.zeros(shape=(n_train_imgs, bot_lay_size))
bottle_labels_train = np.zeros(shape=(n_train_imgs, n_classes))
bottle_tensor_test = np.zeros(shape=(n_test_imgs, bot_lay_size))
bottle_labels_test = np.zeros(shape=(n_test_imgs, n_classes))
def batcher(X_train, y_train, size):
X_batch = [
X_train[indx:indx + size] for indx in range(0, len(X_train), size)
]
y_batch = [
y_train[indx:indx + size] for indx in range(0, len(y_train), size)
]
return zip(X_batch, y_batch)
def get_f_layer(self, layer_name):
return K.function([self.net_input],
[self.get_layer_output(layer_name)])
style_reference_features = layer_features[1, :, :, :]
combination_features = layer_features[3, :, :, :]
sl = style_loss(style_reference_features, combination_features)
loss += (style_weight / len(feature_layers)) * sl
loss += total_variation_weight * total_variation_loss(combination_image)
# get the gradients of the generated image wrt the loss
grads = K.gradients(loss, combination_image)
outputs = [loss]
if type(grads) in {list, tuple}:
outputs += grads
else:
outputs.append(grads)
f_outputs = K.function([combination_image], outputs)
def eval_loss_and_grads(x):
x = x.reshape((1, 3, img_width, img_height))
outs = f_outputs([x])
loss_value = outs[0]
if len(outs[1:]) == 1:
grad_values = outs[1].flatten().astype('float64')
else:
grad_values = np.array(outs[1:]).flatten().astype('float64')
return loss_value, grad_values
"""
"""
b1, b2 = pearsonr(Y_NN, H_e)
print "NN rho-value"
print b1
S, unique, pk = DataFunctions.calculate_entropy(H_t)
print "The entropy value is: ", S
#Saving files
numpy.save('bldg1_B.npy', Y_lstm2)
numpy.save('bldg1_MLP.npy', Y_NN)
############################################################
###########Extracting Data for multi-timescale Analysis
get_1st_layer_output = K.function([save_model.layers[0].input],
[save_model.layers[1].output])
lstm_h1 = int(lstm_hidden['Layer1'])
h_input = numpy.zeros((len(train_data2), lstm_h1))
h_val = numpy.zeros((len(val_data2), lstm_h1))
h_test = numpy.zeros((len(test_data2), lstm_h1))
###Trouble shoot###
print "Len(train1)", len(train_data)
print "Len(train2)", len(train_data2)
for i in range(0, len(train_data)):
X_temp = train_data[i, :, :]
X_temp = X_temp[None, :, :]
h_temp = numpy.asarray(get_1st_layer_output([X_temp]))
h_input[i * 24:(i + 1) * 24, :] = h_temp[0, :]
def __init__(self, replay_filename, group_name, model_filename=''):
# Set learning phase to TEST
self.learning_phase = TEST_MODE
# Loads ReplayData file
self.replay_data = h5py.File('{}'.format(replay_filename), 'r')
try:
self.group = self.replay_data[group_name]
except KeyError:
self.group = self.replay_data[group_name + '_init']
group_name += '_init'
self.group_name = group_name
# If not informed, defaults to '_model' suffix
if model_filename == '':
model_filename = '{}_model.h5'.format(group_name)
# Loads Keras model
self.model = load_model(model_filename)
# Retrieves some basic information from the replay data
self.inputs = self.group['inputs'][:]
self.targets = self.group['targets'][:]
self.n_epochs = self.group.attrs['n_epochs']
self.n_layers = self.group.attrs['n_layers']
# Generates ranges for the number of different weight arrays in each layer
self.n_weights = [
range(len(self.group['layer{}'.format(l)]))
for l in range(self.n_layers)
]
# Retrieves weights as a list, each element being one epoch
self.weights = self._retrieve_weights()
# Gets Tensors for the weights in the same order as the layers
# Keras' model.weights returns the Tensors in a different order!
self._model_weights = [
w for layer in self.model.layers for w in layer.weights
]
### Functions
# Keras function to get the outputs, given inputs and weights
self._get_output = K.function(inputs=[K.learning_phase()] +
self.model.inputs + self._model_weights,
outputs=[self.model.layers[-1].output])
# Keras function to get the loss and metrics, given inputs, targets, weights and sample weights
self._get_metrics = K.function(
inputs=[K.learning_phase()] + self.model.inputs +
self.model.targets + self._model_weights +
self.model.sample_weights,
outputs=[self.model.total_loss] + self.model.metrics_tensors)
# Keras function to compute the binary cross entropy, given inputs, targets, weights and sample weights
self._get_binary_crossentropy = K.function(
inputs=[K.learning_phase()] + self.model.inputs +
self.model.targets + self._model_weights +
self.model.sample_weights,
outputs=[
K.binary_crossentropy(self.model.targets[0],
self.model.outputs[0])
])
# Keras function to compute the gradients for trainable weights, given inputs, targets, weights and
# sample weights
self.__trainable_weights = [
w for layer in self.model.layers for w in layer.trainable_weights
if layer.trainable and ('bias' not in w.op.name)
]
self.__trainable_gradients = self.model.optimizer.get_gradients(
self.model.total_loss, self.__trainable_weights)
self._get_gradients = K.function(
inputs=[K.learning_phase()] + self.model.inputs +
self.model.targets + self._model_weights +
self.model.sample_weights,
outputs=self.__trainable_gradients)
def get_z_op(layer):
op = layer.output.op
if op.type in Z_OPS:
return layer.output
else:
op_layer_name = op.name.split('/')[0]
for input in op.inputs:
input_layer_name = input.name.split('/')[0]
if (input.op.type in Z_OPS) and (op_layer_name
== input_layer_name):
return input
return None
__z_layers = np.array([
i for i, layer in enumerate(self.model.layers)
if get_z_op(layer) is not None
])
__act_layers = np.array([
i for i, layer in enumerate(self.model.layers)
if layer.output.op.type.lower() in ACTIVATIONS
])
__z_layers = np.array([
__z_layers[np.argmax(layer < __z_layers) - 1]
for layer in __act_layers
])
self.z_act_layers = [self.model.layers[i].name for i in __z_layers]
self._z_layers = ['inputs'
] + [self.model.layers[i].name for i in __z_layers]
self._z_tensors = [K.identity(self.model.inputs)] + list(
filter(lambda t: t is not None,
[get_z_op(self.model.layers[i]) for i in __z_layers]))
self._activation_layers = ['inputs'] + [
self.model.layers[i].name for i in __act_layers
]
self._activation_tensors = [K.identity(self.model.inputs)] + [
self.model.layers[i].output for i in __act_layers
]
# Keras function to compute the Z values given inputs and weights
self._get_zvalues = K.function(inputs=[K.learning_phase()] +
self.model.inputs + self._model_weights,
outputs=self._z_tensors)
# Keras function to compute the activation values given inputs and weights
self._get_activations = K.function(inputs=[K.learning_phase()] +
self.model.inputs +
self._model_weights,
outputs=self._activation_tensors)
# Gets names of all layers with arrays of weights of lengths 1 (no biases) or 2 (with biases)
# Layers without weights (e.g. Activation, BatchNorm) are not included
self.weights_layers = [
layer.name
for layer, weights in zip(self.model.layers, self.n_weights)
if len(weights) in (1, 2)
]
# Attributes for the visualizations - Data
self._feature_space_data = None
self._loss_hist_data = None
self._loss_and_metric_data = None
self._prob_hist_data = None
self._decision_boundary_data = None
self._weights_violins_data = None
self._activations_violins_data = None
self._zvalues_violins_data = None
self._gradients_data = None
# Attributes for the visualizations - Plot objects
self._feature_space_plot = None
self._loss_hist_plot = None
self._loss_and_metric_plot = None
self._prob_hist_plot = None
self._decision_boundary_plot = None
self._weights_violins_plot = None
self._activations_violins_plot = None
self._zvalues_violins_plot = None
self._gradients_plot = None
else:
model.add(Dense(k2, activation='softmax', input_dim=k1))
model.add(Dense(cY, activation='softmax', input_dim=k2))
model.layers[0].trainable = False
sgd = SGD(lr=4, decay=1e-6, momentum=0.9, nesterov=True, clipnorm=.01)
model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
# with a Sequential model
start_time = time.time()
Pxy_hat = get_p_hat_mat(XLabels, YLabels)
Bxy_hat = p2b(Pxy_hat)
Px_hat = np.sum(Pxy_hat, axis=0)
Py_hat = np.sum(Pxy_hat, axis=1)
U, s, V = np.linalg.svd(Bxy_hat)
s1 = K.function([model.layers[0].input],
[model.layers[0].output])([np.eye(cX)])[0]
# get the value of s1.
# Return the output of the first layer given a certain input np.eye(cX)
s1m = np.matmul(Px_hat.reshape(1, -1), s1)
Phi1 = info_mat(s1, Px_hat)
plt.figure(figsize=(9, 3))
model.fit(XLabels, YLabels, verbose=0, batch_size=batch_size, epochs=nEpochs)
v2 = model.get_weights()[4].T
Psi2 = info_mat(v2, Py_hat)
b2 = model.get_weights()[5]
b20 = b2 - np.dot(b2, Py_hat)
s2 = K.function([model.layers[0].input],
[model.layers[1].output])([np.eye(cX)])[0]
s2m = np.matmul(Px_hat.reshape(1, -1), s2)
print("\nCompiling model\n")
tgt = Input(shape=(max_sents,SENT_DIM), dtype='float32')
srcs = Input(shape=(max_sents,SENT_DIM), dtype='float32')
attention = _Attention(max_sents,SENT_DIM,dropout=0.2)(tgt,srcs)
align1= _SoftAlignment(max_sents,SENT_DIM)(srcs,attention)
align2= _SoftAlignment(max_sents,SENT_DIM)(tgt,attention,transpose=True)
vec_l = _Comparison(max_sents,SENT_DIM,dropout=0.2)(tgt,align1)
vec_r = _Comparison(max_sents,SENT_DIM,dropout=0.2)(srcs,align2)
pds = _Entailment(SENT_DIM,NUM_CLASSES,dropout=0.2)(vec_l,vec_r)
model = Model(inputs=[tgt,srcs],outputs=pds)
model.summary()
model.compile(loss="categorical_crossentropy",optimizer=Adam(lr=0.001),metrics=["accuracy"])
cb = [ModelCheckpoint("temp1_model.hdf5",monitor="val_loss",verbose=1,save_weights_only=True,save_best_only=True)]
get_attention_matrix = K.function([model.layers[0].input,model.layers[1].input,K.learning_phase()],[model.layers[3].output])
NUM_EPOCHS = 30
BATCH_SIZE = 25
print("\nTraining model\n")
history = model.fit(x=[target_vecs[train],source_vecs[train]],y=gold[train],batch_size=BATCH_SIZE,validation_split=0.1,epochs=NUM_EPOCHS,shuffle=True,verbose=1,callbacks=cb)
model.load_weights("temp1_model.hdf5")
preds = model.predict([target_vecs[test],source_vecs[test]])
probs.append(preds)
preds = np.argmax(preds,axis=1)
gold_test = np.argmax(gold[test],axis=1)
predictions.append(preds)
golds.append(gold_test)
target.append(targets[test])
def plot_sample(sample_number, input_number, neurons, x_train_dstack,
y_train_dstack, model, seq_dur, i, plot_dir, f, string_name):
frecuencias = []
seq_dur = len(x_train_dstack[sample_number, :, 0])
test = x_train_dstack[sample_number:sample_number + 1, :, :]
colors = cm.rainbow(np.linspace(0, 1, neurons + 1))
y_pred = model.predict(test)
###################################
# Status for the sample value at the layer indicated
capa = 0
#First Layer:
get_0_layer_output = K.function([model.layers[capa].input],
[model.layers[capa].output])
layer_output = get_0_layer_output([test])[capa]
#layer_output= model.layers[0].output
#print("layer_output",layer_output)
#Second Layer:
get_1_layer_output = K.function([model.layers[capa].input],
[model.layers[capa].output])
#layer_output_1 = get_1_layer_output([test])[capa]
layer_output_T = layer_output.T
print("layer_output", layer_output_T)
array_red_list = []
####################################
y_pred = model.predict(test)
# To generate the Populational Analysis
for ii in np.arange(0, neurons, 1):
neurona_serie = np.reshape(layer_output_T[ii], len(layer_output_T[ii]))
array_red_list.append(neurona_serie)
#SVD and PCA with scikit learn
array_red = np.asarray(array_red_list)
sdv = sklearn.decomposition.TruncatedSVD(n_components=2)
sdv_3d = sklearn.decomposition.TruncatedSVD(n_components=3)
X_2d = sdv.fit_transform(array_red.T)
X_3d = sdv_3d.fit_transform(array_red.T)
pca = PCA(n_components=3)
X_pca_ = pca.fit(array_red)
X_pca = pca.components_
####################################
#2-Dim plots
fig = plt.figure()
fig.suptitle("SDV Network Population Analysis", fontsize=20)
ax1 = fig.add_subplot(111)
ax1.plot(X_2d[:, 0],
X_2d[:, 1],
c='g',
marker="p",
zorder=2,
label='Dim Reduction of the network')
ax1.scatter(X_2d[0, 0], X_2d[0, 1], c='r', marker='^', s=70, label='start')
ax1.scatter(X_2d[-1, 0],
X_2d[-1, 1],
c='b',
marker='^',
s=70,
label='stop')
plt.legend(loc='upper left', fontsize='x-small')
plt.ylabel('C1')
plt.xlabel('C2')
plt.ylim([-3, 3])
plt.xlim([-4, 4])
figname = str(plot_dir) + "/sample_" + str(sample_number) + "_sdv_" + str(
capa) + "_individual_neurons_state_" + str(i) + ".png"
#plt.savefig(figname,dpi=200)
plt.close()
print("------------")
ordeno_primero_x = X_pca[0]
ordeno_primero_y = X_pca[1]
ordeno_primero_z = X_pca[2]
fig = plt.figure()
fig.suptitle("PCA Network Population Analysis", fontsize=20)
ax1 = fig.add_subplot(111)
ax1.plot(X_pca[0],
X_pca[1],
c='c',
marker="p",
zorder=2,
label='Dim Reduction of the network')
ax1.scatter(ordeno_primero_x[0],
ordeno_primero_y[0],
s=70,
c='r',
marker="^",
label='start')
ax1.scatter(ordeno_primero_x[-1],
ordeno_primero_y[-1],
s=70,
c='b',
marker="^",
label='stop')
plt.legend(loc='upper left', fontsize='x-small')
plt.ylabel('C1')
plt.xlabel('C2')
plt.ylim([-0.3, 0.3])
plt.xlim([-0.15, 0.15])
figname = str(plot_dir) + "/sample_" + str(sample_number) + "_pca_" + str(
capa) + "_individual_neurons_state_" + str(i) + ".png"
#plt.savefig(figname,dpi=200)
plt.close()
#pp.show()
####################################
#3-Dim Plots
# How many 3d angular views you want to define
yy = np.arange(0, 180, 10)
#yy = np.arange(0,90,10)
#yy = np.arange(70,80,10)
for ii, kk in enumerate(yy):
print("ii: ", ii, " kk: ", kk)
fig = plt.figure(figsize=(10, 8))
fig.suptitle("3D plot SDV Network Population Analysis", fontsize=20)
ax = fig.add_subplot(111, projection='3d')
ax.plot(X_3d[:, 0],
X_3d[:, 1],
X_3d[:, 2],
color='gray',
zorder=2,
label="3 d plot",
marker="p")
#The the approximate vertices
ax.scatter(X_3d[0, 0],
X_3d[0, 1],
X_3d[0, 2],
c='r',
marker="^",
label='start',
s=300)
ax.scatter(X_3d[-1, 0],
X_3d[-1, 1],
X_3d[-1, 2],
c='b',
marker="^",
label='stop',
s=300)
ax.scatter(X_3d[40, 0],
X_3d[40, 1],
X_3d[40, 2],
c='deepskyblue',
marker="^",
label='v1: (0,0,0)',
s=300)
ax.scatter(X_3d[100, 0],
X_3d[100, 1],
X_3d[100, 2],
c='gold',
marker="^",
label='v2: (1,1,1)',
s=300)
ax.scatter(X_3d[180, 0],
X_3d[180, 1],
X_3d[180, 2],
c='pink',
marker="^",
label='v3: (1,0,1)',
s=300)
ax.scatter(X_3d[240, 0],
X_3d[240, 1],
X_3d[240, 2],
c='green',
marker="^",
label='v4: (0,1,1)',
s=300)
ax.scatter(X_3d[320, 0],
X_3d[320, 1],
X_3d[320, 2],
c='m',
marker="^",
label='v5:(0,0,1)',
s=300)
ax.scatter(X_3d[380, 0],
X_3d[380, 1],
X_3d[380, 2],
c='y',
marker="^",
label='v6: (1,1,0)',
s=300)
ax.scatter(X_3d[460, 0],
X_3d[460, 1],
X_3d[460, 2],
c='hotpink',
marker="^",
label='v7: (1,0,0)',
s=300)
ax.scatter(X_3d[520, 0],
X_3d[520, 1],
X_3d[520, 2],
c='deeppink',
marker="^",
label='v8:(0,1,0)',
s=300)
ax.set_xlabel('comp 1 (arb. units)', size=16)
ax.set_ylabel('comp 2 (arb. units)', size=16)
ax.set_zlabel('comp 3 (arb. units)', size=16)
ax.legend()
ax.view_init(elev=10, azim=kk)
#ax.view_init(elev=kk, azim=10)
figname = str(plot_dir) + "/sample_" + str(
sample_number) + "_sdv_3d_" + str(
capa) + "_individual_neurons_state_" + str(i) + '_' + str(
kk) + ".png"
plt.savefig(figname, dpi=200)
plt.close()
####################################
#kk=70
for ii, kk in enumerate(yy):
fig = plt.figure(figsize=(18, 7))
plt.subplot(3, 2, 1)
plt.plot(test[0, :, 0], color='pink', label='Input 1')
plt.plot(y_train_dstack[sample_number, :, 0],
color='grey',
linewidth=3,
label='Target Output 1')
plt.plot(y_pred[0, :, 0],
color='r',
linewidth=2,
label=' Output\n 25 individual states')
plt.legend(fontsize='x-small', loc=3)
plt.subplot(3, 2, 3)
plt.plot(test[0, :, 1], color='pink', label='Input 2')
plt.plot(y_train_dstack[sample_number, :, 1],
color='grey',
linewidth=3,
label='Target Output2')
plt.plot(y_pred[0, :, 1], color='r', linewidth=2, label=' Output 2')
plt.xlim(0, seq_dur + 1)
plt.ylim([-1.5, 1.5])
plt.yticks([])
#plt.ylabel('Activity [arb. units]',fontsize = 16)
#plt.xlabel('time [mS]',fontsize = 16)
#plt.xticks(np.arange(0,seq_dur+1,20),fontsize = 8)
plt.legend(fontsize='x-small', loc=1)
plt.subplot(3, 2, 5)
plt.plot(test[0, :, 2], color='pink', label='Input 3')
plt.plot(y_train_dstack[sample_number, :, 2],
color='grey',
linewidth=3,
label='Target Output 3')
plt.plot(y_pred[0, :, 2], color='r', linewidth=2, label=' Output')
plt.xlim(0, seq_dur + 1)
plt.ylim([-1.5, 1.5])
plt.yticks([])
#plt.ylabel('Activity [arb. units]',fontsize = 16)
plt.xlabel('time [mS]', fontsize=16)
#plt.xticks(np.arange(0,seq_dur+1,20),fontsize = 8)
plt.legend(fontsize='x-small', loc=1)
fig.suptitle("Time series and PCA 3D plot", fontsize=20)
ax = fig.add_subplot(122, projection='3d')
x = X_pca[0]
y = X_pca[1]
z = X_pca[2]
N = len(z)
ax.plot(X_3d[:, 0],
X_3d[:, 1],
X_3d[:, 2],
color='gray',
zorder=2,
label="3 d plot",
marker="p")
ax.scatter(X_3d[0, 0],
X_3d[0, 1],
X_3d[0, 2],
c='r',
marker="^",
label='start',
s=300)
ax.scatter(X_3d[-1, 0],
X_3d[-1, 1],
X_3d[-1, 2],
c='b',
marker="^",
label='stop',
s=300)
ax.set_xlabel(' 1 (arb. units)', size=16)
ax.set_ylabel(' 2 (arb. units)', size=16)
ax.set_zlabel(' 3 (arb. units)', size=16)
ax.axes.get_xaxis().set_ticks([])
ax.axes.get_yaxis().set_ticks([])
ax.set_zticks(())
ax.view_init(elev=10, azim=kk)
#ax.view_init(elev=kk, azim=10)
ax.legend(fontsize='small')
fig.text(0.1,
0.5,
'Amplitude [Arb. Units]',
va='center',
ha='center',
rotation='vertical',
fontsize=16)
figname = str(plot_dir) + "/sample_" + str(
sample_number) + "_pca_3D_" + str(
capa) + "_individual_neurons_state_" + str(i) + '_' + str(
kk) + "_" + str(f) + ".png"
plt.savefig(figname, dpi=300, bbox_inches='tight')
plt.close()
# create layer name to layer dictionary
layer_dict = {layer.name: layer for layer in model.layers}
print(layer_dict)
# layer_dict
# visualize gradients at pooling layers
num_pool_layers = 5
lr = 0.01
fig, axes = plt.subplots(1, num_pool_layers, figsize=(20, 10))
for i in range(num_pool_layers):
layer_name = "block{:d}_pool".format(i + 1)
layer_output = layer_dict[layer_name].output
loss = K.mean(layer_output)
grads = K.gradients(loss, dream)[0]
grads /= (K.sqrt(K.mean(K.square(grads))) + 1e-5) * lr
f = K.function([dream], [loss, grads])
img_value = p_img.copy()
loss_value, grads_value = f([img_value])
axes[i].set_title(layer_name)
axes[i].imshow(deprocess(grads_value))
plt.tight_layout()
plt.show()
# deep dreaming
first_layer = model.layers[-1]
input_img = first_layer.input
print(first_layer.name, first_layer.output_shape)
num_pool_layers = 5
num_iters_per_layer = 3
def saveData(model, X_train, y_train, X_test, y_test, nb_classes):
for layer_index in range(1, len(model.layers), 1):
if layer_index % 10 == 0:
print('Layer index is ' + str(layer_index))
shouldIContinue = True
if 'conv2d' in model.layers[layer_index].name and layer_index > 30:
shouldIContinue = False
if 'add' in model.layers[layer_index].name:
shouldIContinue = False
if 'flatten' in model.layers[layer_index].name:
shouldIContinue = False
if 'dense' in model.layers[layer_index].name:
shouldIContinue = False
if shouldIContinue:
continue
if layer_index % 20 == 0:
print("think about it")
directory = 'C:\cifar//residual//layer_output//layer_' + str(
layer_index) + '_' + model.layers[layer_index].name
if os.path.exists(directory):
print('Folder already exists')
else:
os.mkdir(directory)
get_k_layer_output = K.function([model.layers[0].input],
[model.layers[layer_index].output])
batchSize = 2000
with open(directory + '//trainingData.txt', 'w',
newline='') as csvfile:
spamwriter = csv.writer(csvfile, delimiter=' ')
for startingImageIndex in range(int(len(X_train) / batchSize)):
sample = X_train[startingImageIndex *
batchSize:startingImageIndex * batchSize +
batchSize]
layer_output = get_k_layer_output([sample])[0]
for imageIndex in range(len(sample)):
output = layer_output[imageIndex].flatten()
index = np.argmax(output)
row = [0] * len(output)
row[index] = 1
spamwriter.writerow(output)
with open(directory + '//trainingLabel.txt', 'w',
newline='') as csvfile:
spamwriter = csv.writer(csvfile, delimiter=' ')
for label in y_train:
row = [0] * nb_classes
row[label[0]] = 1
spamwriter.writerow(row)
with open(directory + '//testingData.txt', 'w', newline='') as csvfile:
spamwriter = csv.writer(csvfile, delimiter=' ')
for startingImageIndex in range(int(len(X_test) / batchSize)):
sample = X_test[startingImageIndex:startingImageIndex +
batchSize]
layer_output = get_k_layer_output([sample])[0]
for imageIndex in range(len(sample)):
output = layer_output[imageIndex].flatten()
index = np.argmax(output)
row = [0] * len(output)
row[index] = 1
spamwriter.writerow(output)
with open(directory + '//testingLabel.txt', 'w',
newline='') as csvfile:
spamwriter = csv.writer(csvfile, delimiter=' ')
for label in y_test:
row = [0] * nb_classes
row[label[0]] = 1
spamwriter.writerow(row)
def test_time_distributed_softmax():
x = K.placeholder(shape=(1, 1, 5))
f = K.function([x], [activations.softmax(x)])
test_values = get_standard_values()
test_values = np.reshape(test_values, (1, 1, np.size(test_values)))
f([test_values])[0]
from keras.models import load_model
from keras import backend as K
import pandas as pd
import numpy as np
import pickle
'''
根据训练的脸型模型，提取倒数第二层全连接层的信息，作为特征
'''
if __name__ == '__main__':
model = load_model('./model/vggface1-weights-improvement-03-0.82.hdf5')
layer_1 = K.function([model.layers[0].input], [model.layers[-2].output])
# data
print('read data...')
train_data = np.load('./train/images.npy')
train_data = train_data / 255.0
train_data = np.transpose(train_data, (0, 3, 1, 2))
print('train_data shape: ', train_data.shape)
print('Data is done!')
batch = 32
print('get features...')
for i in range(int(np.ceil(len(train_data) / batch))):
if (i + 1) * batch > len(train_data):
batch_data = train_data[i * batch:len(train_data)]
else:
batch_data = train_data[i * batch:(i + 1) * batch]
if i == 0:
f_vgg = layer_1([batch_data])[0]
# print(np.shape(f_vgg))
def CreateModel(self):
'''
定义CNN/LSTM/CTC模型，使用函数式模型
输入层：39维的特征值序列，一条语音数据的最大长度设为1500（大约15s）
隐藏层一：1024个神经元的卷积层
隐藏层二：池化层，池化窗口大小为2
隐藏层三：Dropout层，需要断开的神经元的比例为0.2，防止过拟合
隐藏层四：循环层、LSTM层
隐藏层五：Dropout层，需要断开的神经元的比例为0.2，防止过拟合
隐藏层六：全连接层，神经元数量为self.MS_OUTPUT_SIZE，使用softmax作为激活函数，
输出层：自定义层，即CTC层，使用CTC的loss作为损失函数，实现连接性时序多输出
'''
# 每一帧使用13维mfcc特征及其13维一阶差分和13维二阶差分表示，最大信号序列长度为1500
input_data = Input(name='the_input',
shape=(self.AUDIO_LENGTH, self.AUDIO_FEATURE_LENGTH,
1))
layer_h1 = Conv2D(32, (3, 3),
use_bias=True,
activation='relu',
padding='same',
kernel_initializer='he_normal')(input_data) # 卷积层
layer_h2 = Conv2D(32, (3, 3),
use_bias=True,
activation='relu',
padding='same',
kernel_initializer='he_normal')(layer_h1) # 卷积层
layer_h3 = MaxPooling2D(pool_size=2, strides=None,
padding="valid")(layer_h2) # 池化层
#layer_h3 = Dropout(0.2)(layer_h2) # 随机中断部分神经网络连接，防止过拟合
layer_h4 = Conv2D(64, (3, 3),
use_bias=True,
activation='relu',
padding='same',
kernel_initializer='he_normal')(layer_h3) # 卷积层
layer_h5 = Conv2D(64, (3, 3),
use_bias=True,
activation='relu',
padding='same',
kernel_initializer='he_normal')(layer_h4) # 卷积层
layer_h6 = MaxPooling2D(pool_size=2, strides=None,
padding="valid")(layer_h5) # 池化层
#test=Model(inputs = input_data, outputs = layer_h6)
#test.summary()
layer_h7 = Reshape((400, 3200))(layer_h6) #Reshape层
#layer_h5 = LSTM(256, activation='relu', use_bias=True, return_sequences=True)(layer_h4) # LSTM层
#layer_h6 = Dropout(0.2)(layer_h5) # 随机中断部分神经网络连接，防止过拟合
layer_h8 = Dense(256,
activation="relu",
use_bias=True,
kernel_initializer='he_normal')(layer_h7) # 全连接层
layer_h9 = Dense(1417, use_bias=True,
kernel_initializer='he_normal')(layer_h8) # 全连接层
y_pred = Activation('softmax', name='Activation0')(layer_h9)
model_data = Model(inputs=input_data, outputs=y_pred)
#model_data.summary()
#labels = Input(name='the_labels', shape=[60], dtype='float32')
labels = Input(name='the_labels',
shape=[self.label_max_string_length],
dtype='float32')
input_length = Input(name='input_length', shape=[1], dtype='int64')
label_length = Input(name='label_length', shape=[1], dtype='int64')
# Keras doesn't currently support loss funcs with extra parameters
# so CTC loss is implemented in a lambda layer
#layer_out = Lambda(ctc_lambda_func,output_shape=(self.MS_OUTPUT_SIZE, ), name='ctc')([y_pred, labels, input_length, label_length])#(layer_h6) # CTC
loss_out = Lambda(self.ctc_lambda_func, output_shape=(1, ),
name='ctc')(
[y_pred, labels, input_length, label_length])
model = Model(inputs=[input_data, labels, input_length, label_length],
outputs=loss_out)
model.summary()
# clipnorm seems to speeds up convergence
#sgd = SGD(lr=0.0001, decay=1e-6, momentum=0.9, nesterov=True, clipnorm=5)
ada_d = Adadelta(lr=0.01, rho=0.95, epsilon=1e-06)
#model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer=sgd, metrics=["accuracy"])
model.compile(loss={
'ctc': lambda y_true, y_pred: y_pred
},
optimizer=ada_d,
metrics=['accuracy'])
# captures output of softmax so we can decode the output during visualization
test_func = K.function([input_data], [y_pred])
print('[*提示] 创建模型成功，模型编译成功')
return model, model_data
def generateAdversarialExample(self):
start_time = time.time()
model_layer_times1 = self.init_coverage_times(
) # times of each neuron covered
model_layer_times2 = self.init_coverage_times(
) # update when new image and adversarial images found
model_layer_value1 = self.init_coverage_value()
threshold = 0.25
neuron_to_cover_num = 10
iteration_times = 0
neuron_to_cover_weight = 0.5
predict_weight = 0.5
learning_step = 0.02
total_norm = 0
img_list = []
adv_list = []
adv_labels = []
while len(adv_list) < 1:
iteration_times += 1
total_perturb_adversarial = 0
tmp_img = self.image.reshape(1, 28, 28, 1)
orig_img = tmp_img.copy()
img_list.append(tmp_img)
self.update_coverage(tmp_img, model_layer_times2, threshold)
while len(img_list) > 0:
gen_img = img_list[0]
img_list.remove(gen_img)
# first check if input already induces differences
pred1 = self.model.predict(gen_img)
label1 = np.argmax(pred1[0])
label_top5 = np.argsort(pred1[0])[-5:]
self.update_coverage_value(gen_img, model_layer_value1)
self.update_coverage(gen_img, model_layer_times1, threshold)
orig_label = label1
loss_1 = k.mean(
self.model.get_layer('before_softmax').output[...,
orig_label])
loss_2 = k.mean(
self.model.get_layer('before_softmax').output[
..., label_top5[-2]])
loss_3 = k.mean(
self.model.get_layer('before_softmax').output[
..., label_top5[-3]])
loss_4 = k.mean(
self.model.get_layer('before_softmax').output[
..., label_top5[-4]])
loss_5 = k.mean(
self.model.get_layer('before_softmax').output[
..., label_top5[-5]])
layer_output = (predict_weight *
(loss_2 + loss_3 + loss_4 + loss_5) - loss_1)
# neuron coverage loss
loss_neuron = self.neuron_selection(model_layer_times1,
neuron_to_cover_num)
# extreme value means the activation value for a neuron can be as high as possible ...
layer_output += neuron_to_cover_weight * k.sum(loss_neuron)
# for adversarial image generation
final_loss = k.mean(layer_output)
# we compute the gradient of the input picture wrt this loss
grads = self.normalize(
k.gradients(final_loss, self.model.input)[0])
grads_tensor_list = [loss_1, loss_2, loss_3, loss_4, loss_5]
grads_tensor_list.extend(loss_neuron)
grads_tensor_list.append(grads)
# this function returns the loss and grads given the input picture
iterate = k.function([self.model.input], grads_tensor_list)
# we run gradient ascent for 3 steps
for iters in range(iteration_times):
loss_neuron_list = iterate([gen_img])
perturb = loss_neuron_list[-1] * learning_step
gen_img += perturb
gen_img = np.clip(gen_img, -0.5, 5)
# previous accumulated neuron coverage
previous_coverage = self.neuron_covered(
model_layer_times1)[2]
pred1 = self.model.predict(gen_img)
label1 = np.argmax(pred1[0])
self.update_coverage(gen_img, model_layer_times1,
threshold) # for seed selection
current_coverage = self.neuron_covered(
model_layer_times1)[2]
diff_img = gen_img - orig_img
L2_norm = np.linalg.norm(diff_img)
orig_L2_norm = np.linalg.norm(orig_img)
perturb_adversial = L2_norm / orig_L2_norm
if current_coverage - previous_coverage > 0.0001 and L2_norm < self.similarityMeasure:
img_list.append(np.clip(gen_img, -0.5, 5))
if label1 != orig_label and L2_norm < self.similarityMeasure:
self.update_coverage(gen_img, model_layer_times2,
threshold)
total_norm += L2_norm
total_perturb_adversarial += perturb_adversial
gen_img = np.clip(gen_img, -0.5, 5)
adv_list.append(gen_img)
adv_labels.append(
np.argmax(self.model.predict(gen_img)[0]))
end_time = time.time()
self.time = end_time - start_time
self.advLabel = adv_labels[0]
self.advImage = adv_list[0]
self.completed = True
if self.verbose:
print('\ncovered neurons percentage %d neurons %.3f' %
(len(model_layer_times2),
self.neuron_covered(model_layer_times2)[2]))
negatives = np.random.choice(possibilities, J)
neg_l_Ds.append([pos_l_Ds[negative] for negative in negatives])
# Because we're using the "categorical_crossentropy" loss function, we can pretend that
# we're dealing with a multi-class classification problem and that every sample is a
# member of the "0" class.
y = np.zeros(J + 1).reshape(1, J + 1)
y[0][0] = 1
for i in range(sample_size):
history = model.fit([l_Qs[i], pos_l_Ds[i]] + neg_l_Ds[i],
y,
nb_epoch=1,
verbose=0)
# Here, I walk through how to define a function for calculating output from the
# computational graph. Let's define a function that calculates R(Q, D+) for a given
# query and clicked document. The function depends on two inputs, query and pos_doc.
# That is, if you start at the point in the graph where R(Q, D+) is calculated
# and then work backwards as far as possible, you'll end up at two different starting
# points: query and pos_doc. As a result, we supply those inputs in a list to the
# function. This particular function only calculates a single output, but multiple
# outputs are possible (see the next example).
get_R_Q_D_p = backend.function([query, pos_doc], R_Q_D_p)
get_R_Q_D_p([l_Qs[0], pos_l_Ds[0]])
# A slightly more complex function. Notice that both neg_docs and the output are
# lists.
get_R_Q_D_ns = backend.function([query] + neg_docs, R_Q_D_ns)
get_R_Q_D_ns([l_Qs[0]] + neg_l_Ds[0])
def build_model(self):
"""Build an actor (policy) network that maps states -> actions."""
# Define input layer (states)
states = layers.Input(shape=(self.state_size, ), name='states')
# Add hidden layers
net = layers.Dense(units=32,
use_bias=False,
activation='relu',
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01))(states)
net = layers.BatchNormalization()(net)
net = layers.Dropout(0.5)(net)
net = layers.Dense(units=64,
use_bias=False,
activation='relu',
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01))(net)
net = layers.BatchNormalization()(net)
net = layers.Dropout(0.5)(net)
net = layers.Dense(units=128,
use_bias=False,
activation='relu',
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01))(net)
net = layers.BatchNormalization()(net)
net = layers.Dropout(0.5)(net)
net = layers.Dense(units=64,
use_bias=False,
activation='relu',
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01))(net)
net = layers.BatchNormalization()(net)
net = layers.Dropout(0.5)(net)
net = layers.Dense(units=128,
use_bias=False,
activation='relu',
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01))(net)
net = layers.BatchNormalization()(net)
net = layers.Dropout(0.5)(net)
# Add final output layer with sigmoid activation
raw_actions = layers.Dense(units=self.action_size,
activation='sigmoid',
name='raw_actions')(net)
# Scale [0, 1] output for each action dimension to proper range
actions = layers.Lambda(lambda x:
(x * self.action_range) + self.action_low,
name='actions')(raw_actions)
# Create Keras model
self.model = models.Model(inputs=states, outputs=actions)
# Define loss function using action value (Q value) gradients
action_gradients = layers.Input(shape=(self.action_size, ))
loss = K.mean(-action_gradients * actions)
# Incorporate any additional losses here (e.g. from regularizers)
# Define optimizer and training function
optimizer = optimizers.Adam()
updates_op = optimizer.get_updates(params=self.model.trainable_weights,
loss=loss)
self.train_fn = K.function(
inputs=[self.model.input, action_gradients,
K.learning_phase()],
outputs=[],
updates=updates_op)
def get_hidden_output(model, inp):
get_activations = K.function(model.inputs[:1] + [K.learning_phase()], [
model.layers[5].get_output_at(0),
])
activations = get_activations([inp, 0])
return activations[0]
# #So that's what you're going to do right now! # #X_test from the Banknote Authentication dataset and its model are preloaded. Type model.summary() in the console to check it. # Import keras backend import keras.backend as K # Input tensor from the 1st layer of the model inp = model.layers[0].input # Output tensor from the 1st layer of the model out = model.layers[0].output # Define a function from inputs to outputs inp_to_out = K.function([inp], [out]) # Print the results of passing X_test through the 1st layer print(inp_to_out([X_test])) #Neural separation # #Put on your gloves because you're going to perform brain surgery! # #Neurons learn by updating their weights to output values that help them better distinguish between the different output classes in your dataset. You will make use of the inp_to_out() function you just built to visualize the output of two neurons in the first layer of the Banknote Authentication model as it learns. # #The model you built in chapter 2 is ready for you to use, just like X_test and y_test. Paste show_code(plot) in the console if you want to check plot(). # #You're performing heavy duty, once all is done, click through the graphs to watch the separation live! def plot():
def play():
global mouse_pressed
global current_notes
global audio_pause
global needs_update
global current_params
global prev_mouse_pos
global audio_reset
global instrument
global songs_loaded
global autosavenow
global autosavenum
global autosave
global blend
global blendstate
global blendfactor
global keyframe_params
global keyframe_controls
global keyframe_paths
global cur_controls
global keyframe_magnitudes
global blend_slerp
print("Keras version: " + keras.__version__)
K.set_image_data_format('channels_first')
print("Loading encoder...")
model = load_model(dir_name + 'model.h5')
encoder = Model(inputs=model.input,
outputs=model.get_layer('encoder').output)
decoder = K.function([model.get_layer('decoder').input, K.learning_phase()],
[model.layers[-1].output])
print("Loading gaussian/pca statistics...")
latent_means = np.load(dir_name + sub_dir_name + '/latent_means.npy')
latent_stds = np.load(dir_name + sub_dir_name + '/latent_stds.npy')
latent_pca_values = np.load(
dir_name + sub_dir_name + '/latent_pca_values.npy')
latent_pca_vectors = np.load(
dir_name + sub_dir_name + '/latent_pca_vectors.npy')
# open a window
pygame.init()
pygame.font.init()
screen = pygame.display.set_mode((int(window_w), int(window_h)))
notes_surface = screen.subsurface((notes_x, notes_y, notes_w, notes_h))
pygame.display.set_caption('Neural Composer')
# start the audio stream
audio_stream = audio.open(
format=audio.get_format_from_width(2),
channels=1,
rate=sample_rate,
output=True,
stream_callback=audio_callback)
audio_stream.start_stream()
# main loop
running = True
random_song_ix = 0
cur_len = 0
blendcycle = 0
apply_controls()
while running:
# process events
if autosavenow:
# generate random song
current_params = np.clip(np.random.normal(
0.0, 1.0, (num_params,)), -num_sigmas, num_sigmas)
needs_update = True
audio_reset = True
# save slider values
with open("results/history/autosave" + str(autosavenum)+".txt", "w") as text_file:
text_file.write(sub_dir_name + "\n")
text_file.write(str(instrument) + "\n")
for iter in cur_controls:
text_file.write(str(iter) + "\n")
for iter in current_params:
text_file.write(str(iter) + "\n")
# save song as wave
audio_pause = True
audio_reset = True
save_audio = b''
while True:
save_audio += audio_callback(None, 1024, None, None)[0]
if audio_time == 0:
break
wave_output = wave.open('results/history/autosave' + str(autosavenum)+'.wav', 'w')
wave_output.setparams(
(1, 2, sample_rate, 0, 'NONE', 'not compressed'))
wave_output.writeframes(save_audio)
wave_output.close()
audio_pause = False
autosavenum += 1
autosavenow = False
needs_update = True
audio_reset = True
blendcycle += 1
if blend and blendcycle > 10:
blendcycle = 0
if blendstate%2 == 0:
needs_update = True
current_params = np.copy(keyframe_params[int(blendstate/2)])
cur_controls = np.copy(keyframe_controls[int(blendstate/2)])
apply_controls()
elif blendstate%2 == 1:
for x in range(0,len(current_params)):
current_params[x] = (blendfactor * keyframe_params[int(blendstate/2),x]) + ((1-blendfactor)*keyframe_params[((int(blendstate/2))+1)%len(keyframe_paths),x])
if blend_slerp:
magnitude = (blendfactor * keyframe_magnitudes[int(blendstate/2)]) + ((1-blendfactor)*keyframe_magnitudes[((int(blendstate/2))+1)%len(keyframe_paths)])
current_params = current_params * ((sum(current_params*current_params)**-0.5) * magnitude)
for x in range(0,len(cur_controls)):
cur_controls[x] = (blendfactor * keyframe_controls[int(blendstate/2),x]) + ((1-blendfactor)*keyframe_controls[((int(blendstate/2))+1)%len(keyframe_paths),x])
apply_controls()
needs_update = True
for event in pygame.event.get():
if event.type == pygame.QUIT: # QUIT BUTTON HIT
running = False
break
elif event.type == pygame.MOUSEBUTTONDOWN: # MOUSE BUTTON DOWN
if pygame.mouse.get_pressed()[0]:
prev_mouse_pos = pygame.mouse.get_pos()
update_mouse_click(prev_mouse_pos)
update_mouse_move(prev_mouse_pos)
elif pygame.mouse.get_pressed()[2]:
current_params = np.zeros((num_params,), dtype=np.float32)
needs_update = True
elif event.type == pygame.MOUSEBUTTONUP: # MOUSE BUTTON UP
mouse_pressed = 0
prev_mouse_pos = None
elif event.type == pygame.MOUSEMOTION and mouse_pressed > 0: # MOUSE MOTION WHILE PRESSED
update_mouse_move(pygame.mouse.get_pos())
elif event.type == pygame.KEYDOWN:
if event.key == pygame.K_r: # KEYDOWN R
# generate random song
current_params = np.clip(np.random.normal(
0.0, 1.0, (num_params,)), -num_sigmas, num_sigmas)
needs_update = True
audio_reset = True
if event.key == pygame.K_t: # KEYDOWN T
for x in range(int(num_params/3)+1, num_params):
current_params[x] = np.clip(np.random.normal(0.0,1.0), -num_sigmas, num_sigmas)
needs_update = True
if event.key == pygame.K_x: # KEYDOWN X
# generate random song
current_params += np.clip(np.random.normal(
0.0, 0.3, (num_params,)), -num_sigmas, num_sigmas)
needs_update = True
if event.key == pygame.K_a: # KEYDOWN A
autosave = not autosave
if event.key == pygame.K_b: # KEYDOWN B
blend = not blend
blendstate = 0
blendfactor = 1.0
if blend:
audio_pause = True
audio_reset = True
needs_update = True
blendnum = int(input("The number of songs to be blended "))
keyframe_paths = []
keyframe_controls = np.zeros((blendnum,len(cur_controls)),dtype=np.float32)
keyframe_params = np.zeros((blendnum,num_params),dtype=np.float32)
for y in range(blendnum):
fileName = input("The file name of the next song to be blended ")
if "." not in fileName:
fileName = fileName + ".txt"
keyframe_paths.append((fileName))
fo = open("results/history/" + fileName, "r")
if not sub_dir_name == fo.readline()[:-1]:
running = false
print("incompatable with current model")
break
instrument = int(fo.readline())
for x in range(len(cur_controls)):
keyframe_controls[y,x] = float(fo.readline())
for x in range(len(current_params)):
keyframe_params[y,x] = float(fo.readline())
#keyframe_magnitudes[y] = sum(keyframe_params[y]*keyframe_params[y])**0.5
if event.key == pygame.K_e: # KEYDOWN E
# generate random song with larger variance
current_params = np.clip(np.random.normal(0.0, 2.0, (num_params,)), -num_sigmas, num_sigmas)
needs_update = True
audio_reset = True
if event.key == pygame.K_PERIOD:
current_params /= 1.1
needs_update = True
if event.key == pygame.K_COMMA:
current_params *= 1.1
needs_update = True
if event.key == pygame.K_SLASH:
current_params *= -1
needs_update = True
if event.key == pygame.K_UP:
cur_controls[0] = (210.0 - note_threshold + 1) / 200
apply_controls()
if event.key == pygame.K_DOWN:
cur_controls[0] = (210.0 - note_threshold - 1) / 200
apply_controls()
if event.key == pygame.K_s: # KEYDOWN S
# save slider values
audio_pause = True
fileName = input("File Name to save into ")
if "." not in fileName:
fileName = fileName + ".txt"
with open("results/history/" + fileName, "w") as text_file:
if blend:
text_file.write(sub_dir_name + "\n")
text_file.write("blended song" + "\n")
text_file.write(str(len(keyframe_paths)) + "\n")
for x in range(len(keyframe_paths)):
text_file.write("" + keyframe_paths[x] + "\n")
else:
text_file.write(sub_dir_name + "\n")
text_file.write(str(instrument) + "\n")
for iter in cur_controls:
text_file.write(str(iter) + "\n")
for iter in current_params:
text_file.write(str(iter) + "\n")
if event.key == pygame.K_l: # KEYDOWN L
audio_pause = True
needs_update = True
audio_reset = True
fileName = input("File Name to read ")
if "." not in fileName:
fileName = fileName + ".txt"
fo = open("results/history/" + fileName, "r")
print (fo.name)
if not sub_dir_name == fo.readline()[:-1]:
running = false
print("incompatable with current model")
break
tempDir = fo.readline()
if tempDir.startswith("blended song"):
blend = True
blendnum = int(fo.readline())
keyframe_paths = []
keyframe_controls = np.zeros((blendnum,len(cur_controls)),dtype=np.float32)
keyframe_params = np.zeros((blendnum,num_params),dtype=np.float32)
for y in range(blendnum):
fileName2 = fo.readline()[:-1]
keyframe_paths.append(fileName)
fo2 = open("results/history/" + fileName2, "r")
if not sub_dir_name == fo2.readline()[:-1]:
running = false
print("incompatable with current model")
break
instrument = int(fo2.readline())
for x in range(len(cur_controls)):
keyframe_controls[y,x] = float(fo2.readline())
for x in range(len(current_params)):
keyframe_params[y,x] = float(fo2.readline())
else:
instrument = int(tempDir)
for x in range(len(cur_controls)):
cur_controls[x] = float(fo.readline())
for x in range(len(current_params)):
current_params[x] = float(fo.readline())
apply_controls()
if event.key == pygame.K_o: # KEYDOWN O
if not songs_loaded:
print("Loading songs...")
try:
y_samples = np.load('data/interim/samples.npy')
y_lengths = np.load('data/interim/lengths.npy')
songs_loaded = True
except Exception as e:
print("This functionality is to check if the model training went well by reproducing an original song. "
"The composer could not load samples and lengths from model training. "
"If you have the midi files, the model was trained with, process them by using"
" the preprocess_songs.py to find the requested files in data/interim "
"(Load exception: {0}".format(e))
if songs_loaded:
# check how well the autoencoder can reconstruct a random song
print("Random Song Index: " + str(random_song_ix))
if is_ae:
example_song = y_samples[cur_len:cur_len + num_measures]
current_notes = example_song * 255
latent_x = encoder.predict(np.expand_dims(
example_song, 0), batch_size=1)[0]
cur_len += y_lengths[random_song_ix]
random_song_ix += 1
else:
random_song_ix = np.array(
[random_song_ix], dtype=np.int64)
latent_x = encoder.predict(
random_song_ix, batch_size=1)[0]
random_song_ix = (
random_song_ix + 1) % model.layers[0].input_dim
if use_pca:
current_params = np.dot(
latent_x - latent_means, latent_pca_vectors.T) / latent_pca_values
else:
current_params = (
latent_x - latent_means) / latent_stds
needs_update = True
audio_reset = True
if event.key == pygame.K_m: # KEYDOWN M
# save song as midi
audio_pause = True
audio_reset = True
fileName = input("File Name to save into ")
if "." not in fileName:
fileName = fileName + ".mid"
midi_utils.samples_to_midi(
current_notes, 'results/history/' + fileName, note_threshold)
audio_pause = False
if event.key == pygame.K_w: # KEYDOWN W
# save song as wave
audio_pause = True
audio_reset = True
fileName = input("File Name to save into ")
if "." not in fileName:
fileName = fileName + ".wav"
save_audio = b''
while True:
save_audio += audio_callback(None, 1024, None, None)[0]
if audio_time == 0:
break
wave_output = wave.open('results/history/' + fileName + '.wav', 'w')
wave_output.setparams(
(1, 2, sample_rate, 0, 'NONE', 'not compressed'))
wave_output.writeframes(save_audio)
wave_output.close()
audio_pause = False
if event.key == pygame.K_ESCAPE: # KEYDOWN ESCAPE
# exit application
running = False
break
if event.key == pygame.K_SPACE: # KEYDOWN SPACE
# toggle pause/play audio
audio_pause = not audio_pause
if event.key == pygame.K_TAB: # KEYDOWN TAB
# reset audio playing
audio_reset = True
if autosave and not autosavenow:
autosavenow = True
if event.key == pygame.K_1: # KEYDOWN 1
# play instrument 0
instrument = 0
if event.key == pygame.K_2: # KEYDOWN 2
# play instrument 1
instrument = 1
if event.key == pygame.K_3: # KEYDOWN 3
# play instrument 2
instrument = 2
if event.key == pygame.K_4: # KEYDOWN 4
# play instrument 3
instrument = 3
if event.key == pygame.K_5: # KEYDOWN 5
# play instrument 4
instrument = 4
if event.key == pygame.K_c: # KEYDOWN C
#
y = np.expand_dims(
np.where(current_notes > note_threshold, 1, 0), 0)
latent_x = encoder.predict(y)[0]
if use_pca:
current_params = np.dot(
latent_x - latent_means, latent_pca_vectors.T) / latent_pca_values
else:
current_params = (
latent_x - latent_means) / latent_stds
needs_update = True
# check if params were changed so that a new song should be generated
if needs_update:
if use_pca:
latent_x = latent_means + \
np.dot(current_params * latent_pca_values,
latent_pca_vectors)
else:
latent_x = latent_means + latent_stds * current_params
latent_x = np.expand_dims(latent_x, axis=0)
y = decoder([latent_x, 0])[0][0]
current_notes = (y * (255)).astype(np.uint8)
needs_update = False
# draw GUI to the screen
screen.fill(background_color)
draw_notes(screen, notes_surface)
draw_sliders(screen)
draw_controls(screen)
# flip the screen buffer
pygame.display.flip()
pygame.time.wait(10)
# if app is exited, close the audio stream
audio_stream.stop_stream()
audio_stream.close()
audio.terminate()
combination_features = style_layer_output[2, :, :, :]
loss += beta * style_loss(style_features, combination_features) / len(style_layers)
loss += gamma * total_variation_loss(combination_placeholder)
return loss
content_layer = 'block5_conv2'
style_layers = ['block1_conv1', 'block2_conv1',
'block3_conv1', 'block4_conv1',
'block5_conv1']
loss = loss_tensor(content_layer, style_layers)
grads = K.gradients(loss, combination_placeholder)
outputs = [loss] + grads
f_outputs = K.function(inputs=[combination_placeholder, ], outputs=outputs)
class Evaluator:
def __init__(self):
self.loss_values, self.grad_values = None, None
def eval_loss_grads(self, x):
x = x.reshape((1, nrows, ncols, 3))
outs = f_outputs([x])
loss = outs[0]
if len(outs[1:]) == 1:
grads = outs[1].flatten().astype('float64')
else:
grads = np.array(outs[1:], dtype='float64').flatten()
return loss, grads
def build_model(self):
"""Build a critic (value) network that maps (state, action) pairs -> Q-values."""
# Define input layers
states = layers.Input(shape=(self.state_size, ), name='states')
actions = layers.Input(shape=(self.action_size, ), name='actions')
# Add hidden layer(s) for state pathway
net_states = layers.Dense(
units=32,
use_bias=False,
activation='relu',
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01))(states)
net_states = layers.BatchNormalization()(net_states)
net_states = layers.Dropout(0.5)(net_states)
net_states = layers.Dense(
units=64,
use_bias=False,
activation='relu',
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01))(states)
net_states = layers.BatchNormalization()(net_states)
net_states = layers.Dropout(0.5)(net_states)
net_states = layers.Dense(
units=128,
use_bias=False,
activation='relu',
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01))(states)
net_states = layers.BatchNormalization()(net_states)
net_states = layers.Dropout(0.5)(net_states)
net_states = layers.Dense(
units=64,
use_bias=False,
activation='relu',
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01))(states)
net_states = layers.BatchNormalization()(net_states)
net_states = layers.Dropout(0.5)(net_states)
net_states = layers.Dense(
units=128,
use_bias=False,
activation='relu',
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01))(states)
net_states = layers.BatchNormalization()(net_states)
net_states = layers.Dropout(0.5)(net_states)
# Add hidden layer(s) for action pathway
net_actions = layers.Dense(
units=32,
use_bias=False,
activation='relu',
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01))(actions)
net_actions = layers.BatchNormalization()(net_actions)
net_actions = layers.Dropout(0.5)(net_actions)
net_actions = layers.Dense(
units=64,
use_bias=False,
activation='relu',
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01))(net_actions)
net_actions = layers.BatchNormalization()(net_actions)
net_actions = layers.Dropout(0.5)(net_actions)
net_actions = layers.Dense(
units=128,
use_bias=False,
activation='relu',
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01))(net_actions)
net_actions = layers.BatchNormalization()(net_actions)
net_actions = layers.Dropout(0.5)(net_actions)
net_actions = layers.Dense(
units=64,
use_bias=False,
activation='relu',
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01))(net_actions)
net_actions = layers.BatchNormalization()(net_actions)
net_actions = layers.Dropout(0.5)(net_actions)
net_actions = layers.Dense(
units=128,
use_bias=False,
activation='relu',
kernel_regularizer=regularizers.l2(0.01),
activity_regularizer=regularizers.l1(0.01))(net_actions)
net_actions = layers.BatchNormalization()(net_actions)
net_actions = layers.Dropout(0.5)(net_actions)
# Try different layer sizes, activations, add batch normalization, regularizers, etc.
# Combine state and action pathways
net = layers.Add()([net_states, net_actions])
net = layers.Activation('relu')(net)
# Add more layers to the combined network if needed
# Add final output layer to prduce action values (Q values)
Q_values = layers.Dense(units=1, name='q_values')(net)
# Create Keras model
self.model = models.Model(inputs=[states, actions], outputs=Q_values)
# Define optimizer and compile model for training with built-in loss function
optimizer = optimizers.Adam()
self.model.compile(optimizer=optimizer, loss='mse')
# Compute action gradients (derivative of Q values w.r.t. to actions)
action_gradients = K.gradients(Q_values, actions)
# Define an additional function to fetch action gradients (to be used by actor model)
self.get_action_gradients = K.function(
inputs=[*self.model.input, K.learning_phase()],
outputs=action_gradients)
continue
for j in range(1, noiseLevels + 1):
tempImgName = imgName[0:3] + "_" + "{:0>2}".format(i) + "_" + str(
j) + ".bmp"
tempImgScore = mos_scores[np.where(
tempImgName.lower() == mos_names)[0][0]]
allImgNames.append(tempImgName)
allImgScores.append(tempImgScore)
modelIndex = int(float(sys.argv[4]))
model = constructDNNModel(modelIndex)
# get_layer_output = K.function([model.layers[0].input], [model.layers[-2].get_output(train=False)])
get_layer_output = K.function(
[model.inputs[i].input for i in model.input_order],
[model.nodes['dense2'].get_output(train=False)])
# Making the data for the multi-patch network:
hyperImages = np.empty(
(len(allImgScores), denseLayerSize,
float(imgRows - patchSize) / float(patchSize * patchOverlap) + 1,
float(imgCols - patchSize) / float(patchSize * patchOverlap) + 1),
dtype=float)
labels = np.empty((len(allImgScores), ), dtype=float)
# pdb.set_trace()
for i in range(len(allImgNames)):
# print str(i) + "/" + str(len(allImgNames))
imgName = allImgNames[i]
def main():
## retrieve arguments and print out in shell
args = args_parser()
## print out information on shell
info_print(args)
## create output directory if not available ##
#### Keras Model Loading ####
if args.model.lower() == "vgg16":
from keras.applications.vgg16 import VGG16 as keras_model, preprocess_input
elif args.model.lower() == "vgg19":
from keras.applications.vgg19 import VGG19 as keras_model, preprocess_input
## Define local variables in main environment
if not "content/" in args.content_image_path:
content_image_path = "content/" + args.content_image_path
base_path = args.content_image_path
else:
content_image_path = args.content_image_path
base_path = args.content_image_path[-1]
## remove file extension
base_path = os.path.splitext(base_path)[0]
output_subdir = args.output_subdir
if output_subdir is None:
## Create output subdirectory
output_subdir = "output/{}".format(base_path)
if not os.path.exists(output_subdir):
os.makedirs(output_subdir)
else:
if not "output/" in output_subdir:
output_subdir = "output/" + output_subdir
if not os.path.exists(output_subdir):
os.makedirs(output_subdir)
if not "style/" in args.style_image_path:
style_image_path = "style/" + args.style_image_path
else:
style_image_path = args.style_image_path
init_image = args.init_image
image_width = args.image_width
image_height = args.image_height
img_size = (image_height, image_width)
content_weight = args.content_weight
style_weights = args.style_weights
total_variation_weight = args.total_variation_weight
num_iter = args.num_iter
model = args.model
rescale_image = str_to_bool(args.rescale_image)
content_layer = args.content_layer
if args.style_layers == None:
style_layers = [
'block1_conv1', 'block2_conv1', 'block3_conv1', 'block4_conv1',
'block5_conv1'
]
else:
style_layers = args.style_layers
print(style_layers)
original_size = Image.open(content_image_path).size
###### Content Image ######
## Get preprocessed content image array
content_image = preprocess_image(content_image_path, img_size,
preprocess_input)
## Parse content_image numpy array as Keras Backend Variable
content_image = K.variable(content_image,
dtype="float32",
name="content_image")
###### Style Image ######
## Get preprocessed style image array
style_image = preprocess_image(style_image_path, img_size,
preprocess_input)
## Parse style image numpy array as Keras Backend Variable
style_image = K.variable(style_image, dtype="float32", name="style_image")
###### Generated Image ######
## Init generated image as numpy array and parse into Keras Backend Variable
if init_image == "content":
generated_image = preprocess_image(content_image_path, img_size,
preprocess_input)
elif init_image == "random":
generated_image = np.random.randint(256,
size=(image_width, image_height,
3)).astype("float64")
generated_image = preprocess_input(
np.expand_dims(generated_image, axis=0))
else:
import sys
print("wrong init_image")
sys.exit(1)
fname = output_subdir + "/generated_image_at_iteration_0.jpg"
save_img(path=fname, x=generated_image[0])
## Define generate image variable placeholder for later optimization
# Theano
if K.image_data_format() == "channels_first":
generated_image_placeholder = K.placeholder(shape=(1, 3, image_height,
image_width))
# Tensorflow
else:
generated_image_placeholder = K.placeholder(shape=(1, image_height,
image_width, 3))
###### Initialize one keras models with one input tensors which is concatenated by 3 images ######
input_tensor = K.concatenate(
[content_image, style_image, generated_image_placeholder], axis=0)
## input_tensor is a 4D tensor, with shape (3, image_height, image_width, 3) where the first 3 is the concatenation of 3 images and last 3 the color channel (tf)
# build the keras network with our 3 images as input
model = keras_model(input_tensor=input_tensor,
weights='imagenet',
include_top=False)
# get the symbolic outputs of each layer (we gave them unique names). [Feature representations/maps in form of 4D tensors at each layer]
outputs_dict = dict([(layer.name, layer.output) for layer in model.layers])
# combine these loss functions into a single scalar
loss = K.variable(0.0)
layer_features = outputs_dict[content_layer]
############# Content extraction: #############
# retrieve content_image output for content_layer
content_image_features = layer_features[0, :, :, :]
# retrieve generated_image output from content_layer
generated_image_features = layer_features[2, :, :, :]
# get loss containing only content loss
loss = loss + content_weight * content_loss(content_image_features,
generated_image_features)
############# Style Extraction: #############
if len(style_weights) == 1:
style_weights = [style_weights[0]] * len(style_layers)
else:
assert len(style_weights) == len(style_layers)
style_weights = [float(style_weight) for style_weight in style_weights]
session = K.get_session()
for style_weight, layer_name in zip(style_weights, style_layers):
## get feature activations from layers
layer_features = outputs_dict[layer_name]
## retrieve style_image output activations for a style_layer
style_image_features = layer_features[1, :, :, :]
## retrieve generated_image output activations for a style_layer
generated_image_features = layer_features[2, :, :, :]
## get loss containing content loss and style loss
loss = loss + (style_weight / len(style_layers)) * style_loss(
style_image_features, generated_image_features, img_size, session)
## get loss containing content loss, style loss and total variation loss
loss = loss + total_variation_weight * total_variation_loss(
generated_image_placeholder, img_size)
# get the gradients of the generated image wrt. the loss
grads = K.gradients(loss, generated_image_placeholder)
# Define outputs list to have loss included
outputs = [loss]
# add the gradients to the outputs instance
if isinstance(grads, (list, tuple)):
outputs += grads
else:
outputs.append(grads)
## Define keras function with input the placeholder of the generated image and output the {loss and gradients} for learning
f_outputs = K.function(inputs=[generated_image_placeholder],
outputs=outputs)
class Evaluator(object):
def __init__(self):
self.loss_value = None
self.grads_values = None
def loss(self, x):
assert self.loss_value is None
loss_value, grad_values = eval_loss_and_grads(
x, img_size, f_outputs)
self.loss_value = loss_value
self.grad_values = grad_values
return self.loss_value
def grads(self, x):
assert self.loss_value is not None
grad_values = np.copy(self.grad_values)
self.loss_value = None
self.grad_values = None
return grad_values
# this Evaluator class makes it possible
# to compute loss and gradients in one pass
# while retrieving them via two separate functions,
# "loss" and "grads". This is done because scipy.optimize
# requires separate functions for loss and gradients,
# but computing them separately would be inefficient.
evaluator = Evaluator()
# run scipy-based optimization (L-BFGS) over the pixels of the generated image
# so as to minimize the neural style loss
loss_history = [None] * num_iter
for i in range(num_iter):
print("Start of iteration:", i + 1)
start_time = time.time()
generated_image, loss_history[i], info = fmin_l_bfgs_b(
evaluator.loss,
generated_image.flatten(),
fprime=evaluator.grads,
maxfun=20)
print("Current loss value:", loss_history[i])
# save current generated image
img = deprocess_image(generated_image.copy(), img_shape=img_size)
if rescale_image:
img = array_to_img(img[0])
img = img.resize(original_size)
img = img_to_array(img)
fname = output_subdir + "/generated_image_at_iteration_%s.png" % str(
i + 1)
save_img(path=fname, x=img)
end_time = time.time()
print("Image saved at:", fname)
print("Iteration %s completed in %ds" %
(str(i + 1), end_time - start_time))
# summarize history for loss
plt.figure(3, figsize=(7, 5))
plt.plot(loss_history)
plt.title("loss process during neural style transfer")
plt.ylabel("loss")
plt.xlabel("iteration")
plt.savefig(output_subdir + "/loss_history.jpg")
plt.close()
style_masks = mask_features[layer][STYLE, :, :, :]
target_masks = mask_features[layer][TARGET, :, :, :]
sl = style_loss(style_feat, target_feat, style_masks, target_masks)
loss += (style_weight / len(style_feature_layers)) * sl
loss += total_variation_weight * total_variation_loss(target_image)
loss_grads = K.gradients(loss, target_image)
# Evaluator class for computing efficiency
outputs = [loss]
if isinstance(loss_grads, (list, tuple)):
outputs += loss_grads
else:
outputs.append(loss_grads)
f_outputs = K.function([target_image], outputs)
def eval_loss_and_grads(x):
if K.image_data_format() == 'channels_first':
x = x.reshape((1, 3, img_nrows, img_ncols))
else:
x = x.reshape((1, img_nrows, img_ncols, 3))
outs = f_outputs([x])
loss_value = outs[0]
if len(outs[1:]) == 1:
grad_values = outs[1].flatten().astype('float64')
else:
grad_values = np.array(outs[1:]).flatten().astype('float64')
return loss_value, grad_values
# we will assume the weight of the content loss is 1
# and only weight the style losses
style_weights = [0.2, 0.4, 0.3, 0.5, 0.2]
# create the total loss which is the sum of content + style loss
loss = K.mean(K.square(content_model.output - content_target))
for w, symbolic, actual in zip(style_weights, symbolic_conv_outputs,
style_layers_outputs):
# gram_matrix() expects a (H, W, C) as input
loss += w * style_loss(symbolic[0], actual[0])
# once again, create the gradients and loss + grads function
# note: it doesn't matter which model's input you use
# they are both pointing to the same keras Input layer in memory
grads = K.gradients(loss, vgg.input)
# just like theano.function
get_loss_and_grads = K.function(inputs=[vgg.input], outputs=[loss] + grads)
def get_loss_and_grads_wrapper(x_vec):
l, g = get_loss_and_grads([x_vec.reshape(*batch_shape)])
return l.astype(np.float64), g.flatten().astype(np.float64)
final_img = minimize(get_loss_and_grads_wrapper, 10, batch_shape)
plt.imshow(scale_img(final_img))
plt.show()
