def kde_entropy_from_dists_loo(dists, N, dims, var):
# should have large values on diagonal
dists2 = dists / (2*var)
normconst = (dims/2.0)*K.log(2*np.pi*var)
lprobs = logsumexp(-dists2, axis=1) - np.log(N-1) - normconst
h = -K.mean(lprobs)
return nats2bits * h
def train(num_classes=100, epochs=100, reps=1):
(x_train, y_train) = load_data(num_classes,reps)
model = Sequential()
#model.add(GaussianNoise(0.95, input_shape=(784,)))
model.add(Lambda(lambda x: x+K.random_normal(shape=K.shape(x), mean=0., stddev=0.10), input_shape=(784,)))#permanent noise
#model.add(Dropout(0.95, input_shape=(784,))) #regular dropout
model.add(Lambda(lambda x: K.dropout(x, level=0.9), input_shape=(784,)))#permanent dropout
model.add(Dense(1024, activation='relu', input_shape=(784,)))
model.add(Dense(1024, activation='relu', input_shape=(784,)))
model.add(Dense(num_classes, activation='softmax'))
model.load_weights("saved_noise.hdf5")
model.summary()
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
#add callbacks
tensorboard = TensorBoard(log_dir="logs/{}".format(time()), histogram_freq=100, write_graph=True, write_images=False)
checkpoint = ModelCheckpoint("saved_noise.hdf5", monitor='val_acc', verbose=1, save_best_only=True, save_weights_only=False, mode='auto', period=10)
stopper = EarlyStopping(monitor='val_acc', min_delta=0.01, patience=100, verbose=1, mode='auto')
model.fit(x_train, y_train,
batch_size=num_classes*reps,
epochs=epochs,
verbose=1,
validation_data=(x_train,y_train),
callbacks=[tensorboard, checkpoint, stopper])
return model
def fbeta_score(y_true, y_pred, beta=1):
'''Calculates the F score, the weighted harmonic mean of precision and recall.
This is useful for multi-label classification, where input samples can be
classified as sets of labels. By only using accuracy (precision) a model
would achieve a perfect score by simply assigning every class to every
input. In order to avoid this, a metric should penalize incorrect class
assignments as well (recall). The F-beta score (ranged from 0.0 to 1.0)
computes this, as a weighted mean of the proportion of correct class
assignments vs. the proportion of incorrect class assignments.
With beta = 1, this is equivalent to a F-measure. With beta < 1, assigning
correct classes becomes more important, and with beta > 1 the metric is
instead weighted towards penalizing incorrect class assignments.
'''
if beta < 0:
raise ValueError('The lowest choosable beta is zero (only precision).')
# If there are no true positives, fix the F score at 0 like sklearn.
if K.sum(K.round(K.clip(y_true, 0, 1))) == 0:
return 0
p = precision(y_true, y_pred)
r = recall(y_true, y_pred)
bb = beta**2
fbeta_score = (1 + bb) * (p * r) / (bb * p + r + K.epsilon())
return fbeta_score
def __init__(self, restore, session=None):
self.num_channels = 1
self.image_size = 16
self.num_labels = 2
max_features = 20000
maxlen = 256 # cut texts after this number of words (among top max_features most common words)
K.set_learning_phase(True)
# model = Sequential()
# model.add(Reshape((256,), input_shape=(16, 16, 1)))
# model.add(Embedding(max_features, 50))
# model.add(LSTM(64, dropout=0.2, recurrent_dropout=0.2))
# model.add(Dense(2, activation='sigmoid'))
# try using different optimizers and different optimizer configs
# model.compile(loss='binary_crossentropy',
# optimizer='adam',
# metrics=['accuracy'])
model = Sequential()
model.add(Reshape((256,), input_shape=(16, 16, 1)))
model.add(Embedding(max_features, 128))
model.add(LSTM(128, dropout=0.2, recurrent_dropout=0.2))
model.add(Dense(2, activation='softmax'))
model.load_weights("models/imdb_model_new.h5")
self.model = model
def test_system(test_list: list):
K.set_learning_phase(False)
num_test_batches = int(len(test_list) / BATCH_SIZE)
predicted = []
result = []
error = []
for seed in range(num_test_batches):
batch = get_test_batch(seed, test_list)
predicted_result = SESSION.run(
predicted_player_result,
feed_dict={predicted_player_skills: batch["player_skills"]})
predicted_result = np.swapaxes(predicted_result, 0, 1)
for i in range(len(predicted_result)):
for player in range(len(predicted_result[i])):
predicted.append(predicted_result[i][player])
result.append(batch["player_results"][i][player])
error.append(batch["player_results"][i][player] -
predicted_result[i][player])
prediction = np.array(predicted)
results = np.array(result)
prediction_error = np.array(error)
if DEBUG > 2:
print("Error std: {}".format(np.std(prediction_error, 0)))
print("Original std: {}".format(np.std(results, 0)))
accuracy = np.mean(1 - np.var(prediction_error, 0) / np.var(results, 0))
K.set_learning_phase(True)
data = {
"predicted": prediction,
"error": prediction_error,
"result": results,
"accuracy": accuracy
}
return data
def on_epoch_begin(self, epoch, logs={}):
r = scipy.optimize.minimize(self.obj, K.get_value(self.noiselayer.logvar), jac=self.jac)
best_val = r.x[0]
cval = K.get_value(self.noiselayer.logvar)
max_var = 1.0 + cval
if best_val > max_var:
# don't raise it too fast, so that gradient information is preserved
best_val = max_var
K.set_value(self.noiselayer.logvar, best_val)
def on_epoch_begin(self, epoch, logs={}):
vals = self.nlayerinput(self.entropy_train_data)
dists = self.get_dists(vals)
dists += 10e20 * np.eye(dists.shape[0])
r = scipy.optimize.minimize(self.obj, K.get_value(self.kdelayer.logvar).flat[0],
jac=self.jac,
args=(dists,),
)
best_val = r.x.flat[0]
K.set_value(self.kdelayer.logvar, best_val)
def build(self, input_shape):
super(GaussianNoise2, self).build(input_shape)
K.set_value(self.logvar, self.init_logvar)
#K.set_value(self.alpha, self.init_alpha)
if self.is_trainable:
self.trainable_weights = [self.logvar,]
else:
self.trainable_weights = []
if self.mi_regularizer:
self.add_loss(self.mi_regularizer())
def predict(data_set=None, data_per_batch=32, epoch=1, model_path='model.h5'):
Dataset = None
if data_set == None:
dataloaders = os.listdir('Dataset')
for dataloader in dataloaders:
loader_path = os.path.join('Dataset', dataloader)
if dataloader.endswith('.py') and os.path.isfile(
loader_path) and dataloader != '__init__.py':
try:
Dataset = importlib.import_module("Dataset." +
dataloader[:-3])
except Exception as ex:
print('failed to load Dataset from "%s".' % dataloader, ex)
else:
print('successfuly loaded Dataset from "%s"!' % dataloader)
break
if Dataset == None:
raise Exception('No vaild dataset found!')
else:
try:
Dataset = importlib.import_module("Dataset." + data_set)
except Exception as ex:
raise Exception('"%s" is not a vaild dataset!' % data_set)
data_loader = Dataset.DataLoader(1024, 1, 13)
# 加载网络模型
model = NN.model.create_pridict_model()
# 输出网络结构
model.summary()
# 加载之前训练的数据
model.load_weights(model_path)
# 验证集
validation_data = data_loader.get_validation_generator()
data = next(validation_data)[0]
r = model.predict(data['speech_data_input'])
r = K.ctc_decode(r, data['input_length'][0])
r1 = K.get_value(r[0][0])
r1 = r1[0]
tokens = NN.model.get_tokens()
print('predict: [', end='')
for i in r1:
print(tokens[i], end=', ')
print(']')
print('truth : [', end='')
for i in range(data['label_length'][0][0]):
print(tokens[int(data['speech_labels'][0][i])], end=', ')
print(']')
pass
def __init__(self, init_logvar, kdelayer,
regularize_mi_input=None,
init_alpha=1.0,
get_noise_input_func=None,
trainable=True,
test_phase_noise=True,
*kargs, **kwargs):
self.supports_masking = True
self.init_logvar = init_logvar
#self.uses_learning_phase = True
self.kdelayer = kdelayer
self.get_noise_input_func = get_noise_input_func
if regularize_mi_input is not None:
self.mi_regularizer = MIRegularizer(MIComputer(get_noise_input_func(regularize_mi_input), kdelayer=kdelayer, noiselayer=self),
alpha=init_alpha)
else:
self.mi_regularizer = None
self.logvar = K.variable(0.0)
#self.init_alpha = init_alpha
#self.alpha = K.variable(0.0)
self.is_trainable = trainable
self.test_phase_noise = test_phase_noise
super(GaussianNoise2, self).__init__(*kargs, **kwargs)
def extractFeatures(X, model):
"""
Extract the features of X using the activation layer of the model
Inputs:
- X: data sample to extract features for
- model: model to use to get the features
Returns: the np array of features (output from the last layer of the model)
"""
# https://keras.io/getting-started/faq/#how-can-i-visualize-the-output-of-an-intermediate-layer
# https://github.com/fchollet/keras/issues/1641
# extract layer
get_last_layer_output = K.function(
[model.layers[0].input, K.learning_phase()], [model.layers[-4].output])
layer_output = get_last_layer_output([X, 0])[0]
return layer_output
def __init__(self, restore, session=None):
self.num_channels = 128
self.image_size = 16
self.num_labels = 2
max_features = 20000
maxlen = 256 # cut texts after this number of words (among top max_features most common words)
K.set_learning_phase(True)
print('Build model...')
model = Sequential()
model.add(Reshape((256, ), input_shape=(16, 16, 1)))
model.add(Embedding(max_features, 128))
model.add(LSTM(128))
model.add(Dense(2)) #, activation='softmax'))
model.load_weights(restore)
self.model = model
def recall(y_true, y_pred):
'''Calculates the recall, a metric for multi-label classification of
how many relevant items are selected.
'''
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
recall = true_positives / (possible_positives + K.epsilon())
return recall
def precision(y_true, y_pred):
'''Calculates the precision, a metric for multi-label classification of
how many selected items are relevant.
'''
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
precision = true_positives / (predicted_positives + K.epsilon())
return precision
def evaluate(self, batch_num):
batch_acc = 0
o_acc = 0
generator = self.val_seq.generator()
#关闭学习率
for i in range(batch_num):
inputs = next(generator)
x_test, y_test, source_str = (inputs["the_input"],
inputs["the_labels"],
inputs["source_str"])
out = self.validation_func([x_test, 0])[0]
current_acc = np.zeros([out.shape[0]])
c_acc = np.zeros([out.shape[0]])
#example one
# ctc_decode = K.ctc_decode(y_pred[:, 2:, :], input_length=np.ones(shape[0]) * shape[1])[0][0]
ctc_decode = K.get_value(
K.ctc_decode(out,
input_length=np.ones(out.shape[0]) * out.shape[1],
greedy=True)[0][0])
# print(ctc_decode)
for j in range(ctc_decode.shape[0]):
print("ctc_decode", ctc_decode[j], y_test[j][:4])
# out_best = list(np.argmax(decode_out[j, 2:], 1))
out_best = list(ctc_decode[j])
out_best = [k for k, g in itertools.groupby(out_best)]
if self.val_seq.equals_after_trim(y_test[j],
np.asarray(out_best)):
c_acc[j] = 1
print(source_str[j], y_test[j], out_best)
o_acc += c_acc.mean()
# print(" ctc_acc: %f%%" % (o_acc))
for j in range(out.shape[0]):
# 该层的输出结果是使用 max ,此处推断出最有可能的结果, 对每一列
out_best = list(np.argmax(out[j, 2:], 1))
out_best = [k for k, g in itertools.groupby(out_best)]
if self.val_seq.equals_after_trim(y_test[j],
np.asarray(out_best)):
current_acc[j] = 1
print(source_str[j], y_test[j], out_best)
batch_acc += current_acc.mean()
return batch_acc / batch_num, o_acc / batch_num
def matthews_correlation(y_true, y_pred):
'''Calculates the Matthews correlation coefficient measure for quality
of binary classification problems.
'''
y_pred_pos = K.round(K.clip(y_pred, 0, 1))
y_pred_neg = 1 - y_pred_pos
y_pos = K.round(K.clip(y_true, 0, 1))
y_neg = 1 - y_pos
tp = K.sum(y_pos * y_pred_pos)
tn = K.sum(y_neg * y_pred_neg)
fp = K.sum(y_neg * y_pred_pos)
fn = K.sum(y_pos * y_pred_neg)
numerator = (tp * tn - fp * fn)
denominator = K.sqrt((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn))
return numerator / (denominator + K.epsilon())
def ctc_lambda_func(args):
"""
Here the actual CTC loss calc occurs despite it not being an internal Keras loss function
:param args: [y_pred, labels, input_length, label_length]
:return: ctc_batch_cost
"""
y_pred, labels, input_length, label_length = args
# the 2 is critical here since the first couple outputs of the RNN
# tend to be garbage:
y_pred = y_pred[:, 2:, :]
return K.ctc_batch_cost(labels, y_pred, input_length, label_length)
def extractLearnedFeatures(model, newData):
"""
Using the previously trained model, extract a learned set of features for the new
data from the second to last layer of the model.
Inputs:
- model: the trained model
- newData: the new data to extract features from
Returns:
- learnedFeats: features extracted from the model
"""
# https://keras.io/getting-started/faq/#how-can-i-visualize-the-output-of-an-intermediate-layer
# https://github.com/fchollet/keras/issues/1641
# extract layer
get_last_layer_output = K.function(
[model.layers[0].input, K.learning_phase()], [model.layers[-4].output])
learnedFeats = get_last_layer_output([newData, 0])[0]
return learnedFeats
def w_categorical_crossentropy(y_true, y_pred, weights):
nb_cl = len(weights)
final_mask = K.zeros_like(y_pred[:,:, 0])
y_pred_max = K.max(y_pred, axis=-1)
y_pred_max = K.expand_dims(y_pred_max, axis=-1)
y_pred_max_mat = K.cast(K.equal(y_pred, y_pred_max), K.floatx())
for c_p, c_t in itertools.product(range(nb_cl), range(nb_cl)):
final_mask += (weights[c_t, c_p] * y_pred_max_mat[:,:, c_p] * y_true[:,:, c_t])
#return K.mean( K.categorical_crossentropy(y_pred, y_true) * final_mask )
return K.categorical_crossentropy(y_pred, y_true) * final_mask
def on_train_begin(self, logs={}):
modelobj = self.model.model
inputs = modelobj.inputs + modelobj.targets + modelobj.sample_weights + [ K.learning_phase(),]
lossfunc = K.function(inputs, [modelobj.total_loss])
jacfunc = K.function(inputs, K.gradients(modelobj.total_loss, self.noiselayer.logvar))
sampleweights = np.ones(len(self.traindata.X))
def obj(logvar):
v = K.get_value(self.noiselayer.logvar)
K.set_value(self.noiselayer.logvar, logvar.flat[0])
r = lossfunc([self.traindata.X, self.traindata.Y, sampleweights, 1])[0]
K.set_value(self.noiselayer.logvar, v)
return r
def jac(logvar):
v = K.get_value(self.noiselayer.logvar)
K.set_value(self.noiselayer.logvar, logvar.flat[0])
r = np.atleast_2d(np.array(jacfunc([self.traindata.X, self.traindata.Y, sampleweights, 1])))[0]
K.set_value(self.noiselayer.logvar, v)
return r
self.obj = obj # lambda logvar: lossfunc([self.traindata.X_train, self.traindata.Y_train, self.sampleweights, logvar[0], 1])[0]
self.jac = jac # lambda logvar: np.array(jacfunc([self.traindata.X_train, self.traindata.Y_train, self.sampleweights, logvar[0], 1]))
def kullback_leibler_divergence(y_true, y_pred):
'''Calculates the Kullback-Leibler (KL) divergence between prediction
and target values.
'''
y_true = K.clip(y_true, K.epsilon(), 1)
y_pred = K.clip(y_pred, K.epsilon(), 1)
return K.sum(y_true * K.log(y_true / y_pred), axis=-1)
def obj_det(image, yolo_model, anchors, class_names, sess, font, thickness,
bsc, input_image_shape, colors, output_size):
image_data = np.array(image, dtype='float32')
image_data /= 255.
image_data = np.expand_dims(image_data, 0) # Add batch dimension.
[boxes, scores, classes] = bsc
out_boxes, out_scores, out_classes = sess.run(
[boxes, scores, classes],
feed_dict={
yolo_model.input: image_data,
input_image_shape: [image.shape[1], image.shape[0]],
K.learning_phase(): 0
})
for i, c in reversed(list(enumerate(out_classes))):
# if (c != 0 and c != 1 and c != 2 and c != 3 and c != 5 and c != 7 and c!= 9 and c!= 11):
if (c != 1 and c != 5 and c != 6 and c != 13 and c != 14):
continue
predicted_class = class_names[c]
box = out_boxes[i]
score = out_scores[i]
label = '{} {:.2f}'.format(predicted_class, score)
imagex = Image.frombytes('RGB', (416, 416), image.tobytes())
image = imagex
draw = ImageDraw.Draw(image)
label_size = draw.textsize(label, font)
top, left, bottom, right = box
top = max(0, np.floor(top + 0.5).astype('int32'))
left = max(0, np.floor(left + 0.5).astype('int32'))
bottom = min(image.size[1], np.floor(bottom + 0.5).astype('int32'))
right = min(image.size[0], np.floor(right + 0.5).astype('int32'))
# print(label, (left, top), (right, bottom))
if top - label_size[1] >= 0:
text_origin = np.array([left, top - label_size[1]])
else:
text_origin = np.array([left, top + 1])
# My kingdom for a good redistributable image drawing library.
for i in range(thickness):
draw.rectangle([left + i, top + i, right - i, bottom - i],
outline=colors[c])
draw.rectangle([tuple(text_origin),
tuple(text_origin + label_size)],
fill=colors[c])
draw.text(text_origin, label, fill=(0, 0, 0), font=font)
del draw
return cv2.resize(np.asarray(image),
output_size,
interpolation=cv2.INTER_CUBIC)
def __init__(self):
# Define the neural network model
self.vocab = vocab
#START
vocab_size = len(vocab.x2i) + 1
main_input = Input(shape=(None, ), dtype='int32', name='main_input')
pret_input = Input(shape=(None, ), dtype='int32', name='pret_input')
x1 = Embedding(vocab_size, word_dim, input_length=None)(main_input)
x2 = Embedding(vocab_size,
word_dim,
weights=[prepare_pretrained_embs(vocab)],
input_length=None,
trainable=False)(pret_input)
x = keras.layers.Add()([x1, x2])
#END
l2r_lstm = LSTM(lstm_dim,
return_sequences=True,
return_state=True,
dropout=pdrop_lstm,
recurrent_dropout=pdrop_lstm)
r2l_lstm = LSTM(lstm_dim,
return_sequences=True,
return_state=True,
go_backwards=True,
dropout=pdrop_lstm,
recurrent_dropout=pdrop_lstm)
l2r_outs, r2l_outs = l2r_lstm(x), r2l_lstm(x)
l2r_last_state, r2l_last_state = l2r_outs[1], r2l_outs[1]
keys = keras.layers.concatenate([l2r_last_state, r2l_last_state])
vals = keras.layers.concatenate(
[l2r_outs[0], K.reverse(r2l_outs[0], 1)])
keys = keras.layers.Dense(lstm_dim * 2)(keys)
# Wrap the functions directly borrowed from tensorflow in this function.
def tf_funcs(keys):
from keras import backend as K
keys = K.tf.expand_dims(keys, 2)
attn = K.tf.nn.softmax(K.tf.matmul(vals, keys))
tweet_vec = K.tf.reduce_sum(K.tf.multiply(attn, vals), 1)
return tweet_vec
tweet_vec = keras.layers.Lambda(lambda inputs: tf_funcs(inputs))(keys)
tweet_vec = keras.layers.LeakyReLU()(tweet_vec)
score = keras.layers.Dense(1, activation='sigmoid')(tweet_vec)
self.model = Model(inputs=[main_input, pret_input], outputs=[score])
self.model.summary()
self.model.compile(optimizer=keras.optimizers.adam(decay=decay),
loss='binary_crossentropy',
metrics=["accuracy"])
def model_lstm_with_self_att(embed_dp, max_len):
hidden_states = embed_dp
hidden_states = Bidirectional(
LSTM(max_len, dropout=0.3, return_sequences=True,
return_state=False))(hidden_states)
# Attention mechanism
attention = Conv1D(filters=max_len,
kernel_size=1,
activation='tanh',
padding='same',
use_bias=True,
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
name="attention_layer1")(hidden_states)
attention = Conv1D(filters=max_len,
kernel_size=1,
activation='linear',
padding='same',
use_bias=True,
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
name="attention_layer2")(attention)
attention = Lambda(lambda x: softmax(x, axis=1),
name="attention_vector")(attention)
# Apply attention weights
weighted_sequence_embedding = Dot(axes=[1, 1],
normalize=False,
name="weighted_sequence_embedding")(
[attention, hidden_states])
# Add and normalize to obtain final sequence embedding
sequence_embedding = Lambda(lambda x: K.l2_normalize(K.sum(x, axis=1)))(
weighted_sequence_embedding)
return sequence_embedding
def plotCropImg(img,simg):
from keras.layers.core import K
model = Sequential()
#model.add(Lambda(lambda x: x, input_shape=(160,320,3)))
model.add(Lambda(lambda x: x / 127.5 - 1., input_shape=(160,320,3)))
model.add(Cropping2D(cropping=((50,25),(0,0))))
output = K.function([model.layers[0].input], [model.layers[1].output])
crop_img = output([img[None,...]])[0]
plt.figure()
plt.imshow(img, cmap='gray')
plt.savefig(simg+"_org.png")
plt.imshow(np.uint8(crop_img[0,...]), cmap='gray')
plt.savefig(simg+"_crop.png")
def on_train_begin(self, logs={}):
self.nlayerinput = lambda x: K.function([self.model.layers[0].input], [self.kdelayer.input])([x])[0]
N, dims = self.entropy_train_data.shape
Kdists = K.placeholder(ndim=2)
Klogvar = K.placeholder(ndim=0)
def obj(logvar, dists):
#print 'here', logvar # lossfunc([dists, logvar[0]])[0]
return lossfunc([dists, logvar.flat[0]])[0]
def jac(logvar, dists):
#print logvar, lossfunc([dists, logvar[0]]), jacfunc([dists, logvar[0]])
return np.atleast_2d(np.array(jacfunc([dists, logvar.flat[0]])))[0]
lossfunc = K.function([Kdists, Klogvar,], [kde_entropy_from_dists_loo(Kdists, N, dims, K.exp(Klogvar))])
jacfunc = K.function([Kdists, Klogvar,], K.gradients(kde_entropy_from_dists_loo(Kdists, N, dims, K.exp(Klogvar)), Klogvar))
self.obj =obj # lambda logvar, dists: np.array([lossfunc([dists, logvar[0]]),]) # [0]
self.jac =jac # lambda logvar, dists: jacfunc([dists, np.array([logvar]).flat[0]])[0]
def get_logs(model, data, kdelayer, noiselayer, max_entropy_calc_N=None):
logs = {}
modelobj = model.model
inputs = modelobj.inputs + modelobj.targets + modelobj.sample_weights + [ K.learning_phase(),]
lossfunc = K.function(inputs, [modelobj.total_loss])
sampleweightstrn = np.ones(len(data.train.X))
sampleweightstst = np.ones(len(data.test.X))
noreglosstrn = lambda: lossfunc([data.train.X, data.train.Y, sampleweightstrn, 0])[0]
noreglosstst = lambda: lossfunc([data.test.X , data.test.Y , sampleweightstst, 0])[0]
if kdelayer is not None:
lv1 = K.get_value(kdelayer.logvar)
logs['kdeLV'] = lv1
print 'kdeLV=%.5f,' % lv1,
if noiselayer is not None:
lv2 = K.get_value(noiselayer.logvar)
logs['noiseLV'] = lv2
print 'noiseLV=%.5f' % lv2
if kdelayer is not None and noiselayer is not None:
if max_entropy_calc_N is None:
mitrn = data.train.X
mitst = data.test.X
else:
mitrn = randsample(data.train.X, max_entropy_calc_N)
mitst = randsample(data.test.X, max_entropy_calc_N)
mi_obj_trn = MIComputer(noiselayer.get_noise_input_func(mitrn), kdelayer=kdelayer, noiselayer=noiselayer)
mi_obj_tst = MIComputer(noiselayer.get_noise_input_func(mitst), kdelayer=kdelayer, noiselayer=noiselayer)
if True:
mivals_trn = map(lambda x: float(K.eval(x)), [mi_obj_trn.get_mi(), mi_obj_trn.get_h(), mi_obj_trn.get_hcond()]) # [data.train.X,]))
logs['mi_trn'] = mivals_trn[0]
mivals_tst = map(lambda x: float(K.eval(x)), [mi_obj_tst.get_mi(), mi_obj_tst.get_h(), mi_obj_tst.get_hcond()]) # [data.train.X,]))
logs['mi_tst'] = mivals_tst[0]
logs['kl_trn'] = noreglosstrn()
logs['kl_tst'] = noreglosstst()
print ', mitrn=%s, mitst=%s, kltrn=%.3f, kltst=%.3f' % (mivals_trn, mivals_tst, logs['kl_trn'], logs['kl_tst'])
else:
print
return logs
#logs['tstloss'] = self.totalloss([self.xX_test,0])
def create_model():
tokens = get_tokens()
num_tokens = len(tokens) + 1
input_data = Input(name='speech_data_input', shape=(500, 13))
layer_dense_1 = Dense(256, activation="relu", use_bias=True, kernel_initializer='he_normal')(input_data)
layer_dropout_1 = Dropout(0.4)(layer_dense_1)
layer_dense_2 = Dense(512, activation="relu", use_bias=True, kernel_initializer='he_normal')(layer_dropout_1)
layer_gru1 = GRU(512, return_sequences=True, kernel_initializer='he_normal', dropout=0.4)(layer_dense_2)
layer_gru2 = GRU(512, return_sequences=True, go_backwards=True, kernel_initializer='he_normal', dropout=0.4)(layer_gru1)
layer_dense_3 = Dense(256, activation="relu", use_bias=True, kernel_initializer='he_normal')(layer_gru2)
layer_dropout_2 = Dropout(0.4)(layer_dense_3)
layer_dense_4 = Dense(num_tokens, activation="relu", use_bias=True, kernel_initializer='he_normal')(layer_dropout_2)
output = Activation('softmax', name='Activation0')(layer_dense_4)
#ctc
labels = Input(name='speech_labels', shape=[70], dtype='int64')
input_length = Input(name='input_length', shape=[1], dtype='int64')
label_length = Input(name='label_length', shape=[1], dtype='int64')
loss_out = Lambda(ctc_lambda, output_shape=(1,), name='ctc')([labels, output, input_length, label_length])
model = Model(inputs=[input_data, labels, input_length, label_length], outputs=loss_out)
adad = Adadelta(lr=0.01, rho=0.95, epsilon=K.epsilon())
model.compile(loss={'ctc': lambda y_true, output: output}, optimizer=adad)
print("model compiled successful!")
return model
from keras.preprocessing import image
from keras.applications.inception_v3 import InceptionV3, decode_predictions
from keras.layers.core import K
from keras import backend
import tensorflow as tf
from cleverhans.utils_keras import KerasModelWrapper
from cleverhans.attacks import FastGradientMethod
IMAGE_SIZE=299
TARGET_CLASS=849 # teapot
#TARGET_CLASS=1 # goldfish
IMAGE_PATH="img/goldfish.jpg"
K.set_learning_phase(0)
sess = tf.Session()
backend.set_session(sess)
def deprocess(input_image):
img = input_image.copy()
img /= 2.
img += 0.5
img *= 255. # [-1,1] -> [0,255]
img = image.array_to_img(img).copy()
return img
def preprocess(input_image):
img = image.img_to_array(input_image).copy()
img /= 255.
def generateTest():
words_to_generate = 30
testSeed = np.random.randint(low=1,
high=(len(data2) - words_to_read),
size=1)
testSeed = int(testSeed)
tx = []
for t in range(testSeed, testSeed + words_to_read, 1):
tx.append(int2word[data2[t]])
actualY = data2[testSeed + words_to_read]
actualY = one_hot(int(actualY))
actualY = np.reshape(actualY, (1, vocab_size))
# tx = ["came", "back", "from", "those", "strange", "streets","\n","any","day","of","those","lame","dreams","\n","move","with","the","same","speed","\n"]
print(tx)
for i in range(len(tx)):
try:
tx[i] = word2int[tx[i]]
except:
tx[i] = word2int['UNK']
# tx = [word2int[tx_i] for tx_i in tx]
tx = [embeddings[tx_i] for tx_i in tx]
for j in range(words_to_generate):
tx = np.reshape(tx, (1, words_to_read, embedding_size))
if (j == 0):
print(
"loss : ",
sess.run([loss],
feed_dict={
X: tx,
Y: actualY,
K.learning_phase(): 0
}))
predictedWord = model.predict(tx)
'''
predictedWord = closestEmbedding(predictedWord)
predictedWord2 = embeddings[predictedWord]
predictedWord2 = np.reshape(predictedWord2,[1,embedding_size])
'''
predictedWord = np.asarray(predictedWord).astype('float64')
predictedWord = np.reshape(predictedWord, (vocab_size))
# temperate
# temperature = 0.6
# predictedWord = np.log(predictedWord) / temperature
# pw = np.exp(predictedWord)
# predictedWord = pw / np.sum(pw)
# pm = np.random.multinomial(1, predictedWord, 1)
# pm = np.reshape(pm,(vocab_size))
pm = predictedWord # either this or temp
pw = np.exp(pm)
pm = pw / np.sum(pw)
predictedWord = np.argmax(pm)
#
# to not get UNK as prediction
if (predictedWord == 0):
predictedWord = np.argsort(pm)[-2]
#
predictedWord2 = one_hot(predictedWord)
predictedWord2 = np.reshape(predictedWord2, (1, vocab_size))
tx = np.reshape(tx, (words_to_read, embedding_size))
e = embeddings[predictedWord]
e = np.reshape(e, (1, embedding_size))
tx = np.append(tx, e, axis=0)
tx = tx[1:]
print(int2word[predictedWord], ' ', end='')
def on_epoch_end(self, epoch, logs={}):
lv1 = K.get_value(self.kdelayer.logvar)
lv2 = K.get_value(self.noiselayer.logvar)
logs['kdeLV'] = lv1
logs['noiseLV'] = lv2
print 'kdeLV=%.5f, noiseLV=%.5f' % (lv1, lv2)
def accuracy(y_true, y_predicted):
y = tf.argmax(y_true, axis=-1)
y_ = tf.argmax(y_predicted, axis=-1)
mask = tf.greater(y, 0)
return K.cast(K.equal(tf.boolean_mask(y, mask), tf.boolean_mask(y_, mask)),
K.floatx())
def __init__(self, micomputer, alpha):
super(MIRegularizer, self).__init__()
self.micomputer = micomputer
self.alpha = K.variable(alpha)
def get_noise(self, x):
#if not hasattr(self, 'saved_noise'):
# self.saved_noise = K.random_normal(shape=K.shape(x), mean=0., std=1)
return K.exp(0.5*self.logvar) * K.random_normal(shape=K.shape(x), mean=0., std=1)
def __init__(self, init_logvar):
self.init_logvar = init_logvar
self.logvar = K.variable(0.0)
super(KDEParamLayer, self).__init__()
def build(self, input_shape):
super(KDEParamLayer, self).build(input_shape)
K.set_value(self.logvar, self.init_logvar)
self.trainable_weights = []
def obj(logvar):
v = K.get_value(self.noiselayer.logvar)
K.set_value(self.noiselayer.logvar, logvar.flat[0])
r = lossfunc([self.traindata.X, self.traindata.Y, sampleweights, 1])[0]
K.set_value(self.noiselayer.logvar, v)
return r
def get_activations(model, layer, X_batch):
get_activations = K.function([model.layers[0].input, K.learning_phase()], [model.layers[layer].output,])
activations = get_activations([X_batch,0])
return activations
def jac(logvar):
v = K.get_value(self.noiselayer.logvar)
K.set_value(self.noiselayer.logvar, logvar.flat[0])
r = np.atleast_2d(np.array(jacfunc([self.traindata.X, self.traindata.Y, sampleweights, 1])))[0]
K.set_value(self.noiselayer.logvar, v)
return r
def kde_entropy(output, var):
dims = K.cast(K.shape(output)[1], K.floatx() ) #int(K.shape(output)[1])
N = K.cast(K.shape(output)[0], K.floatx() )
normconst = (dims/2.0)*K.log(2*np.pi*var)
# Kernel density estimation of entropy
# get dists matrix
x2 = K.expand_dims(K.sum(K.square(output), axis=1), 1)
#x2 = x2 + K.transpose(x2)
#return K.shape(x2)
dists = x2 + K.transpose(x2) - 2*K.dot(output, K.transpose(output))
dists = dists / (2*var)
#y1 = K.expand_dims(output, 0)
#y2 = K.expand_dims(output, 1)
#dists = K.sum(K.square(y1-y2), axis=2) / (2*var)
normCount = N
## Removes effect of diagonals, i.e. leave-one-out entropy
#normCount = N-1
#diagvals = get_diag(10e20*K.ones_like(dists[0,:]))
#dists = dists + diagvals
lprobs = logsumexp(-dists, axis=1) - K.log(normCount) - normconst
h = -K.mean(lprobs)
return nats2bits * h # , normconst + (dims/2.0)
def get_hcond(self):
return kde_condentropy(self.input, K.exp(self.noiselayer.logvar))
def get_h(self):
totalvar = K.exp(self.noiselayer.logvar)+K.exp(self.kdelayer.logvar)
return kde_entropy(self.input, totalvar)
def build_and_train(hype_space, save_best_weights=True, log_for_tensorboard=False):
"""Build the model and train it."""
K.set_learning_phase(1)
# if log_for_tensorboard:
# # We need a smaller batch size to not blow memory with tensorboard
# hype_space["lr_rate_mult"] = hype_space["lr_rate_mult"] / 10.0
# hype_space["batch_size"] = hype_space["batch_size"] / 10.0
model = build_model(hype_space)
# K.set_learning_phase(1)
time_str = datetime.now().strftime("%Y_%m_%d-%H_%M")
model_weight_name = MODEL_NAME + "-" + time_str
callbacks = []
# Weight saving callback:
if save_best_weights:
weights_save_path = os.path.join(
WEIGHTS_DIR, '{}.hdf5'.format(model_weight_name))
print("Model's weights will be saved to: {}".format(weights_save_path))
if not os.path.exists(WEIGHTS_DIR):
os.makedirs(WEIGHTS_DIR)
callbacks.append(keras.callbacks.ModelCheckpoint(
weights_save_path,
monitor='val_accuracy',
save_best_only=True, mode='max'))
callbacks.append(keras.callbacks.EarlyStopping(
monitor='val_accuracy',
patience=10, verbose=1, mode='max'))
# TensorBoard logging callback:
log_path = None
if log_for_tensorboard:
log_path = os.path.join(TENSORBOARD_DIR, model_weight_name)
print("Tensorboard log files will be saved to: {}".format(log_path))
if not os.path.exists(log_path):
os.makedirs(log_path)
# Right now Keras's TensorBoard callback and TensorBoard itself are not
# properly documented so we do not save embeddings (e.g.: for T-SNE).
# embeddings_metadata = {
# # Dense layers only:
# l.name: "../10000_test_classes_labels_on_1_row_in_plain_text.tsv"
# for l in model.layers if 'dense' in l.name.lower()
# }
tb_callback = keras.callbacks.TensorBoard(
log_dir=log_path,
histogram_freq=2,
# write_images=True, # Enabling this line would require more than 5 GB at each `histogram_freq` epoch.
write_graph=True
# embeddings_freq=3,
# embeddings_layer_names=list(embeddings_metadata.keys()),
# embeddings_metadata=embeddings_metadata
)
tb_callback.set_model(model)
callbacks.append(tb_callback)
# Train net:
history = model.fit(
X_train_aug,
y_train,
batch_size=int(hype_space['batch_size']),
epochs=EPOCHS,
shuffle=True,
verbose=2,
callbacks=callbacks,
validation_data=(X_val_aug, y_val)
).history
# Test net:
K.set_learning_phase(0)
score = evaluate_model(model, weights_save_path)
print("\n\n")
max_acc = max(history['val_accuracy'])
model_name = MODEL_NAME+"_{}_{}".format(str(max_acc), time_str)
print("Model name: {}".format(model_name))
# Note: to restore the model, you'll need to have a keras callback to
# save the best weights and not the final weights. Only the result is
# saved.
print(history.keys())
print(history)
print('Score: ', score)
result = {
# We plug "-val_accuracy" as a
# minimizing metric named 'loss' by Hyperopt.
'loss': -max_acc,
# Misc:
'model_name': model_name,
'space': hype_space,
'status': STATUS_OK
}
print("RESULT:")
print_json(result)
f = open("/nosave/lange/cu-ssp/model_neu/optimized/logs/test_results_mod6.txt", "a+")
res = ""
for k, v in score.items():
res += str(k)+": "+str(v)+"\t"
f.write("\n"+str(model_weight_name)+"\t"+ res)
f.close()
return model, model_name, result, log_path
def weighted_accuracy(y_true, y_pred):
return K.sum(
K.equal(K.argmax(y_true, axis=-1), K.argmax(y_pred, axis=-1)) *
K.sum(y_true, axis=-1)) / K.sum(y_true)
def train(epoch_num=None, name=MODEL_NAME):
input_tensor = Input(name='the_input',
shape=(width, height, 3),
dtype='float32') #Input((width, height, 1))
x = input_tensor
for i in range(2):
# x = Conv2D(filters=2 ** (3+i), kernel_size=(3, 3), padding="same", activation='relu', kernel_initializer='he_normal')(x)
x = Conv2D(filters=16 * (i + 1),
kernel_size=(3, 3),
padding="same",
activation='relu',
kernel_initializer='he_normal')(x)
x = Conv2D(filters=16 * (i + 1),
kernel_size=(3, 3),
padding="same",
activation='relu',
kernel_initializer='he_normal')(x)
# x = Conv2D(filters=32, kernel_size=(3, 3), activation='relu')(x)
x = MaxPool2D(pool_size=(2, 2))(x)
conv_shape = x.get_shape()
# conv_to_rnn_dims = (width // (2 ** 3),
# (height // (2 ** 3)) * 32)
x = Reshape(target_shape=(int(conv_shape[1]),
int(conv_shape[2] * conv_shape[3])))(x)
x = Dense(dense_size, activation='relu')(x)
# (batch_size, 20, 8 )
gru_1 = GRU(rnn_size,
return_sequences=True,
kernel_initializer='he_normal',
name='gru1')(x)
gru_1b = GRU(rnn_size,
return_sequences=True,
go_backwards=True,
kernel_initializer='he_normal',
name='gru1_b')(x)
gru1_merged = Add()([gru_1, gru_1b]) #sum
gru_2 = GRU(rnn_size,
return_sequences=True,
kernel_initializer='he_normal',
name='gru2')(gru1_merged)
gru_2b = GRU(rnn_size,
return_sequences=True,
go_backwards=True,
kernel_initializer='he_normal',
name='gru2_b')(gru1_merged)
gru_2 = TimeDistributed(BatchNormalization())(gru_2)
gru_2b = TimeDistributed(BatchNormalization())(gru_2b)
x = Concatenate()([gru_2, gru_2b]) #concat
# x = Dropout(0.25)(x)
"""
最后结果是[batch_size, 最大时间序列, 分类总数+1位空白符+1位CTC校验位]，使用softmax函数，将所有结果的概率分布在（0，1）之间，激活用在每一帧时间序列上，求最大概率的分类，得出该帧的预测结果。
因此，此处dense层设置 分类总数的数量为结果，并采用softmax多分类激活函数
"""
x = Dense(n_class, kernel_initializer='he_normal', activation='softmax')(x)
# Model(inputs=input_tensor, outputs=x).summary()
# base_model = Model(inputs=input_tensor, outputs=x)
# 评估回调函数
evaluator_func = K.function([input_tensor, K.learning_phase()], [x])
# evaluator_func.
# base_model.summary()
evaluator = Evaluate(validation_func=evaluator_func,
val_seq=val_obj,
name="keras_cnn_gru_add_batch")
labels = Input(name='the_labels', shape=[n_len], dtype='float32')
input_length = Input(name='input_length', shape=[1], dtype='int64')
label_length = Input(name='label_length', shape=[1], dtype='int64')
loss_out = Lambda(ctc_lambda_func, output_shape=(1, ),
name='ctc')([x, labels, input_length, label_length])
model = Model(inputs=[input_tensor, labels, input_length, label_length],
outputs=[loss_out]) #.summary()
model.summary()
model.compile(loss={
'ctc': lambda y_true, y_pred: y_pred
},
optimizer='adadelta')
if epoch_num is not None:
weight_file = os.path.join(
OUTPUT_DIR, os.path.join(name, 'epoch_%02d.h5' % (epoch_num)))
model.load_weights(weight_file)
# print(base_model == model)
# model.fit_generator(train_gen.gen_batch(n_len, 200), steps_per_epoch=100, epochs=100, max_queue_size=1, workers=1, callbacks=[evaluator])
# model.fit_generator(image_gen.next_val(), steps_per_epoch=1, epochs=100, max_queue_size=1, workers=1, callbacks=[evaluator]) #单线程,易调试
model.fit_generator(image_gen,
steps_per_epoch=200,
epochs=100,
callbacks=[evaluator],
use_multiprocessing=True,
workers=2) #多线程
def call(self, x, mask=None):
if self.test_phase_noise:
return x+self.get_noise(x)
else:
return K.in_train_phase(x+self.get_noise(x), x)
def ctc_lambda_func(args):
y_pred, labels, input_length, label_length = args
y_pred = y_pred[:, 2:, :]
return K.ctc_batch_cost(labels, y_pred, input_length, label_length)
def get_mi_loss(self):
if not hasattr(self, 'layer'):
raise Exception('Need to call `set_layer` on ActivityRegularizer instance before calling the instance.')
return K.in_train_phase(self.alpha * self.micomputer.get_mi(), 0)
def logsumexp(mx, axis):
cmax = K.max(mx, axis=axis)
cmax2 = K.expand_dims(cmax, 1)
mx2 = mx - cmax2
return cmax + K.log(K.sum(K.exp(mx2), axis=1))
def nll(y_true, y_pred):
""" Negative log likelihood. """
# keras.losses.binary_crossentropy give the mean
# over the last axis. we require the sum
return K.sum(K.binary_crossentropy(y_true, y_pred), axis=-1)
# print "X shape : ",X.get_shape()
# i+=1
x = np.reshape(x, (words_to_read, embedding_size))
# print x.shape
return x, y
# nX = len(X)
# print "DATA VECTORIZED............."
# X = np.reshape(X,(nX,words_to_read,1))
# print X.get_shape(),Y.get_shape()
sess = tf.Session()
K.set_session(sess)
model = Sequential()
model.add(
LSTM(256,
input_shape=(int(X.get_shape()[1]), int(X.get_shape()[2])),
return_sequences=True))
model.add(Dropout(0.2))
model.add(LSTM(256, return_sequences=False))
model.add(Dropout(0.2))
model.add(Dense(vocab_size)) #, activation="softmax"))
# model.compile(loss="categorical_crossentropy",optimizer="adam")
# model = LSTM(256,input_shape=(X.get_shape()[1],X.get_shape()[2]),init='uniform',return_sequences=True)(X)
# model = Dropout(0.2)(model)
# model = LSTM(256)(model)
from bottle import route, run, static_file, request, BaseRequest
import base64
import re
import numpy as np
import tensorflow as tf
import cv2
from keras.layers.core import K
K.set_learning_phase(0)
import time
import random
import os
import datetime
from keras.models import Model,load_model
from keras.layers import Input, Conv2D, MaxPooling2D
import threading
seed = random.randint(0, 2**31 - 1)
tf.set_random_seed(seed)
np.random.seed(seed)
random.seed(seed)
BaseRequest.MEMFILE_MAX = 10000 * 1000
import chainer
import chainer.links as L
import chainer.functions as F
class GoogLeNet(chainer.Chain):
def __init__(self):
super(GoogLeNet, self).__init__(
conv1=L.Convolution2D(3, 64, 7, stride=2, pad=3, nobias=True),
norm1=L.BatchNormalization(64),
conv2=L.Convolution2D(64, 192, 3, pad=1, nobias=True),
def kde_condentropy(output, var):
dims = K.cast(K.shape(output)[1], K.floatx() ) # int(output.get_shape()[1])
# #normconst = (dims/2.0)*K.log(2*np.pi*var)
# #return normconst + (dims/2.0)
normconst = (dims/2.0)*K.log(2*np.pi*var)
return nats2bits * normconst
