return trainUser
trainUser = convertUsersToEmbeddings(trainUserMentions, embedding, idMapping)
bla = trainUser.sum(axis=1)
#trainUser = trainUser[bla != 0,]
#classes = classes[bla != 0,]
userGraphBranchI = Input(shape=(trainUser.shape[1], ), name="inputUserGraph")
userGraphBranch = BatchNormalization()(userGraphBranchI)
userGraphBranch = Dropout(0.2, name="domainName")(userGraphBranch)
userGraphBranchO = Dense(max(classes) + 1,
activation='softmax')(userGraphBranch)
userGraphModel = Model(inputs=userGraphBranchI, outputs=userGraphBranchO)
userGraphModel.compile(loss='sparse_categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
start = time.time()
sourceHistory = userGraphModel.fit(trainUser,
classes,
epochs=nb_epoch,
batch_size=batch_size,
verbose=verbosity)
print("tldBranch finished after " +
str(datetime.timedelta(seconds=round(time.time() - start))))
userGraphModel.save(modelPath + 'userGraphModel.h5')
model = Model(inputs=entree, outputs=out)
model.summary()
# print(y_predict)
dict_labels_revert = {v: k for k, v in dict_labels.items()}
dict_words_revert = {v: k for k, v in dict_words.items()}
# print(predictions_list)
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=[tf.metrics.SparseCategoricalAccuracy()])
model.fit(X_train, y_train, batch_size=128, epochs=60)
# score = model.evaluate(X_test, y_test)
# print(score)
y_predict = model.predict(X_test)
predictions_list = []
for line, origin_labels in zip(y_predict, y_test):
line_list = []
for label_num, true_label in zip(line, origin_labels):
line_list.append((dict_labels_revert[np.argmax(label_num)],
dict_labels_revert[true_label[0]]))
predictions_list.append(line_list)
sequence_input = Input(shape=(max_words,), dtype='int32')
embedded_sequences = embedding_layer(sequence_input)
lstm = Bidirectional(LSTM(units=400, dropout=0.5))(embedded_sequences)
# dropout = Dropout(0.5)(lstm)
dense = Dense(2, activation="softmax")(lstm)
model = Model(inputs=[sequence_input],
outputs=[dense])
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
# class_weights = class_weight.compute_class_weight('balanced', np.unique(np.argmax(y_train_data, axis=1)), np.argmax(y_train_data, axis=1))
model.fit(x_train_data, y_train_data,
epochs=100,
batch_size=16,
# class_weight=class_weights,
verbose=1)
preds = model.predict(x_test_data)
preds = np.argmax(preds, axis=1)
y_test = np.argmax(y_test_data, axis=1)
print(classification_report(y_test, preds, digits=3))
print(metrics.precision_score(y_test, preds, average='macro'))
print(metrics.recall_score(y_test, preds, average='macro'))
print(metrics.f1_score(y_test, preds, average='macro'))
output = Dense(1, activation='sigmoid')(x)
MLP = Model(input, output)
# sgd = optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.6, nesterov=True)
adam = optimizers.Adam(lr=0.01,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.001,
amsgrad=False)
MLP.compile(optimizer=adam,
loss='binary_crossentropy',
metrics=['binary_accuracy'])
split = int(X_train.shape[0] * 0.9)
history = MLP.fit(X_train.values,
Y_train.values,
batch_size=64,
epochs=2000,
validation_split=0.2)
predictions = MLP.predict(X_test.values)
# summarize history for accuracy
plt.figure()
plt.plot(history.history['binary_accuracy'])
plt.plot(history.history['val_binary_accuracy'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'val'], loc='upper left')
plt.ion()
plt.show()
plt.pause(0.001)
plt.figure()
def test_supervised(self):
observations = lib.load_lending_club()
# Train /test split
train_observations, test_observations = train_test_split(observations)
train_observations = train_observations.copy()
test_observations = test_observations.copy()
# Supervised
data_type_dict = {
'numerical': [
'loan_amnt', 'annual_inc', 'open_acc', 'dti', 'delinq_2yrs',
'inq_last_6mths', 'mths_since_last_delinq', 'pub_rec',
'revol_bal', 'revol_util', 'total_acc', 'pub_rec_bankruptcies'
],
'categorical': [
'term', 'grade', 'emp_length', 'home_ownership', 'loan_status',
'addr_state', 'application_type'
],
'text': ['desc', 'purpose', 'title']
}
output_var = 'loan_status'
auto = Automater(data_type_dict=data_type_dict, output_var=output_var)
self.assertTrue(auto.supervised)
expected_input_vars = reduce(lambda x, y: x + y,
data_type_dict.values())
expected_input_vars.remove(output_var)
self.assertCountEqual(expected_input_vars, auto.input_vars)
self.assertEqual(output_var, auto.output_var)
self.assertTrue(isinstance(auto.input_mapper, DataFrameMapper))
self.assertTrue(isinstance(auto.output_mapper, DataFrameMapper))
self.assertFalse(auto.fitted)
self.assertRaises(AssertionError, auto._check_fitted)
# Test fit
auto.fit(train_observations)
self.assertTrue(auto.fitted)
self.assertIsNotNone(auto.input_mapper.built_features)
self.assertTrue(isinstance(auto.input_layers, list))
self.assertEqual(len(expected_input_vars), len(auto.input_layers))
self.assertIsNotNone(auto.input_nub)
self.assertIsNotNone(auto.output_nub)
self.assertIsNotNone(auto.output_mapper.built_features)
# Test transform, df_out=False
train_X, train_y = auto.transform(train_observations)
test_X, test_y = auto.transform(test_observations)
self.assertTrue(isinstance(test_X, list))
self.assertTrue(isinstance(test_y, numpy.ndarray))
self.assertEqual(test_observations.shape[0],
test_X[0].shape[0]) # Correct number of rows back
self.assertEqual(test_observations.shape[0],
test_y.shape[0]) # Correct number of rows back
# Test transform, df_out=True
transformed_observations = auto.transform(test_observations,
df_out=True)
self.assertTrue(isinstance(transformed_observations, pandas.DataFrame))
self.assertEqual(
test_observations.shape[0],
transformed_observations.shape[0]) # Correct number of rows back
# Test suggest_loss
suggested_loss = auto.suggest_loss()
self.assertTrue(callable(suggested_loss))
# Test model building
x = auto.input_nub
x = Dense(32)(x)
x = auto.output_nub(x)
model = Model(inputs=auto.input_layers, outputs=x)
model.compile(optimizer='Adam', loss=auto.suggest_loss())
model.fit(train_X, train_y)
pred_y = model.predict(test_X)
# Test inverse_transform_output
inv_transformed_pred_y = auto.inverse_transform_output(pred_y)
self.assertEqual(test_observations.shape[0],
inv_transformed_pred_y.shape[0])
#mask_zero=True
)(descriptionBranchI)
descriptionBranch = SpatialDropout1D(rate=0.2)(descriptionBranch) #Masks the same embedding element for all tokens
descriptionBranch = BatchNormalization()(descriptionBranch)
descriptionBranch = Dropout(0.2)(descriptionBranch)
descriptionBranch = Conv1D(filters,kernel_size, padding='valid',activation='relu', strides=1)(descriptionBranch)
descriptionBranch = GlobalMaxPooling1D()(descriptionBranch)
descriptionBranch = BatchNormalization()(descriptionBranch)
descriptionBranch = Dropout(0.2, name="description")(descriptionBranch)
descriptionBranchO = Dense(len(set(classes)), activation='softmax')(descriptionBranch)
descriptionModel = Model(inputs=descriptionBranchI, outputs=descriptionBranchO)
descriptionModel.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
start = time.time()
descriptionHistory = descriptionModel.fit(trainDescription, classes,
epochs=nb_epoch, batch_size=batch_size,
verbose=verbosity, validation_split=validation_split,callbacks=callbacks
)
print("descriptionBranch finished after " +str(datetime.timedelta(seconds=round(time.time() - start))))
descriptionModel.save(modelPath +'descriptionBranchNorm.h5')
#####################
#2a.) Link Model for Domain
categorial = np.zeros((len(trainDomain), len(domainEncoder.classes_)), dtype="bool")
for i in range(len(trainDomain)):
categorial[i, trainDomain[i]] = True
trainDomain = categorial
domainBranchI = Input(shape=(trainDomain.shape[1],), name="inputDomain")
input_img = Input(shape=(784, )) # encoder layers encoded = Dense(128, activation='relu')(input_img) encoded = Dense(64, activation='relu')(encoded) encoded = Dense(10, activation='relu')(encoded) encoded_output = Dense(encoding_dim)(encoded) # 注意这里没有激活函数 # decoder layers decoded = Dense(10, activation='relu')(encoded_output) decoded = Dense(64, activation='relu')(decoded) decoded = Dense(128, activation='relu')(decoded) decoded = Dense(784, activation='tanh')(decoded) # 注意这里使用tanh # construct the autoencoder model autoencoder = Model(input=input_img, output=decoded) # construct the encoder encoder = Model(input=input_img, output=encoded_output) # compile autoencoder autoencoder.compile(optimizer='adam', loss='mse') # training autoencoder.fit(X_train, X_train, epochs=20, batch_size=256, shuffle=True) # plotting encoded_img = encoder.predict(X_test) plt.scatter(encoded_img[:, 0], encoded_img[:, 1], c=y_test) plt.colorbar() plt.show()
class CategoryClassifier:
def __init__(self):
pass
def training(self, input_data, output_data, val_input, val_output):
print("Get category number")
self._get_category_num(output_data)
print("Build tokenizer")
self._build_tokenizer(input_data)
print("Convert data")
input_data, output_data = self._convert_data(input_data, output_data)
print(input_data)
print(output_data)
print("Build network")
self._build_network(input_data, output_data)
print("Training")
self._training(input_data, output_data, val_input, val_output)
def validation(self, data):
pass
def prediction(self, data):
pass
def save(self, path):
pass
def load(self, path):
pass
def _build_tokenizer(self, input_data):
self._tokenizer = Tokenizer(char_level=True)
self._tokenizer.fit_on_texts(input_data)
def _build_network(self, input_data, output_data):
print(input_data[0])
print(len(input_data[0]))
print(output_data[0])
print(len(output_data[0]))
emb_size = 256
num_tokens = len(self._tokenizer.word_index) + 1
_input = Input(shape=(len(input_data[0]), ))
x = Embedding(num_tokens, emb_size, trainable=True)(_input)
x = Dropout(0.3)(x)
x = Bidirectional(GRU(128, return_sequences=True))(x)
x = Conv1D(256, 3, padding='valid', activation='relu', strides=1)(x)
x = MaxPooling1D(pool_size=3)(x)
x = Flatten()(x)
output = Dense(len(output_data[0]),
activation='softmax',
name="output_dense")(x)
self._model = Model(inputs=_input, outputs=[output])
self._model.summary()
def _get_category_num(self, output_data):
self._num_of_category = len(set(output_data))
def _training(self, input_data, output_data, val_input, val_output):
model_save_path = './model/'
if not os.path.exists(model_save_path):
os.mkdir(model_save_path)
model_path = model_save_path + '{epoch:02d}-{val_loss:.4f}.hdf5'
cb_checkpoint = ModelCheckpoint(filepath=model_path,
monitor='val_loss',
verbose=1,
save_best_only=True)
self._model.compile(optimizer="adam",
loss="categorical_crossentropy",
metrics=[metrics.categorical_accuracy])
self._model.fit(input_data,
output_data,
validation_split=0.2,
epochs=100,
batch_size=256,
callbacks=[cb_checkpoint])
def _convert_data(self, input_data, output_data):
input_data = self._tokenizer.texts_to_sequences(input_data)
maxlen = 0
for data in input_data:
if len(data) > maxlen:
maxlen = len(data)
return pad_sequences(input_data,
maxlen=maxlen), to_categorical(output_data)
output = Dense(RELATION_COUNT, activation="softmax")(output)
model = Model(inputs=[index_input, pos1_input, pos2_input],
outputs=[output])
# model = multi_gpu_model(model, gpus=4)
optimizer = Adam()
model.summary()
model.compile(optimizer=optimizer,
loss="categorical_crossentropy",
metrics=["accuracy"])
model.fit(x=[train_index, train_relative_e1_pos, train_relative_e2_pos],
y=[train_labels],
validation_data=([
test_index, test_relative_e1_pos, test_relative_e2_pos
], [test_labels]),
batch_size=50,
epochs=100,
callbacks=[
f1_calculator(test_index, test_relative_e1_pos,
test_relative_e2_pos, test_labels),
ModelCheckpoint("simple_cnn.model", "f1_score", 0, True,
False, "max"),
EarlyStopping("f1_score", 0.000001, 20, 0, "max")
])
def dnn():
import numpy as np
import pandas as pd
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.pyplot as plt
dlt_df = pd.read_csv("大乐透.csv")
data = dlt_df.values
data = data[1:, 2:9]
data = data.astype(np.float64)
# 使用神经网络预测
mean = data[:1500].mean(axis=0)
std = data[:1500].std(axis=0)
data1 = data.copy()
data1 -= mean
data1 /= std
train_data = data1[:1400]
train_data = np.expand_dims(train_data, axis=1)
val_data = data1[1400:1550]
val_data = np.expand_dims(val_data, axis=1)
test_data = data1[1550:len(data) - 1]
test_data = np.expand_dims(test_data, axis=1)
red1_labels = data[:, 0]
red2_labels = data[:, 1]
red3_labels = data[:, 2]
red4_labels = data[:, 3]
red5_labels = data[:, 4]
blue1_labels = data[:, 5]
blue2_labels = data[:, 6]
train_labels_1 = red1_labels[1:1401]
train_labels_2 = red2_labels[1:1401]
train_labels_3 = red3_labels[1:1401]
train_labels_4 = red4_labels[1:1401]
train_labels_5 = red5_labels[1:1401]
train_labels_6 = blue1_labels[1:1401]
train_labels_7 = blue2_labels[1:1401]
val_labels_1 = red1_labels[1401:1551]
val_labels_2 = red2_labels[1401:1551]
val_labels_3 = red3_labels[1401:1551]
val_labels_4 = red4_labels[1401:1551]
val_labels_5 = red5_labels[1401:1551]
val_labels_6 = blue1_labels[1401:1551]
val_labels_7 = blue2_labels[1401:1551]
test_labels_1 = red1_labels[1551:]
test_labels_2 = red2_labels[1551:]
test_labels_3 = red3_labels[1551:]
test_labels_4 = red4_labels[1551:]
test_labels_5 = red5_labels[1551:]
test_labels_6 = blue1_labels[1551:]
test_labels_7 = blue2_labels[1551:]
from keras import layers
from keras import Model
from keras import Input
from keras.optimizers import RMSprop
post_input = Input(shape=(None, 7), name='post_input')
lstm = layers.LSTM(150,
dropout=0.2,
recurrent_dropout=0.2,
activation='relu',
return_sequences=True)(post_input)
lstm1 = layers.LSTM(250,
dropout=0.2,
recurrent_dropout=0.2,
activation='relu')(lstm)
x = layers.Dense(360, activation='relu')(lstm1)
x = layers.Dense(250, activation='relu')(x)
x = layers.Dense(250, activation='relu')(x)
x = layers.Dense(250, activation='relu')(x)
x = layers.Dense(250, activation='relu')(x)
x = layers.Dense(250, activation='relu')(x)
x = layers.Dense(140, activation='relu')(x)
x = layers.Dense(70, activation='relu')(x)
# x=layers.Dropout(0.3)(x)
red1_predict = layers.Dense(1, name='red1')(x)
red2_predict = layers.Dense(1, name='red2')(x)
red3_predict = layers.Dense(1, name='red3')(x)
red4_predict = layers.Dense(1, name='red4')(x)
red5_predict = layers.Dense(1, name='red5')(x)
blue1_predict = layers.Dense(1, name='blue1')(x)
blue2_predict = layers.Dense(1, name='blue2')(x)
model = Model(post_input, [
red1_predict, red2_predict, red3_predict, red4_predict, red5_predict,
blue1_predict, blue2_predict
])
model.compile(optimizer=RMSprop(1e-4),
loss=['mse', 'mse', 'mse', 'mse', 'mse', 'mse', 'mse'],
metrics=['acc', 'acc', 'acc', 'acc', 'acc', 'acc', 'acc'])
history = model.fit(train_data, [
train_labels_1, train_labels_2, train_labels_3, train_labels_4,
train_labels_5, train_labels_6, train_labels_7
],
batch_size=20,
epochs=50,
validation_data=(val_data, [
val_labels_1, val_labels_2, val_labels_3,
val_labels_4, val_labels_5, val_labels_6,
val_labels_7
]))
loss = history.history['loss']
loss = loss[3:]
val_loss = history.history['val_loss']
val_loss = val_loss[3:]
epochs = range(1, len(loss) + 1)
plt.figure()
plt.plot(epochs, loss, 'b', color='r', label='Training loss')
plt.plot(epochs, val_loss, 'b', label='Validation loss')
plt.title('Training and validation loss')
plt.legend()
plt.show() # 损失图像
# Keras如何保存和载入训练好的模型和参数:https://blog.csdn.net/qq_36142114/article/details/86929132
model.save(filepath="dlt_model.h5", include_optimizer=False)
if N==8:
A = np.array([ False, False, False, True, False, True, True, True])
# Logical vector indicating the nonfrozen bit locations
x = np.zeros((2**k, N),dtype=bool)
u = np.zeros((2**k, N),dtype=bool)
u[:,A] = d
for i in range(0,2**k):
x[i] = d_f.polar_transform_iter(u[i])
# One hot training vector
x_train = np.tile( x, (batch_size_norm ,1))
d_train = np.tile( np.eye(2**k) , (batch_size_norm,1))
# Training the neural net
history = model.fit(x_train, [d_train,d_train] , batch_size=batch_size, epochs=nb_epoch, verbose=2, shuffle=shuffle_var)
decoder.save_weights('decoder_lm_'+str(k)+'_'+str(N)+'_ours_q_'+str(100*train_q)+'.h5')
#if plot_losses==1:
plt.figure()
plt.semilogy(history.history['loss'])
# plt.plot(np.log(history.history['model_16_loss_2']))
# plt.plot(np.log(history.history['model_16_loss_1']))
# plt.title('model loss')
# plt.ylabel('loss')
# plt.xlabel('epoch')
# plt.legend(['loss', 'loss_1', 'loss_2'], loc='upper left')
# plt.show()
# plt.yscale('log')
#####################################################################
text_vocabulary_size,
size=(num_samples, max_length))
question = np.random.randint(1,
question_vocabulary_size,
size=(num_samples, max_length))
answers = np.random.randint(0, 1, size=(num_samples, answer_vocabulary_size))
# train
#model.fit([text, question], answers, epochs=10, batch_size=128) # will also work
print(text)
print(question)
print(answers)
history = model.fit({
'text': text,
'question': question
},
answers,
epochs=10,
batch_size=128,
validation_split=0.2) # Don't work
acc = history.history['acc']
val_acc = history.history['val_acc']
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(acc) + 1)
plt_loss(epochs, loss, val_loss).show()
plt_accuracy(epochs, acc, val_acc).show()
# model_5.compile(optimizer=OPTIMIZER, loss=LOSS)
# model_5.summary()
# for _ in tqdm(range(n_epoch)):
# model_5.fit(X2, y2, epochs=1, batch_size=n_batch, verbose=0)
# model_5.predict(X2, batch_size=n_batch, verbose=0)
y22 = (y2 * np.ones((1, 3)))[..., np.newaxis]
l1_6 = Input(shape=(3, 2))
l2_6 = LSTM(n_neurons, return_sequences=True)(l1_6)
l3_6 = TimeDistributed(Dense(1))(l2_6)
model_6 = Model(inputs=l1_6, outputs=l3_6)
model_6.compile(optimizer=OPTIMIZER, loss=LOSS)
model_6.summary()
for _ in tqdm(range(n_epoch)):
model_6.fit(X2, y22, epochs=1, batch_size=n_batch, verbose=0)
res = model_6.predict(X2, batch_size=n_batch, verbose=0)
print(y22)
print(res)
l1_7 = Input(shape=(3, 2))
l2_7 = Masking(mask_value=0.0)(l1_7)
l3_7 = LSTM(n_neurons, return_sequences=True)(l2_7)
l4_7 = TimeDistributed(Dense(1))(l3_7)
model_7 = Model(inputs=l1_7, outputs=l4_7)
model_7.compile(optimizer=OPTIMIZER, loss=LOSS)
model_7.summary()
model_7.set_weights(model_6.get_weights())
res = model_7.predict(X2, batch_size=n_batch, verbose=0)
print(y22)
print(res)
global_max_pool = SeqWeightedAttention()(embedding)
drop = Dropout(0.2)(global_max_pool)
output = Dense(num_classes, activation='softmax')(drop)
model = Model(inputs=input, outputs=output)
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
print(model.summary())
model_weight_file = './model_attention.h5'
model_file = './model_attention.model'
early_stopping = EarlyStopping(monitor='val_loss', patience=5)
model_checkpoint = ModelCheckpoint(model_weight_file, save_best_only=True, save_weights_only=True)
model.fit(x_train_word_index,
y_train_index,
batch_size=32,
epochs=1000,
verbose=2,
callbacks=[early_stopping, model_checkpoint],
validation_data=(x_dev_word_index, y_dev_index),
shuffle=True)
model.load_weights(model_weight_file)
model.save(model_file)
evaluate = model.evaluate(x_test_word_index, y_test_index, batch_size=32, verbose=2)
print('loss value=' + str(evaluate[0]))
print('metrics value=' + str(evaluate[1]))
# no attention
# loss value=0.7034960370215159
# metrics value=0.753968252076043
# ScaledDotProductAttention
embedded_text = layers.Embedding(text_vocabulary_size, 64)(text_input)
encoded_text = layers.LSTM(32)(embedded_text)
question_input = Input(shape=(None,), dtype='int32', name='question')
embedded_question = layers.Embedding(
question_vocabulary_size, 32)(question_input)
encoded_question = layers.LSTM(16)(embedded_question)
concatenated = layers.concatenate([encoded_text, encoded_question], axis=-1)
answer = layers.Dense(answer_vocabulary_size,
activation='softmax')(concatenated)
model = Model([text_input, question_input], answer)
model.compile(optimizer='rmsprop', loss='categorical_crossentropy',
metrics=['acc'])
# %%
num_samples = 1000
max_length = 100
text = np.random.randint(1, text_vocabulary_size,
size=(num_samples, max_length))
question = np.random.randint(
1, question_vocabulary_size, size=(num_samples, max_length))
answers = np.random.randint(1, answer_vocabulary_size, size=(num_samples,))
answers = keras.utils.to_categorical(answers, answer_vocabulary_size)
model.fit({'text': text, 'question': question},
answers, epochs=10, batch_size=128)
activation='relu',
use_bias=False)(layer)
layer = Flatten()(layer)
layer = Dropout(0.01)(layer)
layer = Dense(units=10, activation='softmax', use_bias=False)(layer)
model = Model(input_layer, layer)
model.summary()
model.compile('adam', 'categorical_crossentropy', ['accuracy'])
# Train model with backprop.
model.fit(x_train,
y_train,
batch_size=64,
epochs=1,
verbose=2,
validation_data=(x_test, y_test))
# Store model so SNN Toolbox can find it.
model_name = 'mnist_cnn'
keras.models.save_model(model, os.path.join(path_wd, model_name + '.h5'))
# SNN TOOLBOX CONFIGURATION #
#############################
# Create a config file with experimental setup for SNN Toolbox.
configparser = import_configparser()
config = configparser.ConfigParser()
config['paths'] = {
# optimizer = raw_input('Enter optimizer (default rmsprop): ')
optimizer = 'adagrad'
# loss = raw_input('Enter loss function (default categorical_crossentropy): ')
loss = 'categorical_crossentropy'
model.summary()
model.compile(loss=loss, optimizer=optimizer, metrics=['acc'])
# plot_model(model, to_file='model.png')
# epoch = input('Enter number of epochs: ')
epoch = 3
# batch = input('Enter number of batch size: ')
batch = 8
model.fit([np.array(x_train.padded),
np.array(x_train_char)], [np.array(y_encoded)],
epochs=epoch,
batch_size=batch)
"""
Converting text data to int using index
"""
x_test_tmp1 = []
for sent in test.words:
x_map = DM(sent, char.index, False)
if x_map.padsize > char_padsize:
char_padsize = x_map.padsize
x_test_tmp1.append(x_map)
x_test_tmp2 = []
for sent in x_test_tmp1:
sent.pad(char_padsize)
x_test_tmp2.append(sent.padded)
def train_model(data, topic, PROCESSED_DIR, SEED_FOLDER, **kwargs):
dropout = kwargs['model_settings']["dropout"]
lstm_size = kwargs['model_settings']["lstm_size"]
monitor = kwargs['model_settings']["monitor"]
batch_size = kwargs['model_settings']["batch_size"]
epochs = kwargs['model_settings']["epochs"]
learning_rate = kwargs['model_settings']["learning_rate"]
train_embeddings = kwargs['model_settings']["train_embeddings"]
# model file eg: 'results/only_sub_and_inst/model_runs/EvLSTM/seed_0/death_penalty_threelabel_crossdomain_monitor-f1_macro_do-0.3_lsize-32_bs-32_epochs-20_lr-0.001_trainemb-False_kl-only_sub_and_inst'
model_file = SEED_FOLDER + topic + "_" + kwargs['model_settings'][
"model_file_suffix"]
seed = kwargs['model_settings']['current_seed']
# clear default graph (new model now)
#tf.reset_default_graph()
# set configs for memory usage and reproducibility: https://stackoverflow.com/questions/38469632/tensorflow-non-repeatable-results
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
os.environ['PYTHONHASHSEED'] = str(seed)
np.random.seed(seed)
rn.seed(seed)
config = tf.ConfigProto()
config.gpu_options.allow_growth = False
config.gpu_options.per_process_gpu_memory_fraction = 0.3
np.random.seed(seed)
#graph_level_seed = seed
operation_level_seed = seed
#tf.set_random_seed(graph_level_seed)
# load data
X_train, X_dev, X_test = data["X_train"], data["X_dev"], data["X_test"]
y_train, y_dev, y_test = data["y_train"], data["y_dev"], data["y_test"]
# some constants
num_labels = y_train.shape[1]
sentence_inputs = Input(shape=(X_train.shape[1], ),
dtype='float32',
name="sentence_inputs")
dense = Dense(128)(sentence_inputs)
dropout_dense = Dropout(dropout)(dense)
output_layer = Dense(num_labels, activation='softmax')(dropout_dense)
model = Model(inputs=sentence_inputs, outputs=output_layer)
adam = Adam(lr=learning_rate)
model.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=['accuracy'])
model.summary()
#e = EarlyStopping(monitor=monitor, mode='auto')
e = ModelCheckpoint(model_file,
monitor=monitor,
verbose=0,
save_best_only=True,
save_weights_only=True,
mode='auto',
period=1)
model.fit(X_train,
y_train,
batch_size=batch_size,
epochs=epochs,
validation_data=(X_dev, y_dev),
callbacks=[e],
verbose=1)
model.load_weights(model_file)
y_pred_test = model.predict(X_test, verbose=False)
y_pred_dev = model.predict(X_dev, verbose=False)
return [np.argmax(pred)
for pred in y_pred_test], [np.argmax(pred) for pred in y_pred_dev]
fc = Dense(512, activation='relu')(flatten)
output = Dense(n_out, activation='softmax')(fc)
# Build model
model = Model(inputs=[X_in, A_in], outputs=output)
optimizer = Adam(lr=learning_rate)
model.compile(optimizer=optimizer,
loss='sparse_categorical_crossentropy',
metrics=['acc'])
model.summary()
# Train model
validation_data = (X_val, y_val)
model.fit(X_train,
y_train,
batch_size=batch_size,
validation_data=validation_data,
epochs=epochs,
callbacks=[
EarlyStopping(patience=es_patience, restore_best_weights=True)
])
# Evaluate model
print('Evaluating model.')
eval_results = model.evaluate(X_test,
y_test,
batch_size=batch_size)
print('Done.\n'
'Test loss: {}\n'
'Test acc: {}'.format(*eval_results))
model = Model(inputs=[user_id_input, movie_id_input], outputs=y)
sgd = optimizers.SGD(lr=param[0],
momentum=param[1],
decay=0.0,
nesterov=False)
model.compile(loss=rmse, optimizer=sgd, metrics=['mae'])
mytime = time.strftime("%Y_%m_%d_%H_%M")
modname = 'embedding_reg-' + str(config) + '_' + str(i) + '_' + mytime
# modname = 'embedding100_' + str(embedding) + '_' + str(i) + '_' + mytime
thename = modname + '.h5'
mcheck = ModelCheckpoint('models/' + thename,
monitor='val_loss',
save_best_only=True)
earlyStop = EarlyStopping(monitor='val_loss',
mode='min',
verbose=1,
patience=15)
history = model.fit([users_l, movies_l],
ratings_l,
batch_size=64,
epochs=100,
validation_data=([users_t, movies_t], ratings_t),
callbacks=[mcheck, earlyStop])
with open('histories/' + modname + '.pk1', 'wb') as file_pi:
pickle.dump(history.history, file_pi)
plot_loss(history.history, title=str(config) + '-' + str(i))
def main():
# Load data
observations = lib.load_lending_club()
# observations = lib.load_lending_club(test_run=False)
print('Observation columns: {}'.format(list(observations.columns)))
print('Class balance:\n {}'.format(
observations['loan_status'].value_counts()))
# Heuristic data transformations
for var in ['int_rate', 'revol_util']:
# Strip out percent signs
observations[var] = observations[var].apply(
lambda x: str(x).replace('%', ''))
observations[var] = pandas.to_numeric(observations[var],
errors='coerce')
for var in ['mths_since_last_delinq', 'annual_inc_joint']:
# Heuristic null filling for some variables
observations[var] = observations[var].fillna(0)
# List out variable types
numerical_vars = [
'loan_amnt', 'annual_inc', 'open_acc', 'dti', 'delinq_2yrs',
'inq_last_6mths', 'mths_since_last_delinq', 'pub_rec', 'revol_bal',
'revol_util', 'total_acc', 'pub_rec_bankruptcies'
]
categorical_vars = [
'term', 'grade', 'emp_length', 'home_ownership', 'loan_status',
'addr_state', 'application_type', 'disbursement_method'
]
text_vars = ['desc', 'purpose', 'title']
train_observations, test_observations = train_test_split(observations)
train_observations = train_observations.copy()
test_observations = test_observations.copy()
# Create and fit Automater
auto = Automater(numerical_vars=numerical_vars,
categorical_vars=categorical_vars,
text_vars=text_vars,
response_var='loan_status')
auto.fit(train_observations)
# Create and fit keras (deep learning) model
# The auto.transform, auto.input_nub, auto.input_layers, and auto.loss are provided by keras-pandas, and
# everything else is core Keras
train_X, train_y = auto.transform(train_observations)
test_X, test_y = auto.transform(test_observations)
x = auto.input_nub
x = Dense(32)(x)
x = Dense(32, activation='relu')(x)
x = Dense(32)(x)
x = auto.output_nub(x)
model = Model(inputs=auto.input_layers, outputs=x)
model.compile(optimizer='Adam', loss=auto.loss, metrics=['accuracy'])
model.fit(train_X, train_y)
test_y_pred = model.predict(test_X)
# Inverse transform model output, to get usable results and save all results
test_observations[auto.response_var +
'_pred'] = auto.inverse_transform_output(test_y_pred)
print('Predictions: {}'.format(test_observations[auto.response_var +
'_pred']))
pass
def main():
gpus = tf.config.experimental.list_physical_devices(device_type='GPU')
for gpu in gpus:
tf.config.experimental.set_memory_growth(gpu, True)
I_Dataset = pd.read_csv('./dataset/I_data.csv').values
V_Dataset = pd.read_csv('./dataset/V_Data.csv').values
Ir_T_Data = pd.read_csv('./dataset/Ir_T.csv').values
Label = pd.read_csv('./dataset/label.csv').transpose()
index = np.arange(0, Label.shape[1])
Label = Label.values[0].reshape(-1) - 1
random.shuffle(index)
Ir_T_Data[:, 0] = Ir_T_Data[:, 0] / 775.75
Ir_T_Data[:, 1] = Ir_T_Data[:, 1] / 52.734
I_Dataset = np.array(I_Dataset, dtype='float32') / 9.12
V_Dataset = np.array(V_Dataset, dtype='float32') / (13 * 37.78)
I_Dataset = np.expand_dims(I_Dataset, axis=2)
V_Dataset = np.expand_dims(V_Dataset, axis=2)
Ir_T_Data = np.expand_dims(Ir_T_Data, axis=2)
print(Ir_T_Data.shape)
I_Dataset = I_Dataset[index]
V_Dataset = V_Dataset[index]
Ir_T_Data = Ir_T_Data[index]
Label_Onehot = np_utils.to_categorical(Label, num_classes=5)[index]
V_train = Input(shape=(1000, 1), name='V_train')
I_train = Input(shape=(1000, 1), name='I_train')
Ir_T_train = Input(shape=(2, 1), name='Ir_T_train')
model_input = [V_train, I_train, Ir_T_train]
IV_train = concatenate([V_train, I_train], axis=1)
LSTM_Model = concatenate([IV_train, Ir_T_train], axis=1)
# Conv_Model = Conv1D(filters=64, kernel_size=10, activation='relu',
# kernel_initializer=initializers.he_normal())(IV_train)
#
# Conv_Model = MaxPooling1D(pool_size=2, strides=2)(Conv_Model)
#
# Conv_Model = Conv1D(filters=128, kernel_size=10, activation='relu',
# kernel_initializer=initializers.he_normal())(Conv_Model)
#
# Conv_Model = MaxPooling1D(pool_size=2, strides=2)(Conv_Model)
#
# Conv_Model = BatchNormalization()(Conv_Model)
# Conv_Model = Dropout(0.6)(Conv_Model)
LSTM_Model = LSTM(256, activation='relu')(LSTM_Model)
LSTM_Model = BatchNormalization()(LSTM_Model)
LSTM_Model = Dropout(0.2)(LSTM_Model)
# Conv_shape = Conv_Model.shape[1] * Conv_Model.shape[2] + 2
#
# Conv_Model = Flatten()(Conv_Model)
#
# Conv_Model = concatenate([Conv_Model, Ir_T_train], axis=1)
#
# Conv_Model = Reshape((Conv_shape, 1))(Conv_Model)
# LSTM_Model = LSTM(128, activation='relu')(Conv_Model)
# LSTM_Model = Dropout(0.5)(LSTM_Model)
Fc_Model = Dense(128, activation='relu')(LSTM_Model)
Fc_Model = Dense(64, activation='relu')(Fc_Model)
Fc_Out = Dense(5,
kernel_regularizer=regularizers.l2(0.0015),
activation='softmax',
name='Fc_Out')(Fc_Model)
model = Model(inputs=model_input, outputs=[Fc_Out])
model.compile(loss='categorical_crossentropy',
optimizer=optimizers.Adam(lr=1e-4),
metrics=['accuracy'])
model.fit(
{
'V_train': V_Dataset,
'I_train': I_Dataset,
'Ir_T_train': Ir_T_Data
}, {'Fc_Out': Label_Onehot},
epochs=200,
batch_size=32,
validation_split=0.2)
model.save('model2.h5')
def test_TensorBoard(tmpdir):
np.random.seed(np.random.randint(1, 1e7))
filepath = str(tmpdir / "logs")
(X_train, y_train), (X_test, y_test) = get_data_callbacks()
y_test = to_categorical(y_test)
y_train = to_categorical(y_train)
class DummyStatefulMetric(Layer):
def __init__(self, name="dummy_stateful_metric", **kwargs):
super(DummyStatefulMetric, self).__init__(name=name, **kwargs)
self.stateful = True
self.state = K.variable(value=0, dtype="int32")
def reset_states(self):
pass
def __call__(self, y_true, y_pred):
return self.state
inp = Input((input_dim, ))
hidden = Dense(num_hidden, activation="relu")(inp)
hidden = Dropout(0.1)(hidden)
hidden = BatchNormalization()(hidden)
output = Dense(num_classes, activation="softmax")(hidden)
model = Model(inputs=inp, outputs=output)
model.compile(
loss="categorical_crossentropy",
optimizer="sgd",
metrics=["accuracy", DummyStatefulMetric()],
)
# we must generate new callbacks for each test, as they aren't stateless
def callbacks_factory(histogram_freq):
return [
TensorBoardGrouped(
log_dir=filepath,
histogram_freq=histogram_freq,
write_images=True,
write_grads=True,
batch_size=5,
)
]
# fit without validation data
model.fit(
X_train,
y_train,
batch_size=batch_size,
callbacks=callbacks_factory(histogram_freq=0),
epochs=3,
)
# fit with validation data and accuracy
model.fit(
X_train,
y_train,
batch_size=batch_size,
validation_data=(X_test, y_test),
callbacks=callbacks_factory(histogram_freq=0),
epochs=2,
)
# fit generator without validation data
train_generator = data_generator(X_train, y_train, batch_size)
model.fit_generator(
train_generator,
len(X_train),
epochs=2,
callbacks=callbacks_factory(histogram_freq=0),
)
# fit generator with validation data and accuracy
train_generator = data_generator(X_train, y_train, batch_size)
model.fit_generator(
train_generator,
len(X_train),
epochs=2,
validation_data=(X_test, y_test),
callbacks=callbacks_factory(histogram_freq=1),
)
assert os.path.isdir(filepath)
shutil.rmtree(filepath)
assert not tmpdir.listdir()
from keras import Input, Model, layers input_tensor = Input(shape=(64, )) x0 = layers.Dense(32, activation='relu')(input_tensor) x1 = layers.Dense(32, activation='relu')(x0) output_tensor = layers.Dense(10, activation='softmax')(x1) model = Model(input_tensor, output_tensor) print(model.summary()) model.compile(optimizer='rmsprop', loss='categorical_crossentropy') x = np.random.random((1000, 64)) y = np.random.random((1000, 10)) model.fit(x, y, epochs=10, batch_size=128) score = model.evaluate(x, y)
pool2 = wl.MyLayer(output_dim=(None, 8, 8, 64), haar_matrix=haarMatrix16)(relu2) conv3 = Conv2D(64, kernel_size=(5,5), padding='same')(pool2) batch3 = BatchNormalization()(conv3) relu3 = LeakyReLU(alpha=0)(batch3) pool3 = wl.MyLayer(output_dim=(None, 4, 4, 64), haar_matrix=haarMatrix8)(relu3) ## NO DROPOUT: drop = Dropout(rate=0.1)(pool3) conv4 = Conv2D(128, kernel_size=(4,4), padding='same')(pool3) relu3 = LeakyReLU(alpha=0)(conv4) ##NO DROPOUT: drop2 = Dropout(rate=0.1)(relu3) conv5 = Conv2D(10, kernel_size=(1,1), padding='same')(relu3) flat = Flatten()(conv5) activ = Dense(units=10, activation='softmax')(flat) model = Model(inputs=input, outputs=activ) model.compile(loss='categorical_crossentropy', optimizer='sgd', metrics=['accuracy']) model.fit(x_train, y_train, epochs=2, batch_size=32) loss_and_metrics = model.evaluate(x_test, y_test, batch_size=128) print(loss_and_metrics)
class StyleTransfer():
def __init__(self,
style_image_path,
path,
style_weight=1.,
content_weight=1.,
input_size=(256, 256),
optimizer=None,
sub_path=None,
test_photo_path=None,
load_model=None,
load_only=False):
"""Style transfer class
Args:
style_image_path: path to style image
path: base path
style_weight: weight for the style loss
content_weight: weight for the content loss
input_size: size used for training
optimizer: pass a keras optimizer, Adam used standard
sub_path: sub path used for every model, automatically generated if not given
test_photo_path: a path to a folder of test photos to run on after training
load_model: path to a pretrained model
load_only: do not set up with a new directory for training, only prediction"""
if load_model is None:
self.build_generator_net(input_size=input_size)
else:
self.build_from_path(load_model)
if not load_only:
self.style_image = load_and_process_img(style_image_path,
resize=False)
self.style_image_path = style_image_path
self.style_weight = style_weight
self.content_weight = content_weight
self.loss, self.cl, self.sl = get_loss_func(
self.style_image,
style_weight=style_weight,
content_weight=content_weight)
if optimizer is None:
self.optimizer = optimizers.Adam
else:
self.optimizer = optimizer
self.path = path
if sub_path is None:
i = 0
while os.path.exists(self.path + '{:05d}/'.format(i)):
i += 1
self.sub_path = '{:05d}/'.format(i)
os.mkdir(self.path + self.sub_path)
self.test_photo_path = test_photo_path
def build_generator_net(self, input_size=(256, 256)):
inp = layers.Input(shape=(input_size[0], input_size[1], 3))
#encoder:
y = Conv2DReflect(inp, 32, kernel_size=(9, 9), strides=(1, 1))
y = InstanceNormalization()(y)
y = layers.ReLU()(y)
y = Conv2DReflect(y, 64, kernel_size=(3, 3), strides=(2, 2))
y = InstanceNormalization()(y)
y = layers.ReLU()(y)
y = Conv2DReflect(y, 128, kernel_size=(3, 3), strides=(2, 2))
y = InstanceNormalization()(y)
y = layers.ReLU()(y)
y = residual_block(y, 128)
y = residual_block(y, 128)
y = residual_block(y, 128)
y = residual_block(y, 128)
y = residual_block(y, 128)
y = layers.UpSampling2D(size=(2, 2))(y)
y = Conv2DReflect(y, 64, kernel_size=(3, 3), strides=(1, 1))
y = InstanceNormalization()(y)
y = layers.ReLU()(y)
y = layers.UpSampling2D(size=(2, 2))(y)
y = Conv2DReflect(y, 32, kernel_size=(3, 3), strides=(1, 1))
y = InstanceNormalization()(y)
y = layers.ReLU()(y)
y = Conv2DReflect(y, 3, kernel_size=(9, 9), strides=(1, 1))
# y = layers.BatchNormalization()(y)
y = layers.Activation('tanh')(y)
y = layers.Lambda(lambda x: (x * 150))(
y) #scale output to imagenet means
self.generator = Model(inp, y)
def train(self,
files,
verbose=1,
epochs=2,
batch_size=8,
decay_lr=[1e-4, 1e-5],
save_checkpoint=20,
chunk_size=1000,
save_img=True,
save_best_loss=True):
"""Train the classifier
Args:
files: a list of filepaths
verbose: verbosity for output generation
epochs: number of full iterations of the dataset
batch_size: size of each training barch
decay_lr: list of learn_rates size of epochs
save_checkpoint: save the model every n times chunk_size
chunk_size: number of files to load at once
save_img: if true, save a training image every chunk
save_best_loss: save on the best loss"""
iterations = len(files) // chunk_size
self.cl_history = []
self.sl_history = []
self.loss_history = []
self.epochs = epochs
self.batch_size = batch_size
self.decay_lr = decay_lr
self.chunk_size = chunk_size
self.save_checkpoint = save_checkpoint
self.best_loss = np.inf
self.best_cl = 0
self.best_sl = 0
self.best_loss_epoch = 0
self.best_loss_iter = 0
for j in range(epochs):
self.generator.compile(loss=self.loss,
optimizer=self.optimizer(decay_lr[j]),
metrics=[self.cl, self.sl])
for i in range(iterations):
if verbose:
print('Iteration {} out of {}'.format(i, iterations))
data = load_train_data(chunk_size,
files=files[i * chunk_size:(i + 1) *
chunk_size])
history = self.generator.fit(data,
data,
batch_size=batch_size,
epochs=1,
verbose=verbose)
self.cl_history.append(history.history['cl'][0])
self.sl_history.append(history.history['sl'][0])
self.loss_history.append(history.history['loss'][0])
if (i % 5) == 0 and save_img:
im_save_dir = self.path + self.sub_path + 'im_checkpoints/'
if not os.path.isdir(im_save_dir):
os.mkdir(im_save_dir)
pred = self.generator.predict(data[0:1])
save_and_deprocess_img(
pred[0], self.path + self.sub_path +
'im_checkpoints/pred_epoch_{}_iteration_{}.png'.format(
j, i))
save_and_deprocess_img(
data[0], self.path + self.sub_path +
'im_checkpoints/data_epoch_{}_iteration_{}.png'.format(
j, i))
self.generator.save(self.path + self.sub_path +
'last_checkpoint_.h5',
include_optimizer=False)
if (i % save_checkpoint) == 0:
self.generator.save(self.path + self.sub_path +
'checkpoint_{:02d}.h5'.format(i),
include_optimizer=False)
if save_best_loss and self.loss_history[-1] < self.best_loss:
self.best_loss = self.loss_history[-1]
self.best_cl = self.cl_history[-1]
self.best_sl = self.sl_history[-1]
self.best_loss_epoch = j
self.best_loss_iter = i
self.generator.save(self.path + self.sub_path +
'best_checkpoint.h5',
include_optimizer=False)
self.write_json()
def build_from_path(self, path):
self.generator = load_model(path,
custom_objects={
'ReflectionPadding2D':
ReflectionPadding2D,
'InstanceNormalization':
InstanceNormalization
})
def write_json(self):
"""Write a dictionary to be able to log training"""
out_dict = {
'style_image_path': self.style_image_path,
'style_weight': self.style_weight,
'content_weight': self.content_weight,
'optimizer': self.optimizer.__name__,
'epochs': self.epochs,
'batch_size': self.batch_size,
'chunk_size': self.chunk_size,
'decay_lr': self.decay_lr,
'save_checkpoint': self.save_checkpoint,
'cl_history': self.cl_history,
'sl_history': self.sl_history,
'loss_history': self.loss_history,
'best_loss': self.best_loss,
'best_cl': self.best_cl,
'best_sl': self.best_sl,
'best_loss_epoch': self.best_loss_epoch,
'best_loss_iter': self.best_loss_iter,
}
with open(self.path + self.sub_path + 'log.json', 'w') as f:
json.dump(out_dict, f)
digit_placeholders = [
Dense(9, activation='softmax')(features)
for i in range(81)
]
solver = Model(grid, digit_placeholders) # build the whole model
solver.compile(
optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy']
)
solver.fit(
delete_digits(Xtrain, 0), # we don't delete any digit for now
[ytrain[:, i, j, :] for i in range(9) for j in range(9)], # each digit of solution
batch_size=128,
epochs=1, # 1 epoch should be enough for the task
verbose=1,
)
early_stop = EarlyStopping(patience=2, verbose=1)
i = 1
for nb_epochs, nb_delete in zip(
[1, 2, 3, 4, 6, 8, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10], # epochs for each round
[1, 2, 3, 4, 6, 8, 10, 12, 15, 20, 25, 30, 35, 40, 45, 50, 55] # digit to pull off
):
print('Pass n° {} ...'.format(i))
i += 1
solver.fit(
class VAEArithKeras:
"""
VAE with Arithmetic vector Network class. This class contains the implementation of Variational
Auto-encoder network with Vector Arithmetics.
This model strictly employs 5 unit dimensional space because of the loss function
Parameters
----------
kwargs:
:key `validation_data` : AnnData
must be fed if `use_validation` is true.
:key dropout_rate: float
dropout rate
:key learning_rate: float
learning rate of optimization algorithm
:key model_path: basestring
path to save the model after training
x_dimension: integer
number of gene expression space dimensions.
z_dimension: integer
number of latent space dimensions.
c_max: integer
Value of C used in the loss function.
alpha: float
Weight for the KL Divergence term in loss function.
"""
def __init__(self, x_dimension, z_dimension=100, **kwargs):
self.x_dim = x_dimension
self.z_dim = z_dimension
self.learning_rate = kwargs.get("learning_rate", 0.001)
self.dropout_rate = kwargs.get("dropout_rate", 0.2)
self.model_to_use = kwargs.get("model_to_use", "./models/")
self.alpha = kwargs.get("alpha", 0.00005)
self.c_max = kwargs.get("c_max", 20)
self.c_current = K.variable(value=0.01)
self.x = Input(shape=(x_dimension, ), name="input")
self.z = Input(shape=(z_dimension, ), name="latent")
self.init_w = keras.initializers.glorot_normal()
self._create_network()
self._loss_function()
self.vae_model.summary()
def _encoder(self):
"""
Constructs the encoder sub-network of VAE. This function implements the
encoder part of Variational Auto-encoder. It will transform primary
data in the `n_vars` dimension-space to the `z_dimension` latent space.
Parameters
----------
No parameters are needed.
Returns
-------
mean: Tensor
A dense layer consists of means of gaussian distributions of latent space dimensions.
log_var: Tensor
A dense layer consists of log transformed variances of gaussian distributions of latent space dimensions.
"""
h = Dense(800, kernel_initializer=self.init_w, use_bias=False)(self.x)
h = BatchNormalization(axis=1)(h)
h = LeakyReLU()(h)
h = Dropout(self.dropout_rate)(h)
h = Dense(800, kernel_initializer=self.init_w, use_bias=False)(h)
h = BatchNormalization(axis=1)(h)
h = LeakyReLU()(h)
h = Dropout(self.dropout_rate)(h)
# h = Dense(512, kernel_initializer=self.init_w, use_bias=False)(h)
# h = BatchNormalization()(h)
# h = LeakyReLU()(h)
# h = Dropout(self.dropout_rate)(h)
# h = Dense(256, kernel_initializer=self.init_w, use_bias=False)(h)
# h = BatchNormalization()(h)
# h = LeakyReLU()(h)
# h = Dropout(self.dropout_rate)(h)
mean = Dense(self.z_dim, kernel_initializer=self.init_w)(h)
log_var = Dense(self.z_dim, kernel_initializer=self.init_w)(h)
z = Lambda(self._sample_z, output_shape=(self.z_dim, ),
name="Z")([mean, log_var])
self.encoder_model = Model(inputs=self.x, outputs=z, name="encoder")
return mean, log_var
def _decoder(self):
"""
Constructs the decoder sub-network of VAE. This function implements the
decoder part of Variational Auto-encoder. It will transform constructed
latent space to the previous space of data with n_dimensions = n_vars.
Parameters
----------
No parameters are needed.
Returns
-------
h: Tensor
A Tensor for last dense layer with the shape of [n_vars, ] to reconstruct data.
"""
h = Dense(800, kernel_initializer=self.init_w, use_bias=False)(self.z)
h = BatchNormalization(axis=1)(h)
h = LeakyReLU()(h)
h = Dropout(self.dropout_rate)(h)
h = Dense(800, kernel_initializer=self.init_w, use_bias=False)(h)
h = BatchNormalization(axis=1)(h)
h = LeakyReLU()(h)
h = Dropout(self.dropout_rate)(h)
# h = Dense(768, kernel_initializer=self.init_w, use_bias=False)(h)
# h = BatchNormalization()(h)
# h = LeakyReLU()(h)
# h = Dropout(self.dropout_rate)(h)
# h = Dense(1024, kernel_initializer=self.init_w, use_bias=False)(h)
# h = BatchNormalization()(h)
# h = LeakyReLU()(h)
# h = Dropout(self.dropout_rate)(h)
h = Dense(self.x_dim, kernel_initializer=self.init_w, use_bias=True)(h)
self.decoder_model = Model(inputs=self.z, outputs=h, name="decoder")
return h
@staticmethod
def _sample_z(args):
"""
Samples from standard Normal distribution with shape [size, z_dim] and
applies re-parametrization trick. It is actually sampling from latent
space distributions with N(mu, var) computed in `_encoder` function.
Parameters
----------
No parameters are needed.
Returns
-------
The computed Tensor of samples with shape [size, z_dim].
"""
mu, log_var = args
batch_size = K.shape(mu)[0]
z_dim = K.shape(mu)[1]
eps = K.random_normal(shape=[batch_size, z_dim])
return mu + K.exp(log_var / 2) * eps
def _create_network(self):
"""
Constructs the whole VAE network. It is step-by-step constructing the VAE
network. First, It will construct the encoder part and get mu, log_var of
latent space. Second, It will sample from the latent space to feed the
decoder part in next step. Finally, It will reconstruct the data by
constructing decoder part of VAE.
Parameters
----------
No parameters are needed.
Returns
-------
Nothing will be returned.
"""
self.mu, self.log_var = self._encoder()
self.x_hat = self._decoder()
self.vae_model = Model(inputs=self.x,
outputs=self.decoder_model(
self.encoder_model(self.x)),
name="VAE")
def _loss_function(self):
"""
Defines the loss function of VAE network after constructing the whole
network. This will define the KL Divergence and Reconstruction loss for
VAE and also defines the Optimization algorithm for network. The VAE Loss
will be weighted sum of reconstruction loss and KL Divergence loss.
The loss function also returns KL Divergence for every latent space dimension.
Parameters
----------
No parameters are needed.
Returns
-------
Nothing will be returned.
"""
def vae_loss(y_true, y_pred):
print(self.c_current)
return K.mean(
recon_loss(y_true, y_pred) +
self.alpha * abs(kl_loss(y_true, y_pred) - self.c_current))
def kl_loss(y_true, y_pred):
return 0.5 * K.sum(
K.exp(self.log_var) + K.square(self.mu) - 1. - self.log_var,
axis=1)
def kl_loss_monitor0(y_true, y_pred):
klds = K.mean(K.exp(self.log_var) + K.square(self.mu) - 1. -
self.log_var,
axis=0)
return klds[0]
def kl_loss_monitor1(y_true, y_pred):
klds = K.mean(K.exp(self.log_var) + K.square(self.mu) - 1. -
self.log_var,
axis=0)
#K.print_tensor(klds)
return klds[1]
def kl_loss_monitor2(y_true, y_pred):
klds = K.mean(K.exp(self.log_var) + K.square(self.mu) - 1. -
self.log_var,
axis=0)
#K.print_tensor(klds)
return klds[2]
def kl_loss_monitor3(y_true, y_pred):
klds = K.mean(K.exp(self.log_var) + K.square(self.mu) - 1. -
self.log_var,
axis=0)
#K.print_tensor(klds)
return klds[3]
def kl_loss_monitor4(y_true, y_pred):
klds = K.mean(K.exp(self.log_var) + K.square(self.mu) - 1. -
self.log_var,
axis=0)
#K.print_tensor(klds)
return klds[4]
def recon_loss(y_true, y_pred):
return 0.5 * K.sum(K.square((y_true - y_pred)), axis=1)
def get_c_current(y_true, y_pred):
return self.c_current
self.vae_optimizer = keras.optimizers.Adam(lr=self.learning_rate)
self.vae_model.compile(optimizer=self.vae_optimizer,
loss=vae_loss,
metrics=[
kl_loss, recon_loss, get_c_current,
kl_loss_monitor0, kl_loss_monitor1,
kl_loss_monitor2, kl_loss_monitor3,
kl_loss_monitor4
])
def to_latent(self, data):
"""
Map `data` in to the latent space. This function will feed data
in encoder part of VAE and compute the latent space coordinates
for each sample in data.
Parameters
----------
data: numpy nd-array
Numpy nd-array to be mapped to latent space. `data.X` has to be in shape [n_obs, n_vars].
Returns
-------
latent: numpy nd-array
Returns array containing latent space encoding of 'data'
"""
latent = self.encoder_model.predict(data)
return latent
def _avg_vector(self, data):
"""
Computes the average of points which computed from mapping `data`
to encoder part of VAE.
Parameters
----------
data: numpy nd-array
Numpy nd-array matrix to be mapped to latent space. Note that `data.X` has to be in shape [n_obs, n_vars].
Returns
-------
The average of latent space mapping in numpy nd-array.
"""
latent = self.to_latent(data)
latent_avg = numpy.average(latent, axis=0)
return latent_avg
def reconstruct(self, data):
"""
Map back the latent space encoding via the decoder.
Parameters
----------
data: `~anndata.AnnData`
Annotated data matrix whether in latent space or gene expression space.
use_data: bool
This flag determines whether the `data` is already in latent space or not.
if `True`: The `data` is in latent space (`data.X` is in shape [n_obs, z_dim]).
if `False`: The `data` is not in latent space (`data.X` is in shape [n_obs, n_vars]).
Returns
-------
rec_data: 'numpy nd-array'
Returns 'numpy nd-array` containing reconstructed 'data' in shape [n_obs, n_vars].
"""
rec_data = self.decoder_model.predict(x=data)
return rec_data
def linear_interpolation(self, source_adata, dest_adata, n_steps):
"""
Maps `source_adata` and `dest_adata` into latent space and linearly interpolate
`n_steps` points between them.
Parameters
----------
source_adata: `~anndata.AnnData`
Annotated data matrix of source cells in gene expression space (`x.X` must be in shape [n_obs, n_vars])
dest_adata: `~anndata.AnnData`
Annotated data matrix of destinations cells in gene expression space (`y.X` must be in shape [n_obs, n_vars])
n_steps: int
Number of steps to interpolate points between `source_adata`, `dest_adata`.
Returns
-------
interpolation: numpy nd-array
Returns the `numpy nd-array` of interpolated points in gene expression space.
Example
--------
>>> import anndata
>>> import scgen
>>> train_data = anndata.read("./data/train.h5ad")
>>> validation_data = anndata.read("./data/validation.h5ad")
>>> network = scgen.VAEArith(x_dimension= train_data.shape[1], model_path="./models/test" )
>>> network.train(train_data=train_data, use_validation=True, validation_data=validation_data, shuffle=True, n_epochs=2)
>>> souece = train_data[((train_data.obs["cell_type"] == "CD8T") & (train_data.obs["condition"] == "control"))]
>>> destination = train_data[((train_data.obs["cell_type"] == "CD8T") & (train_data.obs["condition"] == "stimulated"))]
>>> interpolation = network.linear_interpolation(souece, destination, n_steps=25)
"""
if sparse.issparse(source_adata.X):
source_average = source_adata.X.A.mean(axis=0).reshape(
(1, source_adata.shape[1]))
else:
source_average = source_adata.X.A.mean(axis=0).reshape(
(1, source_adata.shape[1]))
if sparse.issparse(dest_adata.X):
dest_average = dest_adata.X.A.mean(axis=0).reshape(
(1, dest_adata.shape[1]))
else:
dest_average = dest_adata.X.A.mean(axis=0).reshape(
(1, dest_adata.shape[1]))
start = self.to_latent(source_average)
end = self.to_latent(dest_average)
vectors = numpy.zeros((n_steps, start.shape[1]))
alpha_values = numpy.linspace(0, 1, n_steps)
for i, alpha in enumerate(alpha_values):
vector = start * (1 - alpha) + end * alpha
vectors[i, :] = vector
vectors = numpy.array(vectors)
interpolation = self.reconstruct(vectors)
return interpolation
def predict(self,
adata,
conditions,
cell_type_key,
condition_key,
adata_to_predict=None,
celltype_to_predict=None,
obs_key="all"):
"""
Predicts the cell type provided by the user in stimulated condition.
Parameters
----------
celltype_to_predict: basestring
The cell type you want to be predicted.
obs_key: basestring or dict
Dictionary of celltypes you want to be observed for prediction.
adata_to_predict: `~anndata.AnnData`
Adata for unpertubed cells you want to be predicted.
Returns
-------
predicted_cells: numpy nd-array
`numpy nd-array` of predicted cells in primary space.
delta: float
Difference between stimulated and control cells in latent space
Example
--------
>>> import anndata
>>> import scgen
>>> train_data = anndata.read("./data/train.h5ad"
>>> validation_data = anndata.read("./data/validation.h5ad")
>>> network = scgen.VAEArith(x_dimension= train_data.shape[1], model_path="./models/test" )
>>> network.train(train_data=train_data, use_validation=True, validation_data=validation_data, shuffle=True, n_epochs=2)
>>> prediction, delta = pred, delta = scg.predict(adata= train_new,conditions={"ctrl": "control", "stim":"stimulated"},
cell_type_key="cell_type",condition_key="condition",adata_to_predict=unperturbed_cd4t)
"""
if obs_key == "all":
ctrl_x = adata[adata.obs["condition"] == conditions["ctrl"], :]
stim_x = adata[adata.obs["condition"] == conditions["stim"], :]
ctrl_x = ul.balancer(ctrl_x,
cell_type_key=cell_type_key,
condition_key=condition_key)
stim_x = ul.balancer(stim_x,
cell_type_key=cell_type_key,
condition_key=condition_key)
else:
key = list(obs_key.keys())[0]
values = obs_key[key]
subset = adata[adata.obs[key].isin(values)]
ctrl_x = subset[subset.obs["condition"] == conditions["ctrl"], :]
stim_x = subset[subset.obs["condition"] == conditions["stim"], :]
if len(values) > 1:
ctrl_x = ul.balancer(ctrl_x,
cell_type_key=cell_type_key,
condition_key=condition_key)
stim_x = ul.balancer(stim_x,
cell_type_key=cell_type_key,
condition_key=condition_key)
if celltype_to_predict is not None and adata_to_predict is not None:
raise Exception(
"Please provide either a cell type or adata not both!")
if celltype_to_predict is None and adata_to_predict is None:
raise Exception(
"Please provide a cell type name or adata for your unperturbed cells"
)
if celltype_to_predict is not None:
ctrl_pred = ul.extractor(adata, celltype_to_predict, conditions,
cell_type_key, condition_key)[1]
else:
ctrl_pred = adata_to_predict
eq = min(ctrl_x.X.shape[0], stim_x.X.shape[0])
cd_ind = numpy.random.choice(range(ctrl_x.shape[0]),
size=eq,
replace=False)
stim_ind = numpy.random.choice(range(stim_x.shape[0]),
size=eq,
replace=False)
if sparse.issparse(ctrl_x.X) and sparse.issparse(stim_x.X):
latent_ctrl = self._avg_vector(ctrl_x.X.A[cd_ind, :])
latent_sim = self._avg_vector(stim_x.X.A[stim_ind, :])
else:
latent_ctrl = self._avg_vector(ctrl_x.X[cd_ind, :])
latent_sim = self._avg_vector(stim_x.X[stim_ind, :])
delta = latent_sim - latent_ctrl
if sparse.issparse(ctrl_pred.X):
latent_cd = self.to_latent(ctrl_pred.X.A)
else:
latent_cd = self.to_latent(ctrl_pred.X)
stim_pred = delta + latent_cd
predicted_cells = self.reconstruct(stim_pred)
return predicted_cells, delta
def restore_model(self):
"""K.variable(value=0.0)
restores model weights from `model_to_use`.
Parameters
----------
No parameters are needed.
Returns
-------
Nothing will be returned.
Example
--------
>>> import anndata
>>> import scgen
>>> train_data = anndata.read("./data/train.h5ad")
>>> validation_data = anndata.read("./data/validation.h5ad")
>>> network = scgen.VAEArith(x_dimension= train_data.shape[1], model_path="./models/test" )
>>> network.restore_model()
"""
self.vae_model = load_model(os.path.join(self.model_to_use, 'vae.h5'),
compile=False)
self.encoder_model = load_model(os.path.join(self.model_to_use,
'encoder.h5'),
compile=False)
self.decoder_model = load_model(os.path.join(self.model_to_use,
'decoder.h5'),
compile=False)
self._loss_function()
def train(self,
train_data,
validation_data=None,
n_epochs=25,
batch_size=32,
early_stop_limit=20,
threshold=0.0025,
initial_run=True,
shuffle=True,
verbose=1,
save=True,
checkpoint=50,
**kwargs):
"""
Trains the network `n_epochs` times with given `train_data`
and validates the model using validation_data if it was given
in the constructor function. This function is using `early stopping`
technique to prevent over-fitting.
Parameters
----------
train_data: scanpy AnnData
Annotated Data Matrix for training VAE network.
validation_data: scanpy AnnData
Annotated Data Matrix for validating VAE network after each epoch.
n_epochs: int
Number of epochs to iterate and optimize network weights
batch_size: integer
size of each batch of training dataset to be fed to network while training.
early_stop_limit: int
Number of consecutive epochs in which network loss is not going lower.
After this limit, the network will stop training.
threshold: float
Threshold for difference between consecutive validation loss values
if the difference is upper than this `threshold`, this epoch will not
considered as an epoch in early stopping.
initial_run: bool
if `True`: The network will initiate training and log some useful initial messages.
if `False`: Network will resume the training using `restore_model` function in order
to restore last model which has been trained with some training dataset.
shuffle: bool
if `True`: shuffles the training dataset
Returns
-------
Nothing will be returned
Example
--------
```python
import anndata
import scgen
train_data = anndata.read("./data/train.h5ad"
validation_data = anndata.read("./data/validation.h5ad"
network = scgen.VAEArith(x_dimension= train_data.shape[1], model_path="./models/test")
network.train(train_data=train_data, use_validation=True, valid_data=validation_data, shuffle=True, n_epochs=2)
```
"""
if initial_run:
log.info("----Training----")
if shuffle:
train_data = ul.shuffle_adata(train_data)
if sparse.issparse(train_data.X):
train_data.X = train_data.X.A
# def on_epoch_end(epoch, logs):
# if epoch % checkpoint == 0:
# path_to_save = os.path.join(kwargs.get("path_to_save"), f"epoch_{epoch}") + "/"
# scgen.visualize_trained_network_results(self, vis_data, kwargs.get("cell_type"),
# kwargs.get("conditions"),
# kwargs.get("condition_key"), kwargs.get("cell_type_key"),
# path_to_save,
# plot_umap=False,
# plot_reg=True)
os.makedirs(self.model_to_use, exist_ok=True)
def update_val_c(epoch):
print(epoch)
value = (self.c_max / n_epochs) + K.get_value(self.c_current)
K.set_value(self.c_current, value)
callbacks = [
LambdaCallback(
on_epoch_end=lambda epoch, log: update_val_c(epoch)),
# EarlyStopping(patience=early_stop_limit, monitor='loss', min_delta=threshold),
CSVLogger(filename=self.model_to_use + "/csv_logger.log"),
ModelCheckpoint(os.path.join(self.model_to_use +
"/model_checkpoint.h5"),
monitor='vae_loss',
verbose=1),
EarlyStopping(monitor='vae_loss', patience=5, verbose=1)
]
K.set_value(self.c_current, (self.c_max / n_epochs))
if validation_data is not None:
result = self.vae_model.fit(x=train_data.X,
y=train_data.X,
epochs=n_epochs,
batch_size=batch_size,
validation_data=(validation_data.X,
validation_data.X),
shuffle=shuffle,
callbacks=callbacks,
verbose=verbose)
else:
result = self.vae_model.fit(x=train_data.X,
y=train_data.X,
epochs=n_epochs,
batch_size=batch_size,
shuffle=shuffle,
callbacks=callbacks,
verbose=verbose)
if save is True:
#os.chdir(self.model_to_use)
self.vae_model.save(os.path.join(self.model_to_use + "/vae.h5"),
overwrite=True)
self.encoder_model.save(os.path.join(self.model_to_use +
"/encoder.h5"),
overwrite=True)
self.decoder_model.save(os.path.join(self.model_to_use +
"/decoder.h5"),
overwrite=True)
log.info(
f"Models are saved in file: {self.model_to_use}. Training finished"
)
return result
model.save_weights('before_gap_param.h5')
# データセットの読み込み
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
x_train = x_train.astype('float') / 255.
x_test = x_test.astype('float') / 255.
y_train = to_categorical(y_train, num_classes=10)
y_test = to_categorical(y_test, num_classes=10)
# 学習設定
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
_results = model.fit(x=x_train,
y=y_train,
batch_size=100,
epochs=150,
verbose=1,
validation_data=(x_test, y_test))
# 結果のプロット
loss = _results.history['loss']
val_loss = _results.history['val_loss']
acc = _results.history['acc']
val_acc = _results.history['val_acc']
model.save_weights('after_gap_param.h5')
plt.figure()
plt.plot(range(1, 150 + 1), loss, marker='.', label='train')
plt.plot(range(1, 150 + 1), val_loss, marker='.', label='test')
plt.legend(loc='best', fontsize=10)
(1 - Y_true) * K.maximum((margin - D), 0))
#callbacks과 ModelCheckpoint설정
callback_list = [
keras.callbacks.EarlyStopping(monitor='acc', patience=5),
keras.callbacks.ModelCheckpoint(filepath='model.h5',
monitor='loss',
save_best_only=True)
]
basic_model.compile(loss=contrastive_loss, optimizer='adam', metrics=['acc'])
# Model Fit! 제발 잘 됐으면 좋.겠.다.
history = basic_model.fit([training_pairs[:, 0], training_pairs[:, 1]],
training_labels,
epochs=10,
callbacks=callback_list)
#테스트 이미지 쌍으로 얼마나 잘 작동하는 지 확인
idx1, idx2 = 2, 60
img1 = np.expand_dims(test_image[idx1], axis=0)
img2 = np.expand_dims(test_image[idx2], axis=0)
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(10, 7))
ax1.imshow(np.squeeze(img1))
ax2.imshow(np.squeeze(img2))
for ax in [ax1, ax2]:
ax.grid(False)
ax.set_xticks([])
ax.set_yticks([])
model = Model(sequence_input, crf)
if model_choice == 2:
model = Model(sequence_input, preds)
model.summary()
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['acc'])
plot_model(model, to_file='model.png')
epoch = input('Enter number of epochs: ')
batch = input('Enter number of batch size: ')
model.fit(np.array(x_train.padded),
np.array(y_encoded),
epochs=epoch,
batch_size=batch)
"""
Evaluate
"""
mateval = []
for labr in range(label.cnt):
row = []
for labc in range(label.cnt):
row.append(0)
mateval.append(row)
x_test.pad(padsize)
results = []
print "Computing..."
# %% from keras import Input, layers from keras import Model input_tensor = Input(shape=(64,)) x = layers.Dense(32, activation='relu')(input_tensor) x = layers.Dense(32, activation='relu')(x) output_tensor = layers.Dense(10, activation='softmax')(x) model = Model(input_tensor, output_tensor) model.summary() # %% import numpy as np model.compile(optimizer='rmsprop', loss='categorical_crossentropy') x_train = np.random.random((1000, 64)) y_train = np.random.random((1000, 10)) model.fit(x_train, y_train, epochs=10, batch_size=128) score = model.evaluate(x_train, y_train)
