def resnet_model(trainFilePath,
testFilePath,
batch_size,
epochs,
name,
lr,
key_file,
which_line,
which_letter,
key_length,
load_weight=False,
weight_path=None,
evalONtest=True):
input_tensor = Input(shape=(35000, 1))
seq = input_tensor
seq = BLOCK(seq, 64)
seq = BatchNormalization(axis=1)(seq)
seq = MaxPooling1D(2)(seq)
seq = BLOCK(seq, 64)
seq = BatchNormalization(axis=1)(seq)
seq = MaxPooling1D(2)(seq)
seq = BLOCK(seq, 128)
seq = BatchNormalization(axis=1)(seq)
seq = MaxPooling1D(2)(seq)
seq = BLOCK(seq, 128)
seq = BatchNormalization(axis=1)(seq)
seq = MaxPooling1D(2)(seq)
seq = BLOCK(seq, 256)
seq = BatchNormalization(axis=1)(seq)
seq = MaxPooling1D(2)(seq)
seq = BLOCK(seq, 256)
seq = BatchNormalization(axis=1)(seq)
seq = MaxPooling1D(2)(seq)
seq = BLOCK(seq, 512)
seq = BatchNormalization(axis=1)(seq)
seq = MaxPooling1D(2)(seq)
seq = BLOCK(seq, 512)
seq = BatchNormalization(axis=1)(seq)
seq = Dropout(0.6)(seq)
seq = GlobalMaxPooling1D()(seq)
output_tensor = Dense(16, activation='softmax')(seq)
# model = Model(inputs=[input_tensor], outputs=[output_tensor])
model = Model(inputs=[input_tensor], outputs=[output_tensor])
model = multi_gpu_model(model, gpus=3)
model.summary()
if load_weight == True:
model.load_weights(weight_path, by_name=True)
else:
pass
from keras.optimizers import Adam
model.compile(
loss='categorical_crossentropy', # 交叉熵作为loss
optimizer=Adam(lr),
metrics=['accuracy'])
if evalONtest == True:
test_model = Model(inputs=[input_tensor], outputs=[output_tensor])
test_model.compile(
loss='categorical_crossentropy', # 交叉熵作为loss
optimizer=Adam(lr),
metrics=['accuracy'])
CSV_FILE_PATH2 = testFilePath
data_to_test = load_data(CSV_FILE_PATH2)
# train_x2, test_x2, train_y2, test_y2, Class_dict2
# train_x2 = np.expand_dims(train_x2, axis=2)
# test_x2 = np.expand_dims(test_x2, axis=2)
else:
pass
print('开始加载数据')
data_to_train = load_data(trainFilePath)
# data_to_train = data_to_train[:10000]
label_to_train = cut_letter(key_file, which_line, which_letter, key_length)
label_to_train_lb = to_categorical(label_to_train, 16)
train_x, test_x, train_y, test_y = train_test_split(data_to_train,
label_to_train_lb,
test_size=0.3,
shuffle=True)
train_x = np.expand_dims(train_x, axis=2)
test_x = np.expand_dims(test_x, axis=2)
b_size = batch_size
max_epochs = epochs
print("Starting training ")
learnratedecay = ReduceLROnPlateau(monitor='val_loss',
factor=0.1,
patience=8,
verbose=1,
mode='auto',
epsilon=0.0001,
cooldown=0,
min_lr=0)
os.makedirs('/data/wuchenxi/allmodel/new_simeck_model/' + name + '/model',
exist_ok=True)
os.makedirs('/data/wuchenxi/allmodel/new_simeck_model/' + name + '/csvlog',
exist_ok=True)
os.makedirs('/data/wuchenxi/allmodel/new_simeck_model/' + name +
'/tensorboard',
exist_ok=True)
checkpointer = ModelCheckpoint(
monitor='val_loss',
filepath='/data/wuchenxi/allmodel/new_simeck_model/' + name +
'/model/' + name + '.hdf5',
verbose=1,
save_best_only=True)
picture_output = TensorBoard(
log_dir='/data/wuchenxi/allmodel/new_simeck_model/' + name +
'/tensorboard/' + name + '_log',
histogram_freq=0,
write_graph=True,
write_grads=True,
write_images=True,
)
csvlog = CSVLogger(filename='/data/wuchenxi/allmodel/new_simeck_model/' +
name + '/csvlog/' + name + '.csv',
separator=',',
append=False)
if evalONtest == True:
pass
# callback = [checkpointer, picture_output, csvlog, learnratedecay,
# EvaluateInputTensor(test_model, train_x2, train_y2,
# '/data/wuchenxi/allmodel/simeck_key_model/'+name+'/csvlog/' + name + '_test.csv')]
else:
callback = [checkpointer, picture_output, csvlog, learnratedecay]
h = model.fit(train_x,
train_y,
batch_size=b_size,
epochs=max_epochs,
validation_data=(test_x, test_y),
shuffle=True,
verbose=1,
callbacks=callback)
def resnet_model(trainFilePath,
testFilePath,
batch_size,
epochs,
name,
lr,
key_file,
which_line,
which_letter,
key_length,
test_size,
use_add=False,
each_class_number=None,
choose_number=None,
load_weight=False,
weight_path=None,
evalONtest=True):
# input_tensor = Input(shape=(35000, 1))
# seq = input_tensor
# seq = BLOCK(seq, 64)
# seq = BatchNormalization(axis=1)(seq)
# seq = MaxPooling1D(2)(seq)
#
# seq = BLOCK(seq, 64)
# seq = BatchNormalization(axis=1)(seq)
# seq = MaxPooling1D(2)(seq)
#
# seq = BLOCK(seq, 128)
# seq = BatchNormalization(axis=1)(seq)
# seq = MaxPooling1D(2)(seq)
#
# seq = BLOCK(seq, 128)
# seq = BatchNormalization(axis=1)(seq)
# seq = MaxPooling1D(2)(seq)
#
# seq = BLOCK(seq, 256)
# seq = BatchNormalization(axis=1)(seq)
# seq = MaxPooling1D(2)(seq)
#
# seq = BLOCK(seq, 256)
# seq = BatchNormalization(axis=1)(seq)
# seq = MaxPooling1D(2)(seq)
#
# seq = BLOCK(seq, 512)
# seq = BatchNormalization(axis=1)(seq)
# seq = MaxPooling1D(2)(seq)
#
# seq = BLOCK(seq, 512)
# seq = BatchNormalization(axis=1)(seq)
#
# # seq = Dropout(0.6)(seq)
# seq = Dropout(0.1)(seq)
#
# seq = GlobalMaxPooling1D()(seq)
#
# output_tensor = Dense(16, activation='softmax')(seq)
# model = Model(inputs=[input_tensor], outputs=[output_tensor])
anchor_input = Input((
35000,
1,
), name='anchor_input')
positive_input = Input((
35000,
1,
), name='positive_input')
negative_input = Input((
35000,
1,
), name='negative_input')
Shared_DNN = create_base_network([35000, 1])
encoded_anchor = Shared_DNN(anchor_input)
encoded_positive = Shared_DNN(positive_input)
encoded_negative = Shared_DNN(negative_input)
merged_vector = concatenate(
[encoded_anchor, encoded_positive, encoded_negative],
axis=-1,
name='merged_layer')
model = Model(inputs=[anchor_input, positive_input, negative_input],
outputs=merged_vector)
# model = Model(inputs=[input_tensor], outputs=[output_tensor])
# model = multi_gpu_model(model,gpus=4)
model.summary()
if load_weight == True:
model.load_weights(weight_path, by_name=True)
else:
pass
from keras.optimizers import Adam
model.compile(
loss=triplet_loss, # 交叉熵作为loss
optimizer=Adam(lr))
# if evalONtest == True:
# test_model = Model(inputs=[input_tensor], outputs=[output_tensor])
#
# test_model.compile(loss='categorical_crossentropy', # 交叉熵作为loss
# optimizer=Adam(lr),
# metrics=['accuracy'])
# CSV_FILE_PATH2 = testFilePath
# data_to_test = load_data(CSV_FILE_PATH2)
#
#
# # train_x2, test_x2, train_y2, test_y2, Class_dict2
# # train_x2 = np.expand_dims(train_x2, axis=2)
# # test_x2 = np.expand_dims(test_x2, axis=2)
#
# else:
# pass
print('开始加载数据')
if use_add == True:
data_to_train, label_to_train_lb = add_all_class_in_mem_return_ori_and_add_data(
signal_data_path=trainFilePath,
label_path=key_file,
which_line=which_line,
which_letter=which_letter,
key_length=key_length,
each_class_number=each_class_number,
choose_number=choose_number)
else:
data_to_train = load_data(trainFilePath)
# data_to_train = data_to_train[:10000]
label_to_train = cut_letter(key_file, which_line, which_letter,
key_length)
label_to_train_lb = to_categorical(label_to_train, 16)
train_x, test_x, train_y, test_y = train_test_split(data_to_train,
label_to_train,
test_size=test_size,
shuffle=True)
train_x = np.expand_dims(train_x, axis=2)
test_x = np.expand_dims(test_x, axis=2)
X_train_triplet, X_test_triplet = generate_triplet(train_x,
train_y,
testsize=0.3,
ap_pairs=10,
an_pairs=10)
print(X_train_triplet.shape)
print(X_test_triplet.shape)
# print(X_train_triplet)
Anchor = X_train_triplet[:, 0, :, :].reshape(-1, 35000, 1)
Positive = X_train_triplet[:, 1, :, :].reshape(-1, 35000, 1)
Negative = X_train_triplet[:, 2, :, :].reshape(-1, 35000, 1)
Anchor_test = X_test_triplet[:, 0, :, :].reshape(-1, 35000, 1)
Positive_test = X_test_triplet[:, 1, :, :].reshape(-1, 35000, 1)
Negative_test = X_test_triplet[:, 2, :, :].reshape(-1, 35000, 1)
Y_dummy = np.empty((Anchor.shape[0], 300))
Y_dummy2 = np.empty((Anchor_test.shape[0], 1))
model.fit([Anchor, Positive, Negative],
y=Y_dummy,
validation_data=([Anchor_test, Positive_test,
Negative_test], Y_dummy2),
batch_size=512,
epochs=500)
print(Anchor.shape)
def resnet_model(trainFilePath,
testFilePath,
batch_size,
epochs,
name,
lr,
key_file,
which_line,
which_letter,
load_weight=False,
weight_path=None,
evalONtest=True,
validation_proportion=0.1,
test_num=100,
rm_already_folder=True):
input_tensor = Input(shape=(35000, 1))
seq = input_tensor
seq = BLOCK(seq, 64)
seq = BatchNormalization(axis=1)(seq)
seq = MaxPooling1D(2)(seq)
seq = BLOCK(seq, 64)
seq = BatchNormalization(axis=1)(seq)
seq = MaxPooling1D(2)(seq)
seq = BLOCK(seq, 128)
seq = BatchNormalization(axis=1)(seq)
seq = MaxPooling1D(2)(seq)
seq = BLOCK(seq, 128)
seq = BatchNormalization(axis=1)(seq)
seq = MaxPooling1D(2)(seq)
seq = BLOCK(seq, 256)
seq = BatchNormalization(axis=1)(seq)
seq = MaxPooling1D(2)(seq)
seq = BLOCK(seq, 256)
seq = BatchNormalization(axis=1)(seq)
share_bottom = MaxPooling1D(2)(seq)
seq0 = BLOCK(share_bottom, 512)
seq0 = BatchNormalization(axis=1)(seq0)
seq0 = MaxPooling1D(2)(seq0)
seq0 = BLOCK(seq0, 512)
seq0 = BatchNormalization(axis=1)(seq0)
# seq0 = Dropout(0.1)(seq0)
seq0 = GlobalMaxPooling1D()(seq0)
output_tensor0 = Dense(16, activation='softmax',
name='output_tensor0')(seq0)
seq1 = BLOCK(share_bottom, 512)
seq1 = BatchNormalization(axis=1)(seq1)
seq1 = MaxPooling1D(2)(seq1)
seq1 = BLOCK(seq1, 512)
seq1 = BatchNormalization(axis=1)(seq1)
# seq1 = Dropout(0.1)(seq1)
seq1 = GlobalMaxPooling1D()(seq1)
output_tensor1 = Dense(16, activation='softmax',
name='output_tensor1')(seq1)
seq2 = BLOCK(share_bottom, 512)
seq2 = BatchNormalization(axis=1)(seq2)
seq2 = MaxPooling1D(2)(seq2)
seq2 = BLOCK(seq2, 512)
seq2 = BatchNormalization(axis=1)(seq2)
# seq2 = Dropout(0.1)(seq2)
seq2 = GlobalMaxPooling1D()(seq2)
output_tensor2 = Dense(16, activation='softmax',
name='output_tensor2')(seq2)
seq3 = BLOCK(share_bottom, 512)
seq3 = BatchNormalization(axis=1)(seq3)
seq3 = MaxPooling1D(2)(seq3)
seq3 = BLOCK(seq3, 512)
seq3 = BatchNormalization(axis=1)(seq3)
# seq3 = Dropout(0.1)(seq3)
seq3 = GlobalMaxPooling1D()(seq3)
output_tensor3 = Dense(16, activation='softmax',
name='output_tensor3')(seq3)
model = Model(inputs=[input_tensor],
outputs=[
output_tensor0, output_tensor1, output_tensor2,
output_tensor3
])
model = multi_gpu_model(model, gpus=2)
model.summary()
if load_weight == True:
model.load_weights(weight_path, by_name=True)
else:
pass
from keras.optimizers import Adam
model.compile(
loss=[
'categorical_crossentropy', 'categorical_crossentropy',
'categorical_crossentropy', 'categorical_crossentropy'
], # 交叉熵作为loss
optimizer=Adam(lr),
metrics=['accuracy'])
if evalONtest == True:
test_model = Model(inputs=[input_tensor],
outputs=[
output_tensor0, output_tensor1, output_tensor2,
output_tensor3
])
test_model.compile(
loss='categorical_crossentropy', # 交叉熵作为loss
optimizer=Adam(lr),
metrics=['accuracy'])
CSV_FILE_PATH2 = testFilePath
data_to_test = load_data(CSV_FILE_PATH2)
# train_x2, test_x2, train_y2, test_y2, Class_dict2
# train_x2 = np.expand_dims(train_x2, axis=2)
# test_x2 = np.expand_dims(test_x2, axis=2)
else:
pass
print('开始加载数据')
b_size = batch_size
max_epochs = epochs
train_generater,val_generater,test_generater,train_data_num,val_data_num = \
get_three_generater(trainFilePath,key_file,which_line,which_letter,
validation_proportion=validation_proportion,
test_num=test_num,batch_size=b_size,rm_already_folder = rm_already_folder)
print("Starting training ")
learnratedecay = ReduceLROnPlateau(monitor='val_loss',
factor=0.1,
patience=8,
verbose=1,
mode='auto',
epsilon=0.0001,
cooldown=0,
min_lr=0)
os.makedirs('/data/wuchenxi/allmodel/new_simeck_model/' + name + '/model',
exist_ok=True)
os.makedirs('/data/wuchenxi/allmodel/new_simeck_model/' + name + '/csvlog',
exist_ok=True)
os.makedirs('/data/wuchenxi/allmodel/new_simeck_model/' + name +
'/tensorboard',
exist_ok=True)
checkpointer = ModelCheckpoint(
monitor='val_loss',
filepath='/data/wuchenxi/allmodel/new_simeck_model/' + name +
'/model/' + name + '.hdf5',
verbose=1,
save_best_only=True)
picture_output = TensorBoard(
log_dir='/data/wuchenxi/allmodel/new_simeck_model/' + name +
'/tensorboard/' + name + '_log',
histogram_freq=0,
write_graph=True,
write_grads=True,
write_images=True,
)
csvlog = CSVLogger(filename='/data/wuchenxi/allmodel/new_simeck_model/' +
name + '/csvlog/' + name + '.csv',
separator=',',
append=False)
if evalONtest == True:
pass
# callback = [checkpointer, picture_output, csvlog, learnratedecay,
# EvaluateInputTensor(test_model, train_x2, train_y2,
# '/data/wuchenxi/allmodel/simeck_key_model/'+name+'/csvlog/' + name + '_test.csv')]
else:
callback = [checkpointer, picture_output, csvlog, learnratedecay]
step_per_epoch_default = int(train_data_num / batch_size)
step_per_epoch_default_val = int(val_data_num / batch_size)
h = model.fit_generator(train_generater,
steps_per_epoch=step_per_epoch_default,
epochs=max_epochs,
verbose=1,
callbacks=callback,
validation_data=val_generater,
validation_steps=step_per_epoch_default_val)
)
# /data/wuchenxi/allmodel/new_simeck_model/15000_1_1_v1/model/15000_1_1_v1.hdf5
# /data/wuchenxi/allmodel/new_simeck_model/54080_1_1_v1/model/54080_1_1_v1.hdf5
model.summary()
print('开始加载数据')
# data_to_train,label_to_train_lb = add_all_class_in_mem_return_ori_and_add_data(signal_data_path='/data/wuchenxi/new_simeck_data/signal54400_circle/signal_320_circle/',
# label_path='/data/wuchenxi/new_simeck_data/signal54400_circle/new_simeck_320.txt',
# which_line=0,
# which_letter=0,
# key_length=4*320,
# each_class_number=10,
# choose_number=2)
data_to_train = load_data(
'/data/wuchenxi/new_simeck_data/signal54400_circle/signal_320_circle/')
# data_to_train = data_to_train[10000:13000]
label_to_train = cut_letter(
'/data/wuchenxi/new_simeck_data/signal54400_circle/new_simeck_320.txt', 0,
0, 4 * 320)
label_to_train_lb = to_categorical(label_to_train, 16)
data_to_train = np.expand_dims(data_to_train, axis=2)
eval2 = model.evaluate(
data_to_train,
label_to_train_lb,
verbose=1,
)
"""