def resnet_model(trainFilePath, testFilePath, batch_size, epochs, name, lr, key_file, which_line, which_letter, key_length, load_weight=False, weight_path=None, evalONtest=True): input_tensor = Input(shape=(35000, 1)) seq = input_tensor seq = BLOCK(seq, 64) seq = BatchNormalization(axis=1)(seq) seq = MaxPooling1D(2)(seq) seq = BLOCK(seq, 64) seq = BatchNormalization(axis=1)(seq) seq = MaxPooling1D(2)(seq) seq = BLOCK(seq, 128) seq = BatchNormalization(axis=1)(seq) seq = MaxPooling1D(2)(seq) seq = BLOCK(seq, 128) seq = BatchNormalization(axis=1)(seq) seq = MaxPooling1D(2)(seq) seq = BLOCK(seq, 256) seq = BatchNormalization(axis=1)(seq) seq = MaxPooling1D(2)(seq) seq = BLOCK(seq, 256) seq = BatchNormalization(axis=1)(seq) seq = MaxPooling1D(2)(seq) seq = BLOCK(seq, 512) seq = BatchNormalization(axis=1)(seq) seq = MaxPooling1D(2)(seq) seq = BLOCK(seq, 512) seq = BatchNormalization(axis=1)(seq) seq = Dropout(0.6)(seq) seq = GlobalMaxPooling1D()(seq) output_tensor = Dense(16, activation='softmax')(seq) # model = Model(inputs=[input_tensor], outputs=[output_tensor]) model = Model(inputs=[input_tensor], outputs=[output_tensor]) model = multi_gpu_model(model, gpus=3) model.summary() if load_weight == True: model.load_weights(weight_path, by_name=True) else: pass from keras.optimizers import Adam model.compile( loss='categorical_crossentropy', # 交叉熵作为loss optimizer=Adam(lr), metrics=['accuracy']) if evalONtest == True: test_model = Model(inputs=[input_tensor], outputs=[output_tensor]) test_model.compile( loss='categorical_crossentropy', # 交叉熵作为loss optimizer=Adam(lr), metrics=['accuracy']) CSV_FILE_PATH2 = testFilePath data_to_test = load_data(CSV_FILE_PATH2) # train_x2, test_x2, train_y2, test_y2, Class_dict2 # train_x2 = np.expand_dims(train_x2, axis=2) # test_x2 = np.expand_dims(test_x2, axis=2) else: pass print('开始加载数据') data_to_train = load_data(trainFilePath) # data_to_train = data_to_train[:10000] label_to_train = cut_letter(key_file, which_line, which_letter, key_length) label_to_train_lb = to_categorical(label_to_train, 16) train_x, test_x, train_y, test_y = train_test_split(data_to_train, label_to_train_lb, test_size=0.3, shuffle=True) train_x = np.expand_dims(train_x, axis=2) test_x = np.expand_dims(test_x, axis=2) b_size = batch_size max_epochs = epochs print("Starting training ") learnratedecay = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=8, verbose=1, mode='auto', epsilon=0.0001, cooldown=0, min_lr=0) os.makedirs('/data/wuchenxi/allmodel/new_simeck_model/' + name + '/model', exist_ok=True) os.makedirs('/data/wuchenxi/allmodel/new_simeck_model/' + name + '/csvlog', exist_ok=True) os.makedirs('/data/wuchenxi/allmodel/new_simeck_model/' + name + '/tensorboard', exist_ok=True) checkpointer = ModelCheckpoint( monitor='val_loss', filepath='/data/wuchenxi/allmodel/new_simeck_model/' + name + '/model/' + name + '.hdf5', verbose=1, save_best_only=True) picture_output = TensorBoard( log_dir='/data/wuchenxi/allmodel/new_simeck_model/' + name + '/tensorboard/' + name + '_log', histogram_freq=0, write_graph=True, write_grads=True, write_images=True, ) csvlog = CSVLogger(filename='/data/wuchenxi/allmodel/new_simeck_model/' + name + '/csvlog/' + name + '.csv', separator=',', append=False) if evalONtest == True: pass # callback = [checkpointer, picture_output, csvlog, learnratedecay, # EvaluateInputTensor(test_model, train_x2, train_y2, # '/data/wuchenxi/allmodel/simeck_key_model/'+name+'/csvlog/' + name + '_test.csv')] else: callback = [checkpointer, picture_output, csvlog, learnratedecay] h = model.fit(train_x, train_y, batch_size=b_size, epochs=max_epochs, validation_data=(test_x, test_y), shuffle=True, verbose=1, callbacks=callback)
def resnet_model(trainFilePath, testFilePath, batch_size, epochs, name, lr, key_file, which_line, which_letter, key_length, test_size, use_add=False, each_class_number=None, choose_number=None, load_weight=False, weight_path=None, evalONtest=True): # input_tensor = Input(shape=(35000, 1)) # seq = input_tensor # seq = BLOCK(seq, 64) # seq = BatchNormalization(axis=1)(seq) # seq = MaxPooling1D(2)(seq) # # seq = BLOCK(seq, 64) # seq = BatchNormalization(axis=1)(seq) # seq = MaxPooling1D(2)(seq) # # seq = BLOCK(seq, 128) # seq = BatchNormalization(axis=1)(seq) # seq = MaxPooling1D(2)(seq) # # seq = BLOCK(seq, 128) # seq = BatchNormalization(axis=1)(seq) # seq = MaxPooling1D(2)(seq) # # seq = BLOCK(seq, 256) # seq = BatchNormalization(axis=1)(seq) # seq = MaxPooling1D(2)(seq) # # seq = BLOCK(seq, 256) # seq = BatchNormalization(axis=1)(seq) # seq = MaxPooling1D(2)(seq) # # seq = BLOCK(seq, 512) # seq = BatchNormalization(axis=1)(seq) # seq = MaxPooling1D(2)(seq) # # seq = BLOCK(seq, 512) # seq = BatchNormalization(axis=1)(seq) # # # seq = Dropout(0.6)(seq) # seq = Dropout(0.1)(seq) # # seq = GlobalMaxPooling1D()(seq) # # output_tensor = Dense(16, activation='softmax')(seq) # model = Model(inputs=[input_tensor], outputs=[output_tensor]) anchor_input = Input(( 35000, 1, ), name='anchor_input') positive_input = Input(( 35000, 1, ), name='positive_input') negative_input = Input(( 35000, 1, ), name='negative_input') Shared_DNN = create_base_network([35000, 1]) encoded_anchor = Shared_DNN(anchor_input) encoded_positive = Shared_DNN(positive_input) encoded_negative = Shared_DNN(negative_input) merged_vector = concatenate( [encoded_anchor, encoded_positive, encoded_negative], axis=-1, name='merged_layer') model = Model(inputs=[anchor_input, positive_input, negative_input], outputs=merged_vector) # model = Model(inputs=[input_tensor], outputs=[output_tensor]) # model = multi_gpu_model(model,gpus=4) model.summary() if load_weight == True: model.load_weights(weight_path, by_name=True) else: pass from keras.optimizers import Adam model.compile( loss=triplet_loss, # 交叉熵作为loss optimizer=Adam(lr)) # if evalONtest == True: # test_model = Model(inputs=[input_tensor], outputs=[output_tensor]) # # test_model.compile(loss='categorical_crossentropy', # 交叉熵作为loss # optimizer=Adam(lr), # metrics=['accuracy']) # CSV_FILE_PATH2 = testFilePath # data_to_test = load_data(CSV_FILE_PATH2) # # # # train_x2, test_x2, train_y2, test_y2, Class_dict2 # # train_x2 = np.expand_dims(train_x2, axis=2) # # test_x2 = np.expand_dims(test_x2, axis=2) # # else: # pass print('开始加载数据') if use_add == True: data_to_train, label_to_train_lb = add_all_class_in_mem_return_ori_and_add_data( signal_data_path=trainFilePath, label_path=key_file, which_line=which_line, which_letter=which_letter, key_length=key_length, each_class_number=each_class_number, choose_number=choose_number) else: data_to_train = load_data(trainFilePath) # data_to_train = data_to_train[:10000] label_to_train = cut_letter(key_file, which_line, which_letter, key_length) label_to_train_lb = to_categorical(label_to_train, 16) train_x, test_x, train_y, test_y = train_test_split(data_to_train, label_to_train, test_size=test_size, shuffle=True) train_x = np.expand_dims(train_x, axis=2) test_x = np.expand_dims(test_x, axis=2) X_train_triplet, X_test_triplet = generate_triplet(train_x, train_y, testsize=0.3, ap_pairs=10, an_pairs=10) print(X_train_triplet.shape) print(X_test_triplet.shape) # print(X_train_triplet) Anchor = X_train_triplet[:, 0, :, :].reshape(-1, 35000, 1) Positive = X_train_triplet[:, 1, :, :].reshape(-1, 35000, 1) Negative = X_train_triplet[:, 2, :, :].reshape(-1, 35000, 1) Anchor_test = X_test_triplet[:, 0, :, :].reshape(-1, 35000, 1) Positive_test = X_test_triplet[:, 1, :, :].reshape(-1, 35000, 1) Negative_test = X_test_triplet[:, 2, :, :].reshape(-1, 35000, 1) Y_dummy = np.empty((Anchor.shape[0], 300)) Y_dummy2 = np.empty((Anchor_test.shape[0], 1)) model.fit([Anchor, Positive, Negative], y=Y_dummy, validation_data=([Anchor_test, Positive_test, Negative_test], Y_dummy2), batch_size=512, epochs=500) print(Anchor.shape)
def resnet_model(trainFilePath, testFilePath, batch_size, epochs, name, lr, key_file, which_line, which_letter, load_weight=False, weight_path=None, evalONtest=True, validation_proportion=0.1, test_num=100, rm_already_folder=True): input_tensor = Input(shape=(35000, 1)) seq = input_tensor seq = BLOCK(seq, 64) seq = BatchNormalization(axis=1)(seq) seq = MaxPooling1D(2)(seq) seq = BLOCK(seq, 64) seq = BatchNormalization(axis=1)(seq) seq = MaxPooling1D(2)(seq) seq = BLOCK(seq, 128) seq = BatchNormalization(axis=1)(seq) seq = MaxPooling1D(2)(seq) seq = BLOCK(seq, 128) seq = BatchNormalization(axis=1)(seq) seq = MaxPooling1D(2)(seq) seq = BLOCK(seq, 256) seq = BatchNormalization(axis=1)(seq) seq = MaxPooling1D(2)(seq) seq = BLOCK(seq, 256) seq = BatchNormalization(axis=1)(seq) share_bottom = MaxPooling1D(2)(seq) seq0 = BLOCK(share_bottom, 512) seq0 = BatchNormalization(axis=1)(seq0) seq0 = MaxPooling1D(2)(seq0) seq0 = BLOCK(seq0, 512) seq0 = BatchNormalization(axis=1)(seq0) # seq0 = Dropout(0.1)(seq0) seq0 = GlobalMaxPooling1D()(seq0) output_tensor0 = Dense(16, activation='softmax', name='output_tensor0')(seq0) seq1 = BLOCK(share_bottom, 512) seq1 = BatchNormalization(axis=1)(seq1) seq1 = MaxPooling1D(2)(seq1) seq1 = BLOCK(seq1, 512) seq1 = BatchNormalization(axis=1)(seq1) # seq1 = Dropout(0.1)(seq1) seq1 = GlobalMaxPooling1D()(seq1) output_tensor1 = Dense(16, activation='softmax', name='output_tensor1')(seq1) seq2 = BLOCK(share_bottom, 512) seq2 = BatchNormalization(axis=1)(seq2) seq2 = MaxPooling1D(2)(seq2) seq2 = BLOCK(seq2, 512) seq2 = BatchNormalization(axis=1)(seq2) # seq2 = Dropout(0.1)(seq2) seq2 = GlobalMaxPooling1D()(seq2) output_tensor2 = Dense(16, activation='softmax', name='output_tensor2')(seq2) seq3 = BLOCK(share_bottom, 512) seq3 = BatchNormalization(axis=1)(seq3) seq3 = MaxPooling1D(2)(seq3) seq3 = BLOCK(seq3, 512) seq3 = BatchNormalization(axis=1)(seq3) # seq3 = Dropout(0.1)(seq3) seq3 = GlobalMaxPooling1D()(seq3) output_tensor3 = Dense(16, activation='softmax', name='output_tensor3')(seq3) model = Model(inputs=[input_tensor], outputs=[ output_tensor0, output_tensor1, output_tensor2, output_tensor3 ]) model = multi_gpu_model(model, gpus=2) model.summary() if load_weight == True: model.load_weights(weight_path, by_name=True) else: pass from keras.optimizers import Adam model.compile( loss=[ 'categorical_crossentropy', 'categorical_crossentropy', 'categorical_crossentropy', 'categorical_crossentropy' ], # 交叉熵作为loss optimizer=Adam(lr), metrics=['accuracy']) if evalONtest == True: test_model = Model(inputs=[input_tensor], outputs=[ output_tensor0, output_tensor1, output_tensor2, output_tensor3 ]) test_model.compile( loss='categorical_crossentropy', # 交叉熵作为loss optimizer=Adam(lr), metrics=['accuracy']) CSV_FILE_PATH2 = testFilePath data_to_test = load_data(CSV_FILE_PATH2) # train_x2, test_x2, train_y2, test_y2, Class_dict2 # train_x2 = np.expand_dims(train_x2, axis=2) # test_x2 = np.expand_dims(test_x2, axis=2) else: pass print('开始加载数据') b_size = batch_size max_epochs = epochs train_generater,val_generater,test_generater,train_data_num,val_data_num = \ get_three_generater(trainFilePath,key_file,which_line,which_letter, validation_proportion=validation_proportion, test_num=test_num,batch_size=b_size,rm_already_folder = rm_already_folder) print("Starting training ") learnratedecay = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=8, verbose=1, mode='auto', epsilon=0.0001, cooldown=0, min_lr=0) os.makedirs('/data/wuchenxi/allmodel/new_simeck_model/' + name + '/model', exist_ok=True) os.makedirs('/data/wuchenxi/allmodel/new_simeck_model/' + name + '/csvlog', exist_ok=True) os.makedirs('/data/wuchenxi/allmodel/new_simeck_model/' + name + '/tensorboard', exist_ok=True) checkpointer = ModelCheckpoint( monitor='val_loss', filepath='/data/wuchenxi/allmodel/new_simeck_model/' + name + '/model/' + name + '.hdf5', verbose=1, save_best_only=True) picture_output = TensorBoard( log_dir='/data/wuchenxi/allmodel/new_simeck_model/' + name + '/tensorboard/' + name + '_log', histogram_freq=0, write_graph=True, write_grads=True, write_images=True, ) csvlog = CSVLogger(filename='/data/wuchenxi/allmodel/new_simeck_model/' + name + '/csvlog/' + name + '.csv', separator=',', append=False) if evalONtest == True: pass # callback = [checkpointer, picture_output, csvlog, learnratedecay, # EvaluateInputTensor(test_model, train_x2, train_y2, # '/data/wuchenxi/allmodel/simeck_key_model/'+name+'/csvlog/' + name + '_test.csv')] else: callback = [checkpointer, picture_output, csvlog, learnratedecay] step_per_epoch_default = int(train_data_num / batch_size) step_per_epoch_default_val = int(val_data_num / batch_size) h = model.fit_generator(train_generater, steps_per_epoch=step_per_epoch_default, epochs=max_epochs, verbose=1, callbacks=callback, validation_data=val_generater, validation_steps=step_per_epoch_default_val)
) # /data/wuchenxi/allmodel/new_simeck_model/15000_1_1_v1/model/15000_1_1_v1.hdf5 # /data/wuchenxi/allmodel/new_simeck_model/54080_1_1_v1/model/54080_1_1_v1.hdf5 model.summary() print('开始加载数据') # data_to_train,label_to_train_lb = add_all_class_in_mem_return_ori_and_add_data(signal_data_path='/data/wuchenxi/new_simeck_data/signal54400_circle/signal_320_circle/', # label_path='/data/wuchenxi/new_simeck_data/signal54400_circle/new_simeck_320.txt', # which_line=0, # which_letter=0, # key_length=4*320, # each_class_number=10, # choose_number=2) data_to_train = load_data( '/data/wuchenxi/new_simeck_data/signal54400_circle/signal_320_circle/') # data_to_train = data_to_train[10000:13000] label_to_train = cut_letter( '/data/wuchenxi/new_simeck_data/signal54400_circle/new_simeck_320.txt', 0, 0, 4 * 320) label_to_train_lb = to_categorical(label_to_train, 16) data_to_train = np.expand_dims(data_to_train, axis=2) eval2 = model.evaluate( data_to_train, label_to_train_lb, verbose=1, ) """