def train_model(layers, path, des,model_num): # Load data x_train, y_train, x_val, y_val = load_data_with_validation_sets(path) # Create directories for saving model and csv files try: os.mkdir(f'{des}models') except FileExistsError: pass try: os.mkdir(f'{des}metrics') except FileExistsError: pass # Set callbacks es = EarlyStopping(monitor = 'val_PR', mode='max', verbose=1, patience=30) mc = ModelCheckpoint( filepath=f'{des}/models/model_{model_num}.h5', monitor='val_PR', mode='max', verbose=1, save_weights_only=False, save_best_only=True ) # Compile and train model model = build_model(layers, x_train.shape[1]) history = model.fit(x_train, y_train, epochs=5, batch_size=16, validation_data=(x_val, y_val), callbacks = [es,mc], verbose = 1) # Save training metrics pr = max(history.history['val_PR']) print(f'Max precision recall for {layers} layer(s): {pr}') h = pd.DataFrame(history.history) h.to_csv(f'{des}metrics/{pr}_max_model_{model_num}_results.csv') clear_session()
def fitness(learning_rate, num_lstm_nodes, dropout, batch_size): # Setting seed and clearing model graphs in backend tf.random.set_seed(SimpleCNN.seed_num) K.clear_session() tf.compat.v1.reset_default_graph() # Initializing model, compiling & training it model = self.generate_model(use_optimised_hyperparameters=False, dropout=dropout, number_of_lstm_nodes=num_lstm_nodes) optimizer = SGD(learning_rate=learning_rate) model.compile(loss='mean_squared_error', optimizer=optimizer) model.fit(train_x, train_y, batch_size=batch_size, epochs=model_epochs, verbose=2, shuffle=False, validation_data=(val_x, val_y)) # Generating prediction on Validation data validation_data_prediction = model.predict(val_x) # Calculating MSE for the model trained with candidate Hyperparameters of this iteration mse_validation = mean_squared_error(val_y, validation_data_prediction) # Deleting created model del model return mse_validation
def reset_keras(): """Reset Keras session.""" sess = get_session() clear_session() sess.close() gc.collect() # Set to force CPU calculations os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
def gru_keras(max_features, maxlen, bidirectional, dropout_rate, embed_dim, rec_units, mtype='GRU', reduction=None, classes=4, lr=0.001): if K.backend == 'tensorflow': K.clear_session() input_layer = Input(shape=(maxlen, )) embedding_layer = Embedding(max_features, output_dim=embed_dim, trainable=True)(input_layer) x = SpatialDropout1D(dropout_rate)(embedding_layer) if reduction: if mtype == 'GRU': if bidirectional: x = Bidirectional( CuDNNGRU(units=rec_units, return_sequences=True))(x) else: x = CuDNNGRU(units=rec_units, return_sequences=True)(x) elif mtype == 'LSTM': if bidirectional: x = Bidirectional( CuDNNLSTM(units=rec_units, return_sequences=True))(x) else: x = CuDNNLSTM(units=rec_units, return_sequences=True)(x) if reduction == 'average': x = GlobalAveragePooling1D()(x) elif reduction == 'maximum': x = GlobalMaxPool1D()(x) else: if mtype == 'GRU': if bidirectional: x = Bidirectional( CuDNNGRU(units=rec_units, return_sequences=False))(x) else: x = CuDNNGRU(units=rec_units, return_sequences=False)(x) elif mtype == 'LSTM': if bidirectional: x = Bidirectional( CuDNNLSTM(units=rec_units, return_sequences=False))(x) else: x = CuDNNLSTM(units=rec_units, return_sequences=False)(x) output_layer = Dense(classes, activation="sigmoid")(x) model = Model(inputs=input_layer, outputs=output_layer) model.compile(loss='categorical_crossentropy', optimizer=RMSprop(learning_rate=lr, clipvalue=1, clipnorm=1), metrics=['acc']) return model
def clear_model_memory(self): if self.histogram_bin == 'random': del self.model_list else: del self.fixed_bin_model backend.clear_session() gc.collect() gc.collect()
def reset_tf_session(): curr_session = tf.get_default_session() # close current session if curr_session is not None: curr_session.close() # reset graph K.clear_session() # create new session config = tf.ConfigProto() config.gpu_options.allow_growth = True s = tf.InteractiveSession(config=config) K.set_session(s) return s
def __train(args): benchmark_index, model_index, x_train, x_support, x_test, y_train, y_support, _, y_train_value, y_support_value, y_test_value = args __setup_tf_config(model_index) print("Setting up Model " + str(model_index + 1) + "/" + str(CONFIG["num_models"])) input = Input(shape=CONFIG["input_shape"]) output = build_fsl_attention(input) model = Model(inputs=input, outputs=output) model.compile( optimizer=Adam( # lr=0.0005 # デフォルトは0.001 ), loss=center_loss, ) expanded_y_train = np.array([ np.concatenate( [y, np.full( CONFIG["output_dim"] - y_train.shape[1], 0.0, )]) for y in y_train ]) histories = Histories( x_train, y_train_value, x_test, y_test_value, x_support, y_support_value, benchmark_index, model_index, ) model.fit( x_train, expanded_y_train, batch_size=CONFIG["batch_size"], epochs=CONFIG["epochs"], verbose=False, callbacks=[histories], shuffle=CONFIG["shuffle"], ) K.clear_session()
def reset_keras(self): sess = get_session() clear_session() sess.close() sess = get_session() try: del classifier except: pass print(gc.collect()) # use the same config as you used to create the session config = ConfigProto() config.gpu_options.per_process_gpu_memory_fraction = 0.9 config.gpu_options.allow_growth = True set_session(InteractiveSession(config=config))
def reset_keras(): sess = get_session() clear_session() sess.close() sess = get_session() try: del classifier # this is from global space - change this as you need except: pass # if it's done something you should see a number being outputted print(gc.collect()) # use the same config as you used to create the session config = tensorflow.compat.v1.ConfigProto() config.gpu_options.per_process_gpu_memory_fraction = 1 config.gpu_options.visible_device_list = "0" set_session(tensorflow.compat.v1.Session(config=config))
def set_session_config(per_process_gpu_memory_fraction=None, allow_growth=None, device_list='0'): """ :param allow_growth: When necessary, reserve memory :param float per_process_gpu_memory_fraction: specify GPU memory usage as 0 to 1 :return: """ import tensorflow as tf import tensorflow.compat.v1.keras.backend as K K.clear_session() config = tf.compat.v1.ConfigProto(gpu_options=tf.compat.v1.GPUOptions( per_process_gpu_memory_fraction=per_process_gpu_memory_fraction, allow_growth=allow_growth, visible_device_list=device_list)) sess = tf.compat.v1.Session(config=config) K.set_session(sess)
def cnn_keras(max_features, maxlen, dropout_rate, embed_dim, num_filters=300, classes=4, lr=0.001): if K.backend == 'tensorflow': K.clear_session() input_layer = Input(shape=(maxlen, )) embedding_layer = Embedding(max_features, output_dim=embed_dim, trainable=True)(input_layer) x = SpatialDropout1D(dropout_rate)(embedding_layer) x = Conv1D(num_filters, 7, activation='relu', padding='same')(x) x = GlobalMaxPooling1D()(x) output_layer = Dense(classes, activation="sigmoid")(x) model = Model(inputs=input_layer, outputs=output_layer) model.compile(loss='categorical_crossentropy', optimizer=RMSprop(learning_rate=lr, clipvalue=1, clipnorm=1), metrics=['acc']) return model
def test_arch(layers, node, path, des): # Iteratively train different architectures # Load data x_train, y_train, x_val, y_val = load_data_with_validation_sets(path) for l in range(layers): # Loop through layer range if l > 0: l = l + 1 # Minimum number of layers is 2 for n in node: # Loop though user input node options print( f'Testing precision recall of {l} layer(s) with {n} nodes...' ) print(f'With early stopping patience at 30...') # Create model es = EarlyStopping(monitor='val_PR', mode='max', verbose=1, patience=30) nodes = make_nodes(l, n) model = build_model(nodes, x_train.shape[1]) history = model.fit(x_train, y_train, epochs=1000, batch_size=16, validation_data=(x_val, y_val), callbacks=[es], verbose=1) # Save training metrics to a csv pr = max(history.history['val_PR']) print( f'Max precision recall for {l} layer(s) with {n} nodes: {pr}' ) h = pd.DataFrame(history.history) h.to_csv(f'{des}{pr}_max_{l}_layers_{n}_nodes_results.csv') clear_session()
def get_model_memory_usage(net_list, batch_size, input_shape, target_shape, return_dict): try: model = dag_2_cnn(cgp_2_dag(net_list), 0, input_shape, target_shape, compile=False) except (tf.errors.ResourceExhaustedError, KeyError) as e: print(e) return_dict["memory"] = 1000 return shapes_mem_count = 0 internal_model_mem_count = 0 for l in model.layers: single_layer_mem = 1 out_shape = l.output_shape if type(out_shape) is list: out_shape = out_shape[0] for s in out_shape: if s is None: continue single_layer_mem *= s shapes_mem_count += single_layer_mem trainable_count = np.sum([K.count_params(p) for p in model.trainable_weights]) non_trainable_count = np.sum([K.count_params(p) for p in model.non_trainable_weights]) number_size = 4.0 if K.floatx() == 'float16': number_size = 2.0 if K.floatx() == 'float64': number_size = 8.0 total_memory = number_size * (batch_size * shapes_mem_count + trainable_count + non_trainable_count) gbytes = np.round(total_memory / (1024.0 ** 3), 3) + internal_model_mem_count K.clear_session() return_dict["memory"] = gbytes
def create_model(input_shape, anchors, num_classes, load_pretrained=True, freeze_body=2, weights_path='model_data/yolo_weights.h5'): """create the training model""" K.clear_session() # get a new session image_input = Input(shape=(None, None, 3)) h, w = input_shape num_anchors = len(anchors) y_true = Input(shape=(h // grid_size_multiplier, w // grid_size_multiplier, anchors_per_level, num_classes + 5 + NUM_ANGLES3)) model_body = yolo_body(image_input, anchors_per_level, num_classes) print('Create Poly-YOLO model with {} anchors and {} classes.'.format( num_anchors, num_classes)) if load_pretrained: model_body.load_weights(weights_path, by_name=True, skip_mismatch=True) print('Load weights {}.'.format(weights_path)) model_loss = Lambda(yolo_loss, output_shape=(1, ), name='yolo_loss', arguments={ 'anchors': anchors, 'num_classes': num_classes, 'ignore_thresh': 0.5 })([model_body.output, y_true]) model = Model([model_body.input, y_true], model_loss) # print(model.summary()) return model
def run_single(mode, config, word_embedding, cognitive_data, cognitive_parent, modality, feature, truncate_first_line, gpu_id): ''' Takes a configuration dictionary and keys for a word embedding and cognitive data source, runs model, logs results and prepares output for plotting. :param mode: Type of embeddings, either 'proper' or 'random' :param config: Configuration dictionary :param word_embedding: String specifying word embedding (configuration key) :param cognitive_data: String specifying cognitiv data source (configuration key) :param feature: Cognitive data feature to be predicted :param truncate_first_line: If the first line of the embedding file should be truncated (when containing meta data) :param gpu_ids: IDs of available GPUs ''' # Tensorflow configuration import tensorflow as tf from tensorflow.compat.v1.keras.backend import set_session, clear_session if gpu_id is not None: gpu_count = 1 soft_placement = True else: gpu_count = 0 soft_placement = False tf_config = tf.compat.v1.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=1, allow_soft_placement=soft_placement, device_count={ 'GPU': gpu_count, 'CPU': 1 }) if gpu_id is not None: tf_config.gpu_options.allow_growth = True tf_config.gpu_options.per_process_gpu_memory_fraction = 0.25 tf_config.gpu_options.visible_device_list = str(gpu_id) session = tf.compat.v1.Session(config=tf_config) set_session(session) ############################################################################## # Create logging information ############################################################################## logging = {"folds": []} logging["wordEmbedding"] = word_embedding logging["cognitiveData"] = cognitive_data logging["cognitiveParent"] = cognitive_parent logging["modality"] = modality logging["feature"] = feature ############################################################################## # Run model ############################################################################## startTime = datetime.now() word_error, grids_result, mserrors = handler(mode, config, word_embedding, cognitive_data, feature, truncate_first_line) history = {'loss': [], 'val_loss': []} loss_list = [] val_loss_list = [] ############################################################################## # logging results ############################################################################## for i in range(len(grids_result)): fold = {} logging['folds'].append(fold) # BEST PARAMS APPENDING for key in grids_result[i].best_params_: logging['folds'][i][ key.upper()] = grids_result[i].best_params_[key] if config['cogDataConfig'][cognitive_data][ 'type'] == "multivariate_output": logging['folds'][i]['MSE_PREDICTION_ALL_DIM'] = list(mserrors[i]) logging['folds'][i]['MSE_PREDICTION'] = np.mean(mserrors[i]) elif config['cogDataConfig'][cognitive_data][ 'type'] == "single_output": logging['folds'][i]['MSE_PREDICTION'] = mserrors[i] logging['folds'][i]['LOSS'] = grids_result[ i].best_estimator_.model.history.history['loss'] logging['folds'][i]['VALIDATION_LOSS'] = grids_result[ i].best_estimator_.model.history.history['val_loss'] loss_list.append( np.array( grids_result[i].best_estimator_.model.history.history['loss'], dtype='float')) val_loss_list.append( np.array(grids_result[i].best_estimator_.model.history. history['val_loss'], dtype='float')) if config['cogDataConfig'][cognitive_data][ 'type'] == "multivariate_output": mserrors = np.array(mserrors, dtype='float') mse = np.mean(mserrors, axis=0) logging['AVERAGE_MSE_ALL_DIM'] = list(mse) logging['AVERAGE_MSE'] = np.mean(mse) elif config['cogDataConfig'][cognitive_data]['type'] == "single_output": mse = np.array(mserrors, dtype='float').mean() logging['AVERAGE_MSE'] = mse ############################################################################## # Prepare results for plot ############################################################################## history['loss'] = np.mean([loss_list[i] for i in range(len(loss_list))], axis=0) history['val_loss'] = np.mean( [val_loss_list[i] for i in range(len(val_loss_list))], axis=0) timeTaken = datetime.now() - startTime logging["timeTaken"] = str(timeTaken) # Clean-up tf session session.close() clear_session() return word_embedding, logging, word_error, history
def run(file_name, n_samples, p_n, q_n, activation="relu", cifar=False, tinyimagenet=False): np.random.seed(1215) tf.random.set_seed(1215) random.seed(1215) keras_model = load_model(file_name, custom_objects={"fn": fn, "tf": tf}) if tinyimagenet: model = CNNModel(keras_model, inp_shape=(64, 64, 3)) elif cifar: model = CNNModel(keras_model, inp_shape=(32, 32, 3)) else: model = CNNModel(keras_model) # Set correct linear_bounds function global linear_bounds if activation == "relu": linear_bounds = relu_linear_bounds elif activation == "ada": linear_bounds = ada_linear_bounds elif activation == "sigmoid": linear_bounds = sigmoid_linear_bounds elif activation == "tanh": linear_bounds = tanh_linear_bounds elif activation == "arctan": linear_bounds = atan_linear_bounds upper_bound_conv.recompile() lower_bound_conv.recompile() compute_bounds.recompile() if cifar: inputs, targets, true_labels, true_ids, img_info = generate_data( CIFAR(), samples=n_samples, targeted=True, random_and_least_likely=True, target_type=0b0010, predictor=model.model.predict, start=0, ) elif tinyimagenet: inputs, targets, true_labels, true_ids, img_info = generate_data( tinyImagenet(), samples=n_samples, targeted=True, random_and_least_likely=True, target_type=0b0010, predictor=model.model.predict, start=0, ) else: inputs, targets, true_labels, true_ids, img_info = generate_data( MNIST(), samples=n_samples, targeted=True, random_and_least_likely=True, target_type=0b0010, predictor=model.model.predict, start=0, ) # 0b01111 <- all # 0b0010 <- random # 0b0001 <- top2 # 0b0100 <- least steps = 15 eps_0 = 0.05 summation = 0 # warmup(model, inputs[0].astype(np.float32), eps_0, p_n, find_output_bounds) start_time = time.time() for i in range(len(inputs)): print("--- CNN-Cert: Computing eps for input image " + str(i) + "---") predict_label = np.argmax(true_labels[i]) target_label = np.argmax(targets[i]) weights = model.weights[:-1] biases = model.biases[:-1] shapes = model.shapes[:-1] W, b, s = model.weights[-1], model.biases[-1], model.shapes[-1] last_weight = (W[predict_label, :, :, :] - W[target_label, :, :, :]).reshape([1] + list(W.shape[1:])) weights.append(last_weight) biases.append(np.asarray([b[predict_label] - b[target_label]])) shapes.append((1, 1, 1)) # Perform binary search log_eps = np.log(eps_0) log_eps_min = -np.inf log_eps_max = np.inf for j in range(steps): LB, UB = find_output_bounds( weights, biases, shapes, model.pads, model.strides, inputs[i].astype(np.float32), np.exp(log_eps), p_n, ) print( "Step {}, eps = {:.5f}, {:.6s} <= f_c - f_t <= {:.6s}".format( j, np.exp(log_eps), str(np.squeeze(LB)), str(np.squeeze(UB)))) if LB > 0: # Increase eps log_eps_min = log_eps log_eps = np.minimum(log_eps + 1, (log_eps_max + log_eps_min) / 2) else: # Decrease eps log_eps_max = log_eps log_eps = np.maximum(log_eps - 1, (log_eps_max + log_eps_min) / 2) if p_n == 105: str_p_n = "i" else: str_p_n = str(p_n) print( "[L1] method = CNN-Cert-{}, model = {}, image no = {}, true_id = {}, target_label = {}, true_label = {}, norm = {}, robustness = {:.5f}" .format( activation, file_name, i, true_ids[i], target_label, predict_label, str_p_n, np.exp(log_eps_min), )) summation += np.exp(log_eps_min) K.clear_session() eps_avg = summation / len(inputs) total_time = (time.time() - start_time) / len(inputs) print( "[L0] method = CNN-Cert-{}, model = {}, total images = {}, norm = {}, avg robustness = {:.5f}, avg runtime = {:.2f}" .format(activation, file_name, len(inputs), str_p_n, eps_avg, total_time)) return eps_avg, total_time
#### 测试模型 y_pred = np.argmax(model.predict(x_test), axis=1) bca = utils.bca(y_test, y_pred) acc = np.sum(y_pred == y_test).astype(np.float32) / len(y_pred) #计算分类准确度 print('{}_{}: acc-{} bca-{}'.format(data_name, model_used, acc, bca)) # 计算攻击成功率(在分类正确的基础上) idx = y_pred == y_test # 判断前是否等于后,输出true or false x_t, y_t = x_test_poison[idx], y_test[idx] #判断为错误的地方全部去掉 idx = np.where(y_t == 0) #看哪些原来不是1,但是又被判为1了 x_t, y_t = x_t[idx], y_t[idx] #原来判断为1的地方全都去掉了 p_pred = np.argmax(model.predict(x_t), axis=1) # 在均为0.5时卡在这里 poison_s_rate = 1 - np.sum(p_pred == y_t).astype(np.float32) / len( p_pred) #用1减去对poisontest预测与test预测一样的样本占总样本的比例 print('poison attack success rate: {}'.format(poison_s_rate)) K.clear_session() racc.append(acc) rbca.append(bca) rasr.append(poison_s_rate) print('racc:', racc) print('rbca:', rbca) print('rpoison_rate:', rasr) print('Mean RCA={}, mean BCA={}, mean ASR={}'.format(np.mean(racc), np.mean(rbca), np.mean(rasr))) #################存储计算结果以画箱型图npz # results_dir = 'results/{}_{}'.format(data_name,model_used) # if not os.path.exists(results_dir):
def get_model_loss(fp_hdf_out, stepsize=1, shuffles=None, verbose=1): """ Loops across cross validated models and calculates loss and predictions for full experiment length Parameters ---------- fp_hdf_out : str File path to HDF5 file stepsize : int, optional Determines how many samples will be evaluated. 1 -> N samples evaluated, 2 -> N/2 samples evaluated, etc..., by default 1 shuffles : dict, optional If wavelets should be shuffled, important for calculating influence scores, by default None Returns ------- losses : (N,1) array_like Loss between predicted and ground truth observation predictions : dict Dictionary with predictions for each behaviour, each item in dict has size (N, Z) with Z the dimensions of the sample (e.g. Z_position=2, Z_speed=1, ...) indices : (N,1) array_like Indices which were evaluated, important when taking stepsize unequal to 1 """ dirname = os.path.dirname(fp_hdf_out) filename = os.path.basename(fp_hdf_out)[0:-3] cv_results = [] (_, _, _, opts) = util.hdf5.load_model_with_opts(dirname + '/models/' + filename + '_model_{}.h5'.format(0)) loss_names = opts['loss_names'] time_shift = opts['model_timesteps'] if verbose > 0: progress_bar = tf.keras.utils.Progbar(opts['num_cvs'], width=30, verbose=1, interval=0.05, unit_name='run') for k in range(0, opts['num_cvs']): K.clear_session() # Find folders model_path = dirname + '/models/' + filename + '_model_{}.h5'.format(k) # Load model and generators (model, training_generator, testing_generator, opts) = util.hdf5.load_model_with_opts(model_path) # ----------------------------------------------------------------------------------------------- if shuffles is not None: testing_generator = shuffle_wavelets(training_generator, testing_generator, shuffles) losses, predictions, indices = calculate_losses_from_generator( testing_generator, model, verbose=0, stepsize=stepsize) # ----------------------------------------------------------------------------------------------- cv_results.append((losses, predictions, indices)) if verbose > 0: progress_bar.add(1) cv_results = np.array(cv_results) # Reshape cv_results losses = np.concatenate(cv_results[:, 0], axis=0) predictions = {k: [] for k in loss_names} for out in cv_results[:, 1]: for p, name in zip(out, loss_names): predictions[name].append(p) for key, item in predictions.items(): if stepsize > 1: tmp_output = np.concatenate(predictions[key], axis=0)[:, -1, :] else: tmp_output = np.concatenate(predictions[key], axis=0)[:, -1, :] tmp_output = np.array([ np.pad(l, [time_shift, 0], mode='constant', constant_values=[l[0], 0]) for l in tmp_output.transpose() ]).transpose() predictions[key] = tmp_output indices = np.concatenate(cv_results[:, 2], axis=0) # We only take the last timestep for decoding, so decoder does not see any part of the future indices = indices + time_shift if stepsize > 1: losses = losses[:, :, -1] else: losses = losses[:, :, -1] losses = np.array([ np.pad(l, [time_shift, 0], mode='constant', constant_values=[l[0], 0]) for l in losses.transpose() ]).transpose() indices = np.arange(0, losses.shape[0]) # Also save to HDF5 hdf5_file = h5py.File(fp_hdf_out, mode='a') for key, item in predictions.items(): util.hdf5.create_or_update( hdf5_file, dataset_name="analysis/predictions/{}".format(key), dataset_shape=item.shape, dataset_type=np.float32, dataset_value=item) util.hdf5.create_or_update(hdf5_file, dataset_name="analysis/losses", dataset_shape=losses.shape, dataset_type=np.float32, dataset_value=losses) util.hdf5.create_or_update(hdf5_file, dataset_name="analysis/indices", dataset_shape=indices.shape, dataset_type=np.int64, dataset_value=indices) hdf5_file.close() return losses, predictions, indices
def __del__(self): K.clear_session()
def load_preaggregated_data(self): # Return objects of this function X = None Y = None X_valid = None Y_valid = None # Load pre-aggregated training dataset tfrecord_file_list = os.listdir(self.preaggregated_data_path) tfrecord_file_list = [ os.path.join(self.preaggregated_data_path, k) for k in tfrecord_file_list ] print('Pre-aggregated file list = ' + str(tfrecord_file_list)) reader = tf.TFRecordReader() key, examples = reader.read( tf.train.string_input_producer( tfrecord_file_list, num_epochs=1)) # Only generate all data once name_to_features = { "input_ids": tf.io.FixedLenFeature([self.max_seq_length], tf.int64), "input_mask": tf.io.FixedLenFeature([self.max_seq_length], tf.int64), "segment_ids": tf.io.FixedLenFeature([self.max_seq_length], tf.int64), } parsed_example = tf.parse_single_example(examples, name_to_features) parsed_example_values = list(parsed_example.values()) # Reuse Keras Session sess = K.get_session() # Just read all data into array for now. # TODO: Implment generator to support very large dataset that is not fit into RAM all_data = [] sess.run(tf.initialize_local_variables()) tf.train.start_queue_runners(sess=sess) try: while True: data = sess.run(parsed_example_values) for i in range(len(data)): if len(all_data) <= i: all_data.append([]) all_data[i].append(data[i]) except tf.errors.OutOfRangeError: pass all_data = [np.array(a) for a in all_data] X = all_data Y = all_data[0] # Y is only 'input_ids' tensor K.clear_session() # sess object is not valid anymore after this # Load pre-aggregated validation dataset tfrecord_file_list = os.listdir( self.preaggregated_validation_data_path) tfrecord_file_list = [ os.path.join(self.preaggregated_validation_data_path, k) for k in tfrecord_file_list ] print('Pre-aggregated file list = ' + str(tfrecord_file_list)) reader = tf.TFRecordReader() key, examples = reader.read( tf.train.string_input_producer( tfrecord_file_list, num_epochs=1)) # Only generate all data once name_to_features = { "input_ids": tf.io.FixedLenFeature([self.max_seq_length], tf.int64), "input_mask": tf.io.FixedLenFeature([self.max_seq_length], tf.int64), "segment_ids": tf.io.FixedLenFeature([self.max_seq_length], tf.int64), } parsed_example = tf.parse_single_example(examples, name_to_features) parsed_example_values = list(parsed_example.values()) # Reuse Keras Session sess = K.get_session() # Just read all data into array for now. # TODO: Implment generator to support very large dataset that is not fit into RAM all_data = [] sess.run(tf.initialize_local_variables()) tf.train.start_queue_runners(sess=sess) try: while True: data = sess.run(parsed_example_values) for i in range(len(data)): if len(all_data) <= i: all_data.append([]) all_data[i].append(data[i]) except tf.errors.OutOfRangeError: pass all_data = [np.array(a) for a in all_data] X_valid = all_data Y_valid = all_data[0] # Y is only 'input_ids' tensor K.clear_session() # sess object is not valid anymore after this #print(len(X_valid)) #print(len(Y_valid)) return (X, Y, X_valid, Y_valid)
def GNN(A, X): # 使用graph_hi构造图邻接矩阵A np.random.seed(0) # for reproducibility ITER = 10000 # Parameters P = OrderedDict([ ('es_patience', ITER), ('dataset', ['cora' ]), # 'cora', 'citeseer', 'pubmed', 'cloud', or 'synth' ('H_', [None]), ('n_channels', [16]), ('learning_rate', [5e-4]) ]) ############################################################################ # LOAD DATASET ############################################################################ A = np.maximum(A, A.T) A = sp.csr_matrix(A, dtype=np.float32) X = X.todense() n_feat = X.shape[-1] ############################################################################ # GNN MODEL ############################################################################ X_in = Input( tensor=tf.placeholder(tf.float32, shape=(None, n_feat), name='X_in')) A_in = Input(tensor=tf.sparse_placeholder(tf.float32, shape=(None, None)), name='A_in', sparse=True) # S_in = Input(tensor=tf.placeholder(tf.int32, shape=(None,), name='segment_ids_in')) A_norm = normalized_adjacency(A) X_1 = GCSConv(P['n_channels'], kernel_initializer='he_normal', activation='elu')([X_in, A_in]) pool1, adj1, C = MinCutPool(k=n_classes, h=P['H_'], activation='elu', return_mask=True)([X_1, A_in]) model = Model([X_in, A_in], [pool1, adj1, C]) model.compile('adam', None) ############################################################################ # TRAINING ############################################################################ # Setup sess = K.get_session() loss = model.total_loss opt = tf.train.AdamOptimizer(learning_rate=P['learning_rate']) train_step = opt.minimize(loss) # Initialize all variables init_op = tf.global_variables_initializer() sess.run(init_op) # Fit layer tr_feed_dict = {X_in: X, A_in: sp_matrix_to_sp_tensor_value(A_norm)} best_loss = np.inf patience = P['es_patience'] tol = 1e-5 for _ in tqdm(range(ITER)): outs = sess.run([train_step, model.losses[0], model.losses[1], C], feed_dict=tr_feed_dict) # c = np.argmax(outs[3], axis=-1) if outs[1] + outs[2] + tol < best_loss: best_loss = outs[1] + outs[2] patience = P['es_patience'] else: patience -= 1 if patience == 0: break ############################################################################ # RESULTS ############################################################################ C_ = sess.run([C], feed_dict=tr_feed_dict)[0] c = np.argmax(C_, axis=-1) K.clear_session() return c
def train_model(X, y, mtype, cv, epochs, cv_models_path, train, X_test=None, y_test=None, nfolds=None, rs=42, max_features=40000, maxlen=400, dropout_rate=0.25, rec_units=150, embed_dim=50, batch_size=256, fscore=False, threshold=0.3): if cv: kf = StratifiedKFold(n_splits=nfolds, random_state=rs) auc = [] roc = [] fscore_ = [] for c, (train_index, val_index) in enumerate(kf.split(X, y)): print(f' fold {c}') X_train, X_val = X[train_index], X[val_index] y_train, y_val = y[train_index], y[val_index] tokenizer = keras.preprocessing.text.Tokenizer( num_words=max_features) tokenizer.fit_on_texts(X_train) list_tokenized_train = tokenizer.texts_to_sequences(X_train) list_tokenized_val = tokenizer.texts_to_sequences(X_val) X_train = sequence.pad_sequences(list_tokenized_train, maxlen=maxlen) X_val = sequence.pad_sequences(list_tokenized_val, maxlen=maxlen) model = dl_model(model_type=mtype, max_features=max_features, maxlen=maxlen, dropout_rate=dropout_rate, embed_dim=embed_dim, rec_units=rec_units, max_sent_len=max_sen_len, max_sent_amount=max_sent_amount) print('Fitting') if train: model.fit(X_train, y_train, batch_size=batch_size, epochs=epochs, shuffle=True, verbose=1) model.save_weights(f'{cv_models_path}/{mtype}_fold_{c}.h5') else: model.load_weights(f'{cv_models_path}/{mtype}_fold_{c}.h5') probs = model.predict(X_val, batch_size=batch_size, verbose=1) if fscore: #for threshold in [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]: threshold = threshold probs_class = probs.copy() probs_class[probs_class >= threshold] = 1 probs_class[probs_class < threshold] = 0 precision = precision_score(y_val, probs_class) recall = recall_score(y_val, probs_class) fscore = f1_score(y_val, probs_class) print( f' {threshold} fold {c} precision {round(precision, 3)} recall {round(recall, 3)} fscore {round(fscore,3)}' ) fscore_.append(fscore) auc_f = average_precision_score(y_val, probs) auc.append(auc_f) roc_f = roc_auc_score(y_val, probs) roc.append(roc_f) print( f'fold {c} average precision {round(auc_f, 3)} ++++ roc auc {round(roc_f, 3)}' ) del model K.clear_session() if fscore: print( f'PR-C {round(np.array(auc).mean(), 3)} ++++ ROC AUC {round(np.array(roc).mean(), 3)} ++++ FScore {round(np.array(fscore_).mean(), 3)}' ) print( f'PR-C std {round(np.array(auc).std(), 3)} ++++ ROC AUC std {round(np.array(roc).std(), 3)} ++++ FScore std {round(np.array(fscore_).std(), 3)}' ) else: print( f'PR-C {round(np.array(auc).mean(), 3)} ++++ ROC AUC {round(np.array(roc).mean(), 3)}' ) print( f'PR-C std {round(np.array(auc).std(), 3)} ++++ ROC AUC std {round(np.array(roc).std(), 3)}' ) else: X_train = X y_train = y tokenizer = keras.preprocessing.text.Tokenizer(num_words=max_features, oov_token='unknown') tokenizer.fit_on_texts(X_train) list_tokenized_train = tokenizer.texts_to_sequences(X_train) list_tokenized_test = tokenizer.texts_to_sequences(X_test) X_train = sequence.pad_sequences(list_tokenized_train, maxlen=maxlen) X_test = sequence.pad_sequences(list_tokenized_test, maxlen=maxlen) y_train = np.array(y_train) y_test = np.array(y_test) model = dl_model(model_type=mtype, max_features=max_features, maxlen=maxlen, dropout_rate=dropout_rate, embed_dim=embed_dim, rec_units=rec_units, max_sent_len=max_sen_len, max_sent_amount=max_sent_amount) print('Fitting') if train: model.fit(X_train, y_train, batch_size=batch_size, epochs=epochs, shuffle=True, verbose=1) model.save_weights(f'{cv_models_path}/{mtype}.h5') else: model.load_weights(f'{cv_models_path}/{mtype}.h5') probs = model.predict(X_test, batch_size=batch_size, verbose=1) auc_f = average_precision_score(y_test, probs) roc_f = roc_auc_score(y_test, probs) if fscore: threshold = threshold probs_class = probs.copy() probs_class[probs_class >= threshold] = 1 probs_class[probs_class < threshold] = 0 precision = precision_score(y_test, probs_class) recall = recall_score(y_test, probs_class) fscore = f1_score(y_test, probs_class) if fscore: print('_________________________________') print( f'PR-C is {round(auc_f,3)} ++++ ROC AUC is {round(roc_f,3)} +++++ FScore is {round(fscore,3)}' ) print('_________________________________\n') else: print('_________________________________') print( f'PR-C is {round(auc_f,3)} ++++ ROC AUC is {round(roc_f,3)}' ) print('_________________________________\n')
def clear_keras_session(): """Clears Keras session.""" K.clear_session()
def __call__(self, dag, gpuID, epoch_num=100, out_model='cgpunet.hdf5'): if self.verbose: print('GPUID :', gpuID) print('epoch_num :', epoch_num) print('batch_size:', self.batchsize) train_steps = int(self.train_len/self.batchsize) valid_steps = int(self.valid_len/self.batchsize_valid) model = dag_2_cnn(dag, gpuID, self.input_shape, self.target_shape) #print summary model.summary() model_checkpoint = ModelCheckpoint(out_model, monitor='loss',verbose=1, save_best_only=True) history = History() #NOTE: default values: workers=1, multiprocessing=False. #TODO: investigate workers>1 history = model.fit_generator(generator=self.trainGenerator, steps_per_epoch=train_steps, epochs=epoch_num, callbacks=[model_checkpoint], validation_data=self.validGenerator, validation_steps=valid_steps, validation_freq= int(epoch_num)) val_acc = history.history['val_accuracy'] #val_loss = history.history['val_loss'] val_precision = history.history['val_precision'] val_recall = history.history['val_recall'] last_epoch = len(val_precision) - 1 val_f1 = 2*((val_precision[last_epoch]*val_recall[last_epoch])/(val_precision[last_epoch]+val_recall[last_epoch]+K.epsilon())) trainable_count = int(np.sum([K.count_params(p) for p in model.trainable_weights])) if not os.path.isdir('./figures'): os.makedirs('./figures') acc_fig_name = out_model.replace('.hdf5', '_acc.png') acc_fig_name = './figures/' + acc_fig_name loss_fig_name = out_model.replace('.hdf5', '_loss.png') loss_fig_name = './figures/' + loss_fig_name # Plot training & validation accuracy values plt.figure() plt.plot(history.history['accuracy']) plt.plot(history.history['val_accuracy']) plt.title('Model accuracy: {}'.format(out_model)) plt.ylabel('Accuracy') plt.xlabel('Epoch') plt.legend(['Train', 'Validation'], loc='upper left') plt.savefig(acc_fig_name) plt.figure() plt.plot(history.history['loss']) plt.plot(history.history['val_loss']) plt.title('Model loss": {}'.format(out_model)) plt.ylabel('Loss') plt.xlabel('Epoch') plt.legend(['Train', 'Validation'], loc='upper left') plt.savefig(loss_fig_name) pickle_name = out_model.replace('.hdf5', '.gpickle') if not os.path.isdir('./p_files'): os.makedirs('./p_files') pickle_name = './p_files/' + pickle_name nx.write_gpickle(dag, pickle_name) K.clear_session() return (float(val_f1), trainable_count)