def train_model(layers, path, des,model_num):
# Load data
x_train, y_train, x_val, y_val = load_data_with_validation_sets(path)
# Create directories for saving model and csv files
try:
os.mkdir(f'{des}models')
except FileExistsError:
pass
try:
os.mkdir(f'{des}metrics')
except FileExistsError:
pass
# Set callbacks
es = EarlyStopping(monitor = 'val_PR', mode='max', verbose=1, patience=30)
mc = ModelCheckpoint(
filepath=f'{des}/models/model_{model_num}.h5',
monitor='val_PR',
mode='max',
verbose=1,
save_weights_only=False,
save_best_only=True
)
# Compile and train model
model = build_model(layers, x_train.shape[1])
history = model.fit(x_train, y_train, epochs=5, batch_size=16, validation_data=(x_val, y_val), callbacks = [es,mc], verbose = 1)
# Save training metrics
pr = max(history.history['val_PR'])
print(f'Max precision recall for {layers} layer(s): {pr}')
h = pd.DataFrame(history.history)
h.to_csv(f'{des}metrics/{pr}_max_model_{model_num}_results.csv')
clear_session()
def fitness(learning_rate, num_lstm_nodes, dropout, batch_size):
# Setting seed and clearing model graphs in backend
tf.random.set_seed(SimpleCNN.seed_num)
K.clear_session()
tf.compat.v1.reset_default_graph()
# Initializing model, compiling & training it
model = self.generate_model(use_optimised_hyperparameters=False,
dropout=dropout,
number_of_lstm_nodes=num_lstm_nodes)
optimizer = SGD(learning_rate=learning_rate)
model.compile(loss='mean_squared_error', optimizer=optimizer)
model.fit(train_x,
train_y,
batch_size=batch_size,
epochs=model_epochs,
verbose=2,
shuffle=False,
validation_data=(val_x, val_y))
# Generating prediction on Validation data
validation_data_prediction = model.predict(val_x)
# Calculating MSE for the model trained with candidate Hyperparameters of this iteration
mse_validation = mean_squared_error(val_y,
validation_data_prediction)
# Deleting created model
del model
return mse_validation
def reset_keras():
"""Reset Keras session."""
sess = get_session()
clear_session()
sess.close()
gc.collect()
# Set to force CPU calculations
os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
def gru_keras(max_features,
maxlen,
bidirectional,
dropout_rate,
embed_dim,
rec_units,
mtype='GRU',
reduction=None,
classes=4,
lr=0.001):
if K.backend == 'tensorflow':
K.clear_session()
input_layer = Input(shape=(maxlen, ))
embedding_layer = Embedding(max_features,
output_dim=embed_dim,
trainable=True)(input_layer)
x = SpatialDropout1D(dropout_rate)(embedding_layer)
if reduction:
if mtype == 'GRU':
if bidirectional:
x = Bidirectional(
CuDNNGRU(units=rec_units, return_sequences=True))(x)
else:
x = CuDNNGRU(units=rec_units, return_sequences=True)(x)
elif mtype == 'LSTM':
if bidirectional:
x = Bidirectional(
CuDNNLSTM(units=rec_units, return_sequences=True))(x)
else:
x = CuDNNLSTM(units=rec_units, return_sequences=True)(x)
if reduction == 'average':
x = GlobalAveragePooling1D()(x)
elif reduction == 'maximum':
x = GlobalMaxPool1D()(x)
else:
if mtype == 'GRU':
if bidirectional:
x = Bidirectional(
CuDNNGRU(units=rec_units, return_sequences=False))(x)
else:
x = CuDNNGRU(units=rec_units, return_sequences=False)(x)
elif mtype == 'LSTM':
if bidirectional:
x = Bidirectional(
CuDNNLSTM(units=rec_units, return_sequences=False))(x)
else:
x = CuDNNLSTM(units=rec_units, return_sequences=False)(x)
output_layer = Dense(classes, activation="sigmoid")(x)
model = Model(inputs=input_layer, outputs=output_layer)
model.compile(loss='categorical_crossentropy',
optimizer=RMSprop(learning_rate=lr, clipvalue=1, clipnorm=1),
metrics=['acc'])
return model
def clear_model_memory(self):
if self.histogram_bin == 'random':
del self.model_list
else:
del self.fixed_bin_model
backend.clear_session()
gc.collect()
gc.collect()
def reset_tf_session():
curr_session = tf.get_default_session()
# close current session
if curr_session is not None:
curr_session.close()
# reset graph
K.clear_session()
# create new session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
s = tf.InteractiveSession(config=config)
K.set_session(s)
return s
def __train(args):
benchmark_index, model_index, x_train, x_support, x_test, y_train, y_support, _, y_train_value, y_support_value, y_test_value = args
__setup_tf_config(model_index)
print("Setting up Model " + str(model_index + 1) + "/" +
str(CONFIG["num_models"]))
input = Input(shape=CONFIG["input_shape"])
output = build_fsl_attention(input)
model = Model(inputs=input, outputs=output)
model.compile(
optimizer=Adam(
# lr=0.0005 # デフォルトは0.001
),
loss=center_loss,
)
expanded_y_train = np.array([
np.concatenate(
[y, np.full(
CONFIG["output_dim"] - y_train.shape[1],
0.0,
)]) for y in y_train
])
histories = Histories(
x_train,
y_train_value,
x_test,
y_test_value,
x_support,
y_support_value,
benchmark_index,
model_index,
)
model.fit(
x_train,
expanded_y_train,
batch_size=CONFIG["batch_size"],
epochs=CONFIG["epochs"],
verbose=False,
callbacks=[histories],
shuffle=CONFIG["shuffle"],
)
K.clear_session()
def reset_keras(self):
sess = get_session()
clear_session()
sess.close()
sess = get_session()
try:
del classifier
except:
pass
print(gc.collect())
# use the same config as you used to create the session
config = ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.9
config.gpu_options.allow_growth = True
set_session(InteractiveSession(config=config))
def reset_keras():
sess = get_session()
clear_session()
sess.close()
sess = get_session()
try:
del classifier # this is from global space - change this as you need
except:
pass
# if it's done something you should see a number being outputted
print(gc.collect())
# use the same config as you used to create the session
config = tensorflow.compat.v1.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 1
config.gpu_options.visible_device_list = "0"
set_session(tensorflow.compat.v1.Session(config=config))
def set_session_config(per_process_gpu_memory_fraction=None,
allow_growth=None,
device_list='0'):
"""
:param allow_growth: When necessary, reserve memory
:param float per_process_gpu_memory_fraction: specify GPU memory usage as 0 to 1
:return:
"""
import tensorflow as tf
import tensorflow.compat.v1.keras.backend as K
K.clear_session()
config = tf.compat.v1.ConfigProto(gpu_options=tf.compat.v1.GPUOptions(
per_process_gpu_memory_fraction=per_process_gpu_memory_fraction,
allow_growth=allow_growth,
visible_device_list=device_list))
sess = tf.compat.v1.Session(config=config)
K.set_session(sess)
def cnn_keras(max_features,
maxlen,
dropout_rate,
embed_dim,
num_filters=300,
classes=4,
lr=0.001):
if K.backend == 'tensorflow':
K.clear_session()
input_layer = Input(shape=(maxlen, ))
embedding_layer = Embedding(max_features,
output_dim=embed_dim,
trainable=True)(input_layer)
x = SpatialDropout1D(dropout_rate)(embedding_layer)
x = Conv1D(num_filters, 7, activation='relu', padding='same')(x)
x = GlobalMaxPooling1D()(x)
output_layer = Dense(classes, activation="sigmoid")(x)
model = Model(inputs=input_layer, outputs=output_layer)
model.compile(loss='categorical_crossentropy',
optimizer=RMSprop(learning_rate=lr, clipvalue=1, clipnorm=1),
metrics=['acc'])
return model
def test_arch(layers, node, path,
des): # Iteratively train different architectures
# Load data
x_train, y_train, x_val, y_val = load_data_with_validation_sets(path)
for l in range(layers): # Loop through layer range
if l > 0:
l = l + 1 # Minimum number of layers is 2
for n in node: # Loop though user input node options
print(
f'Testing precision recall of {l} layer(s) with {n} nodes...'
)
print(f'With early stopping patience at 30...')
# Create model
es = EarlyStopping(monitor='val_PR',
mode='max',
verbose=1,
patience=30)
nodes = make_nodes(l, n)
model = build_model(nodes, x_train.shape[1])
history = model.fit(x_train,
y_train,
epochs=1000,
batch_size=16,
validation_data=(x_val, y_val),
callbacks=[es],
verbose=1)
# Save training metrics to a csv
pr = max(history.history['val_PR'])
print(
f'Max precision recall for {l} layer(s) with {n} nodes: {pr}'
)
h = pd.DataFrame(history.history)
h.to_csv(f'{des}{pr}_max_{l}_layers_{n}_nodes_results.csv')
clear_session()
def get_model_memory_usage(net_list, batch_size, input_shape, target_shape, return_dict):
try:
model = dag_2_cnn(cgp_2_dag(net_list), 0, input_shape, target_shape, compile=False)
except (tf.errors.ResourceExhaustedError, KeyError) as e:
print(e)
return_dict["memory"] = 1000
return
shapes_mem_count = 0
internal_model_mem_count = 0
for l in model.layers:
single_layer_mem = 1
out_shape = l.output_shape
if type(out_shape) is list:
out_shape = out_shape[0]
for s in out_shape:
if s is None:
continue
single_layer_mem *= s
shapes_mem_count += single_layer_mem
trainable_count = np.sum([K.count_params(p) for p in model.trainable_weights])
non_trainable_count = np.sum([K.count_params(p) for p in model.non_trainable_weights])
number_size = 4.0
if K.floatx() == 'float16':
number_size = 2.0
if K.floatx() == 'float64':
number_size = 8.0
total_memory = number_size * (batch_size * shapes_mem_count + trainable_count + non_trainable_count)
gbytes = np.round(total_memory / (1024.0 ** 3), 3) + internal_model_mem_count
K.clear_session()
return_dict["memory"] = gbytes
def create_model(input_shape,
anchors,
num_classes,
load_pretrained=True,
freeze_body=2,
weights_path='model_data/yolo_weights.h5'):
"""create the training model"""
K.clear_session() # get a new session
image_input = Input(shape=(None, None, 3))
h, w = input_shape
num_anchors = len(anchors)
y_true = Input(shape=(h // grid_size_multiplier,
w // grid_size_multiplier, anchors_per_level,
num_classes + 5 + NUM_ANGLES3))
model_body = yolo_body(image_input, anchors_per_level, num_classes)
print('Create Poly-YOLO model with {} anchors and {} classes.'.format(
num_anchors, num_classes))
if load_pretrained:
model_body.load_weights(weights_path,
by_name=True,
skip_mismatch=True)
print('Load weights {}.'.format(weights_path))
model_loss = Lambda(yolo_loss,
output_shape=(1, ),
name='yolo_loss',
arguments={
'anchors': anchors,
'num_classes': num_classes,
'ignore_thresh': 0.5
})([model_body.output, y_true])
model = Model([model_body.input, y_true], model_loss)
# print(model.summary())
return model
def run_single(mode, config, word_embedding, cognitive_data, cognitive_parent,
modality, feature, truncate_first_line, gpu_id):
'''
Takes a configuration dictionary and keys for a word embedding and cognitive
data source, runs model, logs results and prepares output for plotting.
:param mode: Type of embeddings, either 'proper' or 'random'
:param config: Configuration dictionary
:param word_embedding: String specifying word embedding (configuration key)
:param cognitive_data: String specifying cognitiv data source (configuration key)
:param feature: Cognitive data feature to be predicted
:param truncate_first_line: If the first line of the embedding file should be truncated (when containing meta data)
:param gpu_ids: IDs of available GPUs
'''
# Tensorflow configuration
import tensorflow as tf
from tensorflow.compat.v1.keras.backend import set_session, clear_session
if gpu_id is not None:
gpu_count = 1
soft_placement = True
else:
gpu_count = 0
soft_placement = False
tf_config = tf.compat.v1.ConfigProto(intra_op_parallelism_threads=1,
inter_op_parallelism_threads=1,
allow_soft_placement=soft_placement,
device_count={
'GPU': gpu_count,
'CPU': 1
})
if gpu_id is not None:
tf_config.gpu_options.allow_growth = True
tf_config.gpu_options.per_process_gpu_memory_fraction = 0.25
tf_config.gpu_options.visible_device_list = str(gpu_id)
session = tf.compat.v1.Session(config=tf_config)
set_session(session)
##############################################################################
# Create logging information
##############################################################################
logging = {"folds": []}
logging["wordEmbedding"] = word_embedding
logging["cognitiveData"] = cognitive_data
logging["cognitiveParent"] = cognitive_parent
logging["modality"] = modality
logging["feature"] = feature
##############################################################################
# Run model
##############################################################################
startTime = datetime.now()
word_error, grids_result, mserrors = handler(mode, config, word_embedding,
cognitive_data, feature,
truncate_first_line)
history = {'loss': [], 'val_loss': []}
loss_list = []
val_loss_list = []
##############################################################################
# logging results
##############################################################################
for i in range(len(grids_result)):
fold = {}
logging['folds'].append(fold)
# BEST PARAMS APPENDING
for key in grids_result[i].best_params_:
logging['folds'][i][
key.upper()] = grids_result[i].best_params_[key]
if config['cogDataConfig'][cognitive_data][
'type'] == "multivariate_output":
logging['folds'][i]['MSE_PREDICTION_ALL_DIM'] = list(mserrors[i])
logging['folds'][i]['MSE_PREDICTION'] = np.mean(mserrors[i])
elif config['cogDataConfig'][cognitive_data][
'type'] == "single_output":
logging['folds'][i]['MSE_PREDICTION'] = mserrors[i]
logging['folds'][i]['LOSS'] = grids_result[
i].best_estimator_.model.history.history['loss']
logging['folds'][i]['VALIDATION_LOSS'] = grids_result[
i].best_estimator_.model.history.history['val_loss']
loss_list.append(
np.array(
grids_result[i].best_estimator_.model.history.history['loss'],
dtype='float'))
val_loss_list.append(
np.array(grids_result[i].best_estimator_.model.history.
history['val_loss'],
dtype='float'))
if config['cogDataConfig'][cognitive_data][
'type'] == "multivariate_output":
mserrors = np.array(mserrors, dtype='float')
mse = np.mean(mserrors, axis=0)
logging['AVERAGE_MSE_ALL_DIM'] = list(mse)
logging['AVERAGE_MSE'] = np.mean(mse)
elif config['cogDataConfig'][cognitive_data]['type'] == "single_output":
mse = np.array(mserrors, dtype='float').mean()
logging['AVERAGE_MSE'] = mse
##############################################################################
# Prepare results for plot
##############################################################################
history['loss'] = np.mean([loss_list[i] for i in range(len(loss_list))],
axis=0)
history['val_loss'] = np.mean(
[val_loss_list[i] for i in range(len(val_loss_list))], axis=0)
timeTaken = datetime.now() - startTime
logging["timeTaken"] = str(timeTaken)
# Clean-up tf session
session.close()
clear_session()
return word_embedding, logging, word_error, history
def run(file_name,
n_samples,
p_n,
q_n,
activation="relu",
cifar=False,
tinyimagenet=False):
np.random.seed(1215)
tf.random.set_seed(1215)
random.seed(1215)
keras_model = load_model(file_name, custom_objects={"fn": fn, "tf": tf})
if tinyimagenet:
model = CNNModel(keras_model, inp_shape=(64, 64, 3))
elif cifar:
model = CNNModel(keras_model, inp_shape=(32, 32, 3))
else:
model = CNNModel(keras_model)
# Set correct linear_bounds function
global linear_bounds
if activation == "relu":
linear_bounds = relu_linear_bounds
elif activation == "ada":
linear_bounds = ada_linear_bounds
elif activation == "sigmoid":
linear_bounds = sigmoid_linear_bounds
elif activation == "tanh":
linear_bounds = tanh_linear_bounds
elif activation == "arctan":
linear_bounds = atan_linear_bounds
upper_bound_conv.recompile()
lower_bound_conv.recompile()
compute_bounds.recompile()
if cifar:
inputs, targets, true_labels, true_ids, img_info = generate_data(
CIFAR(),
samples=n_samples,
targeted=True,
random_and_least_likely=True,
target_type=0b0010,
predictor=model.model.predict,
start=0,
)
elif tinyimagenet:
inputs, targets, true_labels, true_ids, img_info = generate_data(
tinyImagenet(),
samples=n_samples,
targeted=True,
random_and_least_likely=True,
target_type=0b0010,
predictor=model.model.predict,
start=0,
)
else:
inputs, targets, true_labels, true_ids, img_info = generate_data(
MNIST(),
samples=n_samples,
targeted=True,
random_and_least_likely=True,
target_type=0b0010,
predictor=model.model.predict,
start=0,
)
# 0b01111 <- all
# 0b0010 <- random
# 0b0001 <- top2
# 0b0100 <- least
steps = 15
eps_0 = 0.05
summation = 0
# warmup(model, inputs[0].astype(np.float32), eps_0, p_n, find_output_bounds)
start_time = time.time()
for i in range(len(inputs)):
print("--- CNN-Cert: Computing eps for input image " + str(i) + "---")
predict_label = np.argmax(true_labels[i])
target_label = np.argmax(targets[i])
weights = model.weights[:-1]
biases = model.biases[:-1]
shapes = model.shapes[:-1]
W, b, s = model.weights[-1], model.biases[-1], model.shapes[-1]
last_weight = (W[predict_label, :, :, :] -
W[target_label, :, :, :]).reshape([1] +
list(W.shape[1:]))
weights.append(last_weight)
biases.append(np.asarray([b[predict_label] - b[target_label]]))
shapes.append((1, 1, 1))
# Perform binary search
log_eps = np.log(eps_0)
log_eps_min = -np.inf
log_eps_max = np.inf
for j in range(steps):
LB, UB = find_output_bounds(
weights,
biases,
shapes,
model.pads,
model.strides,
inputs[i].astype(np.float32),
np.exp(log_eps),
p_n,
)
print(
"Step {}, eps = {:.5f}, {:.6s} <= f_c - f_t <= {:.6s}".format(
j, np.exp(log_eps), str(np.squeeze(LB)),
str(np.squeeze(UB))))
if LB > 0: # Increase eps
log_eps_min = log_eps
log_eps = np.minimum(log_eps + 1,
(log_eps_max + log_eps_min) / 2)
else: # Decrease eps
log_eps_max = log_eps
log_eps = np.maximum(log_eps - 1,
(log_eps_max + log_eps_min) / 2)
if p_n == 105:
str_p_n = "i"
else:
str_p_n = str(p_n)
print(
"[L1] method = CNN-Cert-{}, model = {}, image no = {}, true_id = {}, target_label = {}, true_label = {}, norm = {}, robustness = {:.5f}"
.format(
activation,
file_name,
i,
true_ids[i],
target_label,
predict_label,
str_p_n,
np.exp(log_eps_min),
))
summation += np.exp(log_eps_min)
K.clear_session()
eps_avg = summation / len(inputs)
total_time = (time.time() - start_time) / len(inputs)
print(
"[L0] method = CNN-Cert-{}, model = {}, total images = {}, norm = {}, avg robustness = {:.5f}, avg runtime = {:.2f}"
.format(activation, file_name, len(inputs), str_p_n, eps_avg,
total_time))
return eps_avg, total_time
#### 测试模型
y_pred = np.argmax(model.predict(x_test), axis=1)
bca = utils.bca(y_test, y_pred)
acc = np.sum(y_pred == y_test).astype(np.float32) / len(y_pred) #计算分类准确度
print('{}_{}: acc-{} bca-{}'.format(data_name, model_used, acc, bca))
# 计算攻击成功率（在分类正确的基础上）
idx = y_pred == y_test # 判断前是否等于后，输出true or false
x_t, y_t = x_test_poison[idx], y_test[idx] #判断为错误的地方全部去掉
idx = np.where(y_t == 0) #看哪些原来不是1，但是又被判为1了
x_t, y_t = x_t[idx], y_t[idx] #原来判断为1的地方全都去掉了
p_pred = np.argmax(model.predict(x_t), axis=1) # 在均为0.5时卡在这里
poison_s_rate = 1 - np.sum(p_pred == y_t).astype(np.float32) / len(
p_pred) #用1减去对poisontest预测与test预测一样的样本占总样本的比例
print('poison attack success rate: {}'.format(poison_s_rate))
K.clear_session()
racc.append(acc)
rbca.append(bca)
rasr.append(poison_s_rate)
print('racc:', racc)
print('rbca:', rbca)
print('rpoison_rate:', rasr)
print('Mean RCA={}, mean BCA={}, mean ASR={}'.format(np.mean(racc),
np.mean(rbca),
np.mean(rasr)))
#################存储计算结果以画箱型图npz
# results_dir = 'results/{}_{}'.format(data_name,model_used)
# if not os.path.exists(results_dir):
def get_model_loss(fp_hdf_out, stepsize=1, shuffles=None, verbose=1):
"""
Loops across cross validated models and calculates loss and predictions for full experiment length
Parameters
----------
fp_hdf_out : str
File path to HDF5 file
stepsize : int, optional
Determines how many samples will be evaluated. 1 -> N samples evaluated,
2 -> N/2 samples evaluated, etc..., by default 1
shuffles : dict, optional
If wavelets should be shuffled, important for calculating influence scores, by default None
Returns
-------
losses : (N,1) array_like
Loss between predicted and ground truth observation
predictions : dict
Dictionary with predictions for each behaviour, each item in dict has size (N, Z) with Z the dimensions of the sample (e.g. Z_position=2, Z_speed=1, ...)
indices : (N,1) array_like
Indices which were evaluated, important when taking stepsize unequal to 1
"""
dirname = os.path.dirname(fp_hdf_out)
filename = os.path.basename(fp_hdf_out)[0:-3]
cv_results = []
(_, _, _,
opts) = util.hdf5.load_model_with_opts(dirname + '/models/' + filename +
'_model_{}.h5'.format(0))
loss_names = opts['loss_names']
time_shift = opts['model_timesteps']
if verbose > 0:
progress_bar = tf.keras.utils.Progbar(opts['num_cvs'],
width=30,
verbose=1,
interval=0.05,
unit_name='run')
for k in range(0, opts['num_cvs']):
K.clear_session()
# Find folders
model_path = dirname + '/models/' + filename + '_model_{}.h5'.format(k)
# Load model and generators
(model, training_generator, testing_generator,
opts) = util.hdf5.load_model_with_opts(model_path)
# -----------------------------------------------------------------------------------------------
if shuffles is not None:
testing_generator = shuffle_wavelets(training_generator,
testing_generator, shuffles)
losses, predictions, indices = calculate_losses_from_generator(
testing_generator, model, verbose=0, stepsize=stepsize)
# -----------------------------------------------------------------------------------------------
cv_results.append((losses, predictions, indices))
if verbose > 0:
progress_bar.add(1)
cv_results = np.array(cv_results)
# Reshape cv_results
losses = np.concatenate(cv_results[:, 0], axis=0)
predictions = {k: [] for k in loss_names}
for out in cv_results[:, 1]:
for p, name in zip(out, loss_names):
predictions[name].append(p)
for key, item in predictions.items():
if stepsize > 1:
tmp_output = np.concatenate(predictions[key], axis=0)[:, -1, :]
else:
tmp_output = np.concatenate(predictions[key], axis=0)[:, -1, :]
tmp_output = np.array([
np.pad(l, [time_shift, 0],
mode='constant',
constant_values=[l[0], 0])
for l in tmp_output.transpose()
]).transpose()
predictions[key] = tmp_output
indices = np.concatenate(cv_results[:, 2], axis=0)
# We only take the last timestep for decoding, so decoder does not see any part of the future
indices = indices + time_shift
if stepsize > 1:
losses = losses[:, :, -1]
else:
losses = losses[:, :, -1]
losses = np.array([
np.pad(l, [time_shift, 0],
mode='constant',
constant_values=[l[0], 0]) for l in losses.transpose()
]).transpose()
indices = np.arange(0, losses.shape[0])
# Also save to HDF5
hdf5_file = h5py.File(fp_hdf_out, mode='a')
for key, item in predictions.items():
util.hdf5.create_or_update(
hdf5_file,
dataset_name="analysis/predictions/{}".format(key),
dataset_shape=item.shape,
dataset_type=np.float32,
dataset_value=item)
util.hdf5.create_or_update(hdf5_file,
dataset_name="analysis/losses",
dataset_shape=losses.shape,
dataset_type=np.float32,
dataset_value=losses)
util.hdf5.create_or_update(hdf5_file,
dataset_name="analysis/indices",
dataset_shape=indices.shape,
dataset_type=np.int64,
dataset_value=indices)
hdf5_file.close()
return losses, predictions, indices
def __del__(self):
K.clear_session()
def load_preaggregated_data(self):
# Return objects of this function
X = None
Y = None
X_valid = None
Y_valid = None
# Load pre-aggregated training dataset
tfrecord_file_list = os.listdir(self.preaggregated_data_path)
tfrecord_file_list = [
os.path.join(self.preaggregated_data_path, k)
for k in tfrecord_file_list
]
print('Pre-aggregated file list = ' + str(tfrecord_file_list))
reader = tf.TFRecordReader()
key, examples = reader.read(
tf.train.string_input_producer(
tfrecord_file_list,
num_epochs=1)) # Only generate all data once
name_to_features = {
"input_ids": tf.io.FixedLenFeature([self.max_seq_length],
tf.int64),
"input_mask": tf.io.FixedLenFeature([self.max_seq_length],
tf.int64),
"segment_ids": tf.io.FixedLenFeature([self.max_seq_length],
tf.int64),
}
parsed_example = tf.parse_single_example(examples, name_to_features)
parsed_example_values = list(parsed_example.values())
# Reuse Keras Session
sess = K.get_session()
# Just read all data into array for now.
# TODO: Implment generator to support very large dataset that is not fit into RAM
all_data = []
sess.run(tf.initialize_local_variables())
tf.train.start_queue_runners(sess=sess)
try:
while True:
data = sess.run(parsed_example_values)
for i in range(len(data)):
if len(all_data) <= i:
all_data.append([])
all_data[i].append(data[i])
except tf.errors.OutOfRangeError:
pass
all_data = [np.array(a) for a in all_data]
X = all_data
Y = all_data[0] # Y is only 'input_ids' tensor
K.clear_session() # sess object is not valid anymore after this
# Load pre-aggregated validation dataset
tfrecord_file_list = os.listdir(
self.preaggregated_validation_data_path)
tfrecord_file_list = [
os.path.join(self.preaggregated_validation_data_path, k)
for k in tfrecord_file_list
]
print('Pre-aggregated file list = ' + str(tfrecord_file_list))
reader = tf.TFRecordReader()
key, examples = reader.read(
tf.train.string_input_producer(
tfrecord_file_list,
num_epochs=1)) # Only generate all data once
name_to_features = {
"input_ids": tf.io.FixedLenFeature([self.max_seq_length],
tf.int64),
"input_mask": tf.io.FixedLenFeature([self.max_seq_length],
tf.int64),
"segment_ids": tf.io.FixedLenFeature([self.max_seq_length],
tf.int64),
}
parsed_example = tf.parse_single_example(examples, name_to_features)
parsed_example_values = list(parsed_example.values())
# Reuse Keras Session
sess = K.get_session()
# Just read all data into array for now.
# TODO: Implment generator to support very large dataset that is not fit into RAM
all_data = []
sess.run(tf.initialize_local_variables())
tf.train.start_queue_runners(sess=sess)
try:
while True:
data = sess.run(parsed_example_values)
for i in range(len(data)):
if len(all_data) <= i:
all_data.append([])
all_data[i].append(data[i])
except tf.errors.OutOfRangeError:
pass
all_data = [np.array(a) for a in all_data]
X_valid = all_data
Y_valid = all_data[0] # Y is only 'input_ids' tensor
K.clear_session() # sess object is not valid anymore after this
#print(len(X_valid))
#print(len(Y_valid))
return (X, Y, X_valid, Y_valid)
def GNN(A, X):
# 使用graph_hi构造图邻接矩阵A
np.random.seed(0) # for reproducibility
ITER = 10000
# Parameters
P = OrderedDict([
('es_patience', ITER),
('dataset', ['cora'
]), # 'cora', 'citeseer', 'pubmed', 'cloud', or 'synth'
('H_', [None]),
('n_channels', [16]),
('learning_rate', [5e-4])
])
############################################################################
# LOAD DATASET
############################################################################
A = np.maximum(A, A.T)
A = sp.csr_matrix(A, dtype=np.float32)
X = X.todense()
n_feat = X.shape[-1]
############################################################################
# GNN MODEL
############################################################################
X_in = Input(
tensor=tf.placeholder(tf.float32, shape=(None, n_feat), name='X_in'))
A_in = Input(tensor=tf.sparse_placeholder(tf.float32, shape=(None, None)),
name='A_in',
sparse=True)
# S_in = Input(tensor=tf.placeholder(tf.int32, shape=(None,), name='segment_ids_in'))
A_norm = normalized_adjacency(A)
X_1 = GCSConv(P['n_channels'],
kernel_initializer='he_normal',
activation='elu')([X_in, A_in])
pool1, adj1, C = MinCutPool(k=n_classes,
h=P['H_'],
activation='elu',
return_mask=True)([X_1, A_in])
model = Model([X_in, A_in], [pool1, adj1, C])
model.compile('adam', None)
############################################################################
# TRAINING
############################################################################
# Setup
sess = K.get_session()
loss = model.total_loss
opt = tf.train.AdamOptimizer(learning_rate=P['learning_rate'])
train_step = opt.minimize(loss)
# Initialize all variables
init_op = tf.global_variables_initializer()
sess.run(init_op)
# Fit layer
tr_feed_dict = {X_in: X, A_in: sp_matrix_to_sp_tensor_value(A_norm)}
best_loss = np.inf
patience = P['es_patience']
tol = 1e-5
for _ in tqdm(range(ITER)):
outs = sess.run([train_step, model.losses[0], model.losses[1], C],
feed_dict=tr_feed_dict)
# c = np.argmax(outs[3], axis=-1)
if outs[1] + outs[2] + tol < best_loss:
best_loss = outs[1] + outs[2]
patience = P['es_patience']
else:
patience -= 1
if patience == 0:
break
############################################################################
# RESULTS
############################################################################
C_ = sess.run([C], feed_dict=tr_feed_dict)[0]
c = np.argmax(C_, axis=-1)
K.clear_session()
return c
def train_model(X,
y,
mtype,
cv,
epochs,
cv_models_path,
train,
X_test=None,
y_test=None,
nfolds=None,
rs=42,
max_features=40000,
maxlen=400,
dropout_rate=0.25,
rec_units=150,
embed_dim=50,
batch_size=256,
fscore=False,
threshold=0.3):
if cv:
kf = StratifiedKFold(n_splits=nfolds, random_state=rs)
auc = []
roc = []
fscore_ = []
for c, (train_index, val_index) in enumerate(kf.split(X, y)):
print(f' fold {c}')
X_train, X_val = X[train_index], X[val_index]
y_train, y_val = y[train_index], y[val_index]
tokenizer = keras.preprocessing.text.Tokenizer(
num_words=max_features)
tokenizer.fit_on_texts(X_train)
list_tokenized_train = tokenizer.texts_to_sequences(X_train)
list_tokenized_val = tokenizer.texts_to_sequences(X_val)
X_train = sequence.pad_sequences(list_tokenized_train,
maxlen=maxlen)
X_val = sequence.pad_sequences(list_tokenized_val, maxlen=maxlen)
model = dl_model(model_type=mtype,
max_features=max_features,
maxlen=maxlen,
dropout_rate=dropout_rate,
embed_dim=embed_dim,
rec_units=rec_units,
max_sent_len=max_sen_len,
max_sent_amount=max_sent_amount)
print('Fitting')
if train:
model.fit(X_train,
y_train,
batch_size=batch_size,
epochs=epochs,
shuffle=True,
verbose=1)
model.save_weights(f'{cv_models_path}/{mtype}_fold_{c}.h5')
else:
model.load_weights(f'{cv_models_path}/{mtype}_fold_{c}.h5')
probs = model.predict(X_val, batch_size=batch_size, verbose=1)
if fscore:
#for threshold in [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]:
threshold = threshold
probs_class = probs.copy()
probs_class[probs_class >= threshold] = 1
probs_class[probs_class < threshold] = 0
precision = precision_score(y_val, probs_class)
recall = recall_score(y_val, probs_class)
fscore = f1_score(y_val, probs_class)
print(
f' {threshold} fold {c} precision {round(precision, 3)} recall {round(recall, 3)} fscore {round(fscore,3)}'
)
fscore_.append(fscore)
auc_f = average_precision_score(y_val, probs)
auc.append(auc_f)
roc_f = roc_auc_score(y_val, probs)
roc.append(roc_f)
print(
f'fold {c} average precision {round(auc_f, 3)} ++++ roc auc {round(roc_f, 3)}'
)
del model
K.clear_session()
if fscore:
print(
f'PR-C {round(np.array(auc).mean(), 3)} ++++ ROC AUC {round(np.array(roc).mean(), 3)} ++++ FScore {round(np.array(fscore_).mean(), 3)}'
)
print(
f'PR-C std {round(np.array(auc).std(), 3)} ++++ ROC AUC std {round(np.array(roc).std(), 3)} ++++ FScore std {round(np.array(fscore_).std(), 3)}'
)
else:
print(
f'PR-C {round(np.array(auc).mean(), 3)} ++++ ROC AUC {round(np.array(roc).mean(), 3)}'
)
print(
f'PR-C std {round(np.array(auc).std(), 3)} ++++ ROC AUC std {round(np.array(roc).std(), 3)}'
)
else:
X_train = X
y_train = y
tokenizer = keras.preprocessing.text.Tokenizer(num_words=max_features,
oov_token='unknown')
tokenizer.fit_on_texts(X_train)
list_tokenized_train = tokenizer.texts_to_sequences(X_train)
list_tokenized_test = tokenizer.texts_to_sequences(X_test)
X_train = sequence.pad_sequences(list_tokenized_train, maxlen=maxlen)
X_test = sequence.pad_sequences(list_tokenized_test, maxlen=maxlen)
y_train = np.array(y_train)
y_test = np.array(y_test)
model = dl_model(model_type=mtype,
max_features=max_features,
maxlen=maxlen,
dropout_rate=dropout_rate,
embed_dim=embed_dim,
rec_units=rec_units,
max_sent_len=max_sen_len,
max_sent_amount=max_sent_amount)
print('Fitting')
if train:
model.fit(X_train,
y_train,
batch_size=batch_size,
epochs=epochs,
shuffle=True,
verbose=1)
model.save_weights(f'{cv_models_path}/{mtype}.h5')
else:
model.load_weights(f'{cv_models_path}/{mtype}.h5')
probs = model.predict(X_test, batch_size=batch_size, verbose=1)
auc_f = average_precision_score(y_test, probs)
roc_f = roc_auc_score(y_test, probs)
if fscore:
threshold = threshold
probs_class = probs.copy()
probs_class[probs_class >= threshold] = 1
probs_class[probs_class < threshold] = 0
precision = precision_score(y_test, probs_class)
recall = recall_score(y_test, probs_class)
fscore = f1_score(y_test, probs_class)
if fscore:
print('_________________________________')
print(
f'PR-C is {round(auc_f,3)} ++++ ROC AUC is {round(roc_f,3)} +++++ FScore is {round(fscore,3)}'
)
print('_________________________________\n')
else:
print('_________________________________')
print(
f'PR-C is {round(auc_f,3)} ++++ ROC AUC is {round(roc_f,3)}'
)
print('_________________________________\n')
def clear_keras_session():
"""Clears Keras session."""
K.clear_session()
def __call__(self, dag, gpuID, epoch_num=100, out_model='cgpunet.hdf5'):
if self.verbose:
print('GPUID :', gpuID)
print('epoch_num :', epoch_num)
print('batch_size:', self.batchsize)
train_steps = int(self.train_len/self.batchsize)
valid_steps = int(self.valid_len/self.batchsize_valid)
model = dag_2_cnn(dag, gpuID, self.input_shape, self.target_shape)
#print summary
model.summary()
model_checkpoint = ModelCheckpoint(out_model, monitor='loss',verbose=1, save_best_only=True)
history = History()
#NOTE: default values: workers=1, multiprocessing=False.
#TODO: investigate workers>1
history = model.fit_generator(generator=self.trainGenerator, steps_per_epoch=train_steps, epochs=epoch_num, callbacks=[model_checkpoint], validation_data=self.validGenerator, validation_steps=valid_steps, validation_freq= int(epoch_num))
val_acc = history.history['val_accuracy']
#val_loss = history.history['val_loss']
val_precision = history.history['val_precision']
val_recall = history.history['val_recall']
last_epoch = len(val_precision) - 1
val_f1 = 2*((val_precision[last_epoch]*val_recall[last_epoch])/(val_precision[last_epoch]+val_recall[last_epoch]+K.epsilon()))
trainable_count = int(np.sum([K.count_params(p) for p in model.trainable_weights]))
if not os.path.isdir('./figures'):
os.makedirs('./figures')
acc_fig_name = out_model.replace('.hdf5', '_acc.png')
acc_fig_name = './figures/' + acc_fig_name
loss_fig_name = out_model.replace('.hdf5', '_loss.png')
loss_fig_name = './figures/' + loss_fig_name
# Plot training & validation accuracy values
plt.figure()
plt.plot(history.history['accuracy'])
plt.plot(history.history['val_accuracy'])
plt.title('Model accuracy: {}'.format(out_model))
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(['Train', 'Validation'], loc='upper left')
plt.savefig(acc_fig_name)
plt.figure()
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('Model loss": {}'.format(out_model))
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Train', 'Validation'], loc='upper left')
plt.savefig(loss_fig_name)
pickle_name = out_model.replace('.hdf5', '.gpickle')
if not os.path.isdir('./p_files'):
os.makedirs('./p_files')
pickle_name = './p_files/' + pickle_name
nx.write_gpickle(dag, pickle_name)
K.clear_session()
return (float(val_f1), trainable_count)
