def finetune(model, output_path, log_dir):
for layer in model.layers[:nb_freezed_layers]:
layer.trainable = False
for layer in model.layers[nb_freezed_layers:]:
layer.trainable = True
model.compile(optimizer=SGD(lr=learning_rate, momentum=0.9),
loss='categorical_crossentropy',
metrics=['accuracy'])
train_gen = data_generator(train_data_dir, augmentation=True)
val_gen = data_generator(val_data_dir)
tb = TensorBoard(log_dir=log_dir, write_graph=False)
es = EarlyStopping(monitor=monitor, patience=patiance)
mc = ModelCheckpoint(output_path, monitor=monitor, save_best_only=True)
model.fit_generator(train_gen,
samples_per_epoch=nb_train_samples,
nb_epoch=nb_epoch,
callbacks=[tb, es, mc],
validation_data=val_gen,
nb_val_samples=nb_val_samples)
return model
def evaluate_ubuntu(config, model, test_size=500000):
assert test_size % config.test_batch_size == 0
predict_result = model.predict_generator(
generator=data_generator(config=config, is_train=False),
steps=test_size / config.test_batch_size)
example_id_list, label_list = load_test_label(config)
acc_right_num = acc_all_num = 0
prediction_dict = {}
for example_id, prediction, label in zip(example_id_list, predict_result,
label_list):
if example_id not in prediction_dict:
prediction_dict[example_id] = [(prediction, label)]
else:
prediction_dict[example_id].append((prediction, label))
if (prediction > 0.5 and label == 1) or (prediction < 0.5
and label == 0):
acc_right_num += 1
acc_all_num += 1
print(
f"acc evaluate acc_right_num: {acc_right_num}\tacc_all_num: {acc_all_num}\tacc: "
f"{acc_right_num * 1.0 / acc_all_num}")
recall_ten_at_one = recall_ten_at_two = recall_ten_at_fiv = recall_two_at_one = sample_all_num = 0
for example_id in prediction_dict:
prediction_list = prediction_dict[example_id]
label_pred = prediction_list[0][0]
sec_pred = prediction_list[1][0]
if label_pred > sec_pred:
recall_two_at_one += 1
sorted_list = sorted(prediction_list, key=lambda x: x[0], reverse=True)
sorted_label_list = [y for x, y in sorted_list]
if sorted_label_list[0] == 1:
recall_ten_at_one += 1
recall_ten_at_two += 1
recall_ten_at_fiv += 1
elif 1 in sorted_label_list[:2]:
recall_ten_at_two += 1
recall_ten_at_fiv += 1
elif 1 in sorted_label_list[:5]:
recall_ten_at_fiv += 1
else:
pass
sample_all_num += 1
recall_two_at_one = recall_two_at_one * 1.0 / sample_all_num
recall_ten_at_one = recall_ten_at_one * 1.0 / sample_all_num
recall_ten_at_two = recall_ten_at_two * 1.0 / sample_all_num
recall_ten_at_fiv = recall_ten_at_fiv * 1.0 / sample_all_num
print(f"rank evaluate sample_all_num: {sample_all_num}\trecall_two_at_one: {recall_two_at_one}\trecall_ten_at_one: {recall_ten_at_one}\t" \
f"recall_ten_at_two: {recall_ten_at_two}\trecall_ten_at_fiv: {recall_ten_at_fiv}")
return recall_ten_at_one
def predict(model, result_dir):
test_gen = data_generator(test_data_dir, shuffle=False)
proba = model.predict_generator(test_gen, nb_test_samples)
proba_df = DataFrame(proba, index=test_gen.filenames)
proba_df.to_csv(os.path.join(result_dir, 'proba.csv'))
proba_df.idxmax(axis=1).to_csv(os.path.join(result_dir, 'pred.csv'))
def train_top(model, output_path=None):
model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
train_gen = data_generator(train_data_dir)
val_gen = data_generator(val_data_dir)
model.fit_generator(train_gen,
samples_per_epoch=nb_train_samples,
nb_epoch=1,
validation_data=val_gen,
nb_val_samples=nb_val_samples)
if output_path is not None:
model.save(output_path)
return model
def get_generator(trainingdir, spa, params):
encoder = sampling_encoder(params)
sampled_dsets, event_dsets = read_files(trainingdir, params)
for sampled_dset, event_dset in zip(sampled_dsets, event_dsets):
data_gen = data_generator(
spa,
sampled_dset.data,
window_len=params['window_len'],
labels=event_dset.data,
encoder=encoder,
batch_size=1, # this is required for saving images
amplitude_norm=params['amplitude_norm'],
loop=False)
yield from data_gen
def get_generators(dataset_name, dataset_videos, datasets_frames, fix_len, figure_size, force, classes=1, use_aug=False,
use_crop=True, crop_dark=True):
train_path, valid_path, test_path, \
train_y, valid_y, test_y, \
avg_length = preprocess.createDataset(dataset_videos, dataset_name, datasets_frames, fix_len, force=force)
if fix_len is not None:
avg_length = fix_len
crop_x_y = None
if (crop_dark):
crop_x_y = (11, 38)
batch_size=2
len_train, len_valid = len(train_path), len(valid_path)
train_gen = preprocess.data_generator(train_path, train_y, batch_size, figure_size, avg_length, use_aug=use_aug,
use_crop=use_crop, crop_x_y=crop_x_y, classes=classes)
validate_gen = preprocess.data_generator(valid_path, valid_y, batch_size, figure_size, avg_length,
use_aug=False, use_crop=False, crop_x_y=crop_x_y, classes=classes)
test_x, test_y = preprocess.get_sequences(test_path, test_y, figure_size, avg_length, crop_x_y=crop_x_y,
classes=classes)
return train_gen, validate_gen, test_x, test_y, avg_length, len_train, len_valid
def predict(model, sampled_dset, spa, p):
# note: loop must be set to true to enable using keras's predict_generator
data_gen = data_generator(spa,
sampled_dset.data,
window_len=p['window_len'],
encoder=p['encoder'],
batch_size=p['predict_batch_size'],
amplitude_norm=p['amplitude_norm'],
loop=True)
n_step_size = spa._NFFT - spa._noverlap
n_targets = np.ceil(len(sampled_dset.data) / n_step_size)
n_steps = np.ceil(n_targets / p['predict_batch_size'])
print(n_steps)
print(sampled_dset.name)
y_est = model.predict_generator(data_gen, n_steps, verbose=1)
# save outputs
basename = '{}_{}'.format(
os.path.splitext(sampled_dset.path)[0], p['model'])
save(y_est, basename, p)
def test(model, sampled_dset, event_dset, spa, p):
data_gen = data_generator(spa,
sampled_dset.data,
window_len=p['window_len'],
labels=event_dset.data,
encoder=p['encoder'],
batch_size=p['batch_size'],
amplitude_norm=p['amplitude_norm'],
loop=True)
n_step_size = spa._NFFT - spa._noverlap
n_targets = np.ceil(len(sampled_dset.data) / n_step_size)
n_steps = np.ceil(n_targets / p['batch_size'])
print(n_steps)
print(sampled_dset.name)
y_est = model.predict_generator(data_gen, n_steps, verbose=1)
y_true = all_targets_from_events(event_dset.data, n_targets, n_step_size,
p['encoder'], spa._rate)
print("accuracy score:",
accuracy_score(np.argmax(y_true, 1), np.argmax(y_est, 1)))
print(classification_report(np.argmax(y_true, 1), np.argmax(y_est, 1)))
# save outputs
basename = '{}_{}'.format(
os.path.splitext(sampled_dset.path)[0], p['model'])
save(y_est, basename, p, y_true)
def evaluate_douban(config, model, test_size=10000):
assert test_size % config.test_batch_size == 0
predict_result = model.predict_generator(generator=data_generator(config=config, is_train=False), \
steps=test_size / config.test_batch_size)
example_id_list, label_list = load_test_label(config)
prediction_dict = {}
for example_id, prediction, label in zip(example_id_list, predict_result,
label_list):
if example_id not in prediction_dict:
prediction_dict[example_id] = [(prediction, label)]
else:
prediction_dict[example_id].append((prediction, label))
# del some invalid example
del_num = 0
filtered_prediction_dict = {}
for example_id in prediction_dict.keys():
temp_list = prediction_dict[example_id]
if len(temp_list) != 10:
print(len(temp_list))
print(example_id)
print('ERROR')
print('############')
label0_num = 0
label1_num = 0
for temp in temp_list:
if temp[1] == 0:
label0_num += 1
if temp[1] == 1:
label1_num += 1
if label0_num == 10 or label1_num == 10:
del_num += 1
else:
filtered_prediction_dict[example_id] = temp_list
print(f'there are {del_num} example have been delete')
# now calculate each metrics
mrr_list = []
map_list = []
recall_1 = 0
recall_2 = 0
recall_5 = 0
p1 = 0
example_count = 0
for example_id in filtered_prediction_dict.keys():
prediction_list = filtered_prediction_dict[example_id]
# (score, label)
prediction_list = sorted(prediction_list,
key=lambda x: x[0],
reverse=True)
total_positive = 0
for prediction in prediction_list:
if prediction[1] == 1:
total_positive += 1
if prediction_list[0][1] == 1:
p1 += 1
recall_1 += 1 * 1.0 / total_positive
correct = 0
for i in range(2):
if prediction_list[i][1] == 1:
correct += 1
recall_2 += correct * 1.0 / total_positive
correct = 0
for i in range(5):
if prediction_list[i][1] == 1:
correct += 1
recall_5 += correct * 1.0 / total_positive
for i in range(len(prediction_list)):
if prediction_list[i][1] == 1:
mrr_list.append(1 * 1.0 / (i + 1))
break
correct_count = 1
one_map_list = []
for i in range(len(prediction_list)):
if prediction_list[i][1] == 1:
one_map_list.append(correct_count * 1.0 / (i + 1))
correct_count += 1
map_list.append(sum(one_map_list) / total_positive)
example_count += 1
MRR = sum(mrr_list) / example_count
MAP = sum(map_list) / example_count
P1 = p1 / example_count
R10_1 = recall_1 / example_count
R10_2 = recall_2 / example_count
R10_5 = recall_5 / example_count
print(
f'rank evaluate total:{example_count}\tMRR:{MRR}\tMAP:{MAP}\tP1:{P1}\tR10@1:{R10_1}\tR10@2:{R10_2}\tR10@5:{R10_5}'
)
return MRR
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--train_file",
default='./data/ubuntu/train_shuf.txt',
type=str,
help='train file')
parser.add_argument("--test_file",
default='./data/ubuntu/test.txt',
type=str,
help='valid or test file')
parser.add_argument('--output_dir',
default='./model/',
type=str,
help='a path to output')
parser.add_argument('--config_file',
default=None,
type=str,
help='config file')
parser.add_argument(
'--model_weight_file',
default='./model/model_epoch7_seed19565_prec0.79006.hdf5',
type=str)
parser.add_argument('--stopword_file',
default='./data/ubuntu/stopwords.txt',
type=str)
parser.add_argument('--train_ubuntu',
action='store_true',
help='Whether to run training on ubuntu dataset')
parser.add_argument('--evaluate_ubuntu',
action='store_true',
help='Whether to run evaluate on ubuntu dataset')
parser.add_argument('--train_douban',
action='store_true',
help='Whether to run training on douban dataset')
parser.add_argument('--evaluate_douban',
action='store_true',
help='Whether to run evaluate on douban dataset')
parser.add_argument('--gpu_nums',
default=2,
type=int,
help='How many gpu will use')
parser.add_argument('--train_batch_size', default=64, type=int)
parser.add_argument('--test_batch_size', default=100, type=int)
parser.add_argument('--use_CuDNNRNN',
action='store_true',
help='Whether use CuDNNGRU or CuDNNLSTM')
parser.add_argument('--random_seed',
default=10000,
type=int,
help='random seed')
parser.add_argument('--use_word_embeddings', action='store_true')
parser.add_argument('--word_vocab_file',
default='./data/ubuntu/vocab_ubuntu',
type=str)
parser.add_argument('--word_embeddings_file',
default='./data/ubuntu/glove_ubuntu.txt',
type=str)
parser.add_argument('--word_embeddings_dim', default=200, type=int)
parser.add_argument('--word_vocab_size', default=297989, type=int)
parser.add_argument('--max_utterance_num', default=10, type=int)
parser.add_argument('--max_utterance_len', default=50, type=int)
parser.add_argument('--hidden_dim', default=200, type=int)
parser.add_argument('--epochs', default=10, type=int)
parser.add_argument('--dropout_rate', default=0.3, type=float)
parser.add_argument('--use_char_embeddings', action='store_true')
parser.add_argument('--char_vocab',
default='./data/ubuntu/char_vocab',
type=str)
parser.add_argument('--char_vocab_size', default=300, type=int)
parser.add_argument('--char_embeddings_dim', default=64, type=int)
parser.add_argument('--max_token_len', default=15, type=int)
parser.add_argument('--char_features_dim', default=200, type=int)
parser.add_argument('--char_kernel_shape', default=3, type=int)
args = parser.parse_args()
if args.config_file:
with codecs.open(args.config_file, 'r', encoding='utf-8') as f:
settings = json.load(f)
for k, v in settings:
if k not in ['config_file']:
args.__dict__[k] = v
# check args validation
experiment_on_ubuntu = args.train_ubuntu or args.evaluate_ubuntu
experiment_on_douban = args.train_douban or args.evaluate_douban
if experiment_on_ubuntu and experiment_on_douban:
raise AssertionError(
"You have to do an experiment in one dataset at the same time!")
if not experiment_on_ubuntu and not experiment_on_douban:
raise AssertionError('Must do something')
if os.path.exists(args.output_dir) and os.listdir(args.output_dir):
logger.info("Output directory () already exists and is not empty.")
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir, exist_ok=True)
if not args.use_word_embeddings and not args.use_char_embeddings:
raise AssertionError('At least specified one input!')
args.task = 'ubuntu' if experiment_on_ubuntu else 'douban'
# set seed
np.random.seed(args.random_seed)
random.seed(args.random_seed)
print(args)
if args.train_ubuntu or args.train_douban:
# init model
with tf.device('/cpu:0'):
model = TripleNetModel(args).model
single_model = model
if args.gpu_nums > 1:
model = multi_gpu_model(model, args.gpu_nums)
model.compile(loss='binary_crossentropy',
optimizer='Adamax',
metrics=['accuracy'])
model_save_callback = SaveModelCallback(args, single_model)
print('Training model.....')
model.fit_generator(generator=data_generator(args, True),
steps_per_epoch=1000000 / args.train_batch_size,
epochs=args.epochs,
callbacks=[model_save_callback])
if args.evaluate_ubuntu:
with tf.device('/cpu:0'):
model = TripleNetModel(args).model
model.load_weights(args.model_weight_file)
if args.gpu_nums > 1:
model = multi_gpu_model(model, args.gpu_nums)
evaluate_ubuntu(args, model)
if args.evaluate_douban:
with tf.device('/cpu:0'):
model = TripleNetModel(args).model
model.load_weights(args.model_weight_file)
if args.gpu_nums > 1:
model = multi_gpu_model(model, args.gpu_nums)
evaluate_douban(args, model)