def image_def(image: str, model=None):
"""
Returns the image as Clothing so it can be used for matching
:param image: the str with the location of the file
:param model: the test model, if one is not provided it will be generated in function
:return: the src file classified as a clothing item
"""
class_names = [
'T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal',
'Shirt', 'Sneaker', 'Bag', 'Ankle boot'
]
if model is None:
model = build_model()
__image = rgb2gray(io.imread(image))
__image = transform.resize(__image, (1, 28, 28), mode='symmetric')
__image = model.predict(__image)
predict = np.argmax(__image)
clothing_type = class_names[predict]
if isinstance(image, str):
_image = PIL.Image.open(image)
else:
_image = image
pattern = detect_pattern(_image)
color = detect_clothing_color(_image)
return Clothing(clothing_type, pattern, color)
def first_run():
model = model_builder.build_model()
model = model_trainer.train_model(model, x_train, y_train)
result = model_tester.test_model(model, x_test, y_test)
model.save("./nn_models/mnist_nn_v1.h5")
def _build_model(audio_file,train_wav_file,name_surname,duration, token):
classpath= os.path.join(settings.BASE_DIR, 'plugins_script',"speaker_extractor","lium_spkdiarization-8.4.1.jar" )
ubm= os.path.join(settings.MEDIA_ROOT,"models/audio/globals/ubm.gmm")
start=0
audio_file="/tmp/prova.wav"
gmm_path= build_model (classpath, ubm,audio_file, start, duration, train_wav_file, name_surname, token)
return gmm_path
def _build_model(audio_file,train_wav_file,name_surname,duration, token):
classpath= os.path.join(get_base_dir(), 'plugins_script',"speaker_extractor","lium_spkdiarization-8.4.1.jar" )
ubm= os.path.join(get_media_root(),"models/audio/globals/ubm.gmm")
start=0
audio_file="/tmp/prova.wav"
gmm_path= build_model (classpath, ubm,audio_file, start, duration, train_wav_file, name_surname, token)
return gmm_path
def run(params):
"""Run training
Args:
params: Parameters for training
Returns: None
"""
logging.info(f'params: {params}')
strategy = distribution_utils.get_distribution_strategy(
params.get('tpu_address'))
batch_size = distribution_utils.update_batch_size(strategy,
params['batch_size'])
with strategy.scope():
model = model_builder.build_model()
input_image_size = model.input_shape[1]
# Build dataset
train_image_paths, train_scores = data_loader.read_csv(params['train_csv'],
params['image_dir'],
is_training=True)
validation_image_paths, validation_scores = data_loader.read_csv(
params['validation_csv'], params['image_dir'], is_training=False)
train_dataset = data_loader.build_dataset(train_image_paths,
train_scores,
is_training=True,
batch_size=batch_size,
target_size=input_image_size)
validation_dataset = data_loader.build_dataset(
validation_image_paths,
validation_scores,
is_training=False,
batch_size=batch_size,
target_size=input_image_size)
train_dataset = strategy.experimental_distribute_dataset(train_dataset)
validation_dataset = strategy.experimental_distribute_dataset(
validation_dataset)
loss_fn = loss_builder.build_loss_fn(
loss_name=params['loss'],
trainable_variables=model.trainable_variables)
train(model=model,
loss_fn=loss_fn,
strategy=strategy,
epochs=params['epochs'],
batch_size=batch_size,
train_dataset=train_dataset,
validation_dataset=validation_dataset,
checkpoint_dir=params['checkpoint_dir'],
log_dir=params['log_dir'])
def build_model(self):
if self.input_pix is None:
input_shape = (256, 256, 1)
output_shape = (256, 256, 1)
else:
input_shape = self.input_pix.shape[1:]
output_shape = self.output_pix.shape[1:]
self.model = build_model(input_shape, output_shape)
self.model.summary()
optimizer = Adam(lr=1e-3)
self.model.compile(loss='binary_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
def train_model(opt, logger):
logger.info('---START---')
# initialize for reproduce
np.random.seed(opt.seed)
# load data
logger.info('---LOAD DATA---')
opt, training, training_snli, validation, test_matched, test_mismatched = load_data(opt)
if not opt.skip_train:
logger.info('---TRAIN MODEL---')
for train_counter in range(opt.max_epochs):
if train_counter == 0:
model = build_model(opt)
else:
model = load_model_local(opt)
np.random.seed(train_counter)
lens = len(training_snli[-1])
perm = np.random.permutation(lens)
idx = perm[:int(lens * 0.2)]
train_data = [np.concatenate((training[0], training_snli[0][idx])),
np.concatenate((training[1], training_snli[1][idx])),
np.concatenate((training[2], training_snli[2][idx]))]
csv_logger = CSVLogger('{}{}.csv'.format(opt.log_dir, opt.model_name), append=True)
cp_filepath = opt.save_dir + "cp-" + opt.model_name + "-" + str(train_counter) + "-{val_acc:.2f}.h5"
cp = ModelCheckpoint(cp_filepath, monitor='val_acc', save_best_only=True, save_weights_only=True)
callbacks = [cp, csv_logger]
model.fit(train_data[:-1], train_data[-1], batch_size=opt.batch_size, epochs=1, validation_data=(validation[:-1], validation[-1]), callbacks=callbacks)
save_model_local(opt, model)
else:
logger.info('---LOAD MODEL---')
model = load_model_local(opt)
# predict
logger.info('---TEST MODEL---')
preds_matched = model.predict(test_matched[:-1], batch_size=128, verbose=1)
preds_mismatched = model.predict(test_mismatched[:-1], batch_size=128, verbose=1)
save_preds_matched_to_csv(preds_matched, test_mismatched[-1], opt)
save_preds_mismatched_to_csv(preds_mismatched, test_mismatched[-1], opt)
from model_builder import build_model
for n in range(0, k):
# god I love python
trainFolds = folds[:n] + folds[n + 1:]
train = []
for fold in trainFolds:
train.extend(fold)
test = folds[n]
trainGen = listGenerator(train)
testGen = listGenerator(test)
model = build_model(input_length, line_length, "distcnn", "malware_only")
print("Compiling Model and Training, test fold =", n)
print()
model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
print(model.summary())
csv_logger = CSVLogger(r".\networks\kfold" + str(n) + r".log")
filepath = r".\networks\kfold" + str(n) + r"-{epoch:02d}.hdf5"
checkpoint = ModelCheckpoint(filepath)
model.fit_generator(
def train(cfg):
# startup_prog = fluid.Program()
# train_prog = fluid.Program()
drop_last = True
dataset = SegDataset(
file_list=cfg.DATASET.TRAIN_FILE_LIST,
mode=ModelPhase.TRAIN,
shuffle=True,
data_dir=cfg.DATASET.DATA_DIR)
def data_generator():
if args.use_mpio:
data_gen = dataset.multiprocess_generator(
num_processes=cfg.DATALOADER.NUM_WORKERS,
max_queue_size=cfg.DATALOADER.BUF_SIZE)
else:
data_gen = dataset.generator()
batch_data = []
for b in data_gen:
batch_data.append(b)
if len(batch_data) == (cfg.BATCH_SIZE // cfg.NUM_TRAINERS):
for item in batch_data:
yield item[0], item[1], item[2]
batch_data = []
# If use sync batch norm strategy, drop last batch if number of samples
# in batch_data is less then cfg.BATCH_SIZE to avoid NCCL hang issues
if not cfg.TRAIN.SYNC_BATCH_NORM:
for item in batch_data:
yield item[0], item[1], item[2]
# Get device environment
# places = fluid.cuda_places() if args.use_gpu else fluid.cpu_places()
# place = places[0]
gpu_id = int(os.environ.get('FLAGS_selected_gpus', 0))
place = fluid.CUDAPlace(gpu_id) if args.use_gpu else fluid.CPUPlace()
places = fluid.cuda_places() if args.use_gpu else fluid.cpu_places()
# Get number of GPU
dev_count = cfg.NUM_TRAINERS if cfg.NUM_TRAINERS > 1 else len(places)
print_info("#Device count: {}".format(dev_count))
# Make sure BATCH_SIZE can divided by GPU cards
assert cfg.BATCH_SIZE % dev_count == 0, (
'BATCH_SIZE:{} not divisble by number of GPUs:{}'.format(
cfg.BATCH_SIZE, dev_count))
# If use multi-gpu training mode, batch data will allocated to each GPU evenly
batch_size_per_dev = cfg.BATCH_SIZE // dev_count
print_info("batch_size_per_dev: {}".format(batch_size_per_dev))
data_loader, loss, lr, pred, grts, masks, image = build_model(
phase=ModelPhase.TRAIN)
data_loader.set_sample_generator(
data_generator, batch_size=batch_size_per_dev, drop_last=drop_last)
exe = fluid.Executor(place)
cfg.update_from_file(args.teacher_cfg_file)
# teacher_arch = teacher_cfg.architecture
teacher_program = fluid.Program()
teacher_startup_program = fluid.Program()
with fluid.program_guard(teacher_program, teacher_startup_program):
with fluid.unique_name.guard():
_, teacher_loss, _, _, _, _, _ = build_model(
teacher_program,
teacher_startup_program,
phase=ModelPhase.TRAIN,
image=image,
label=grts,
mask=masks)
exe.run(teacher_startup_program)
teacher_program = teacher_program.clone(for_test=True)
ckpt_dir = cfg.SLIM.KNOWLEDGE_DISTILL_TEACHER_MODEL_DIR
assert ckpt_dir is not None
print('load teacher model:', ckpt_dir)
if os.path.exists(ckpt_dir):
try:
fluid.load(teacher_program, os.path.join(ckpt_dir, 'model'), exe)
except:
fluid.io.load_params(exe, ckpt_dir, main_program=teacher_program)
# cfg = load_config(FLAGS.config)
cfg.update_from_file(args.cfg_file)
data_name_map = {
'image': 'image',
'label': 'label',
'mask': 'mask',
}
merge(teacher_program, fluid.default_main_program(), data_name_map, place)
distill_pairs = [[
'teacher_bilinear_interp_2.tmp_0', 'bilinear_interp_0.tmp_0'
]]
def distill(pairs, weight):
"""
Add 3 pairs of distillation losses, each pair of feature maps is the
input of teacher and student's yolov3_loss respectively
"""
loss = l2_loss(pairs[0][0], pairs[0][1])
weighted_loss = loss * weight
return weighted_loss
distill_loss = distill(distill_pairs, 0.1)
cfg.update_from_file(args.cfg_file)
optimizer = solver.Solver(None, None)
all_loss = loss + distill_loss
lr = optimizer.optimise(all_loss)
exe.run(fluid.default_startup_program())
exec_strategy = fluid.ExecutionStrategy()
# Clear temporary variables every 100 iteration
if args.use_gpu:
exec_strategy.num_threads = fluid.core.get_cuda_device_count()
exec_strategy.num_iteration_per_drop_scope = 100
build_strategy = fluid.BuildStrategy()
build_strategy.fuse_all_reduce_ops = False
build_strategy.fuse_all_optimizer_ops = False
build_strategy.fuse_elewise_add_act_ops = True
if cfg.NUM_TRAINERS > 1 and args.use_gpu:
dist_utils.prepare_for_multi_process(exe, build_strategy,
fluid.default_main_program())
exec_strategy.num_threads = 1
if cfg.TRAIN.SYNC_BATCH_NORM and args.use_gpu:
if dev_count > 1:
# Apply sync batch norm strategy
print_info("Sync BatchNorm strategy is effective.")
build_strategy.sync_batch_norm = True
else:
print_info(
"Sync BatchNorm strategy will not be effective if GPU device"
" count <= 1")
compiled_train_prog = fluid.CompiledProgram(
fluid.default_main_program()).with_data_parallel(
loss_name=all_loss.name,
exec_strategy=exec_strategy,
build_strategy=build_strategy)
# Resume training
begin_epoch = cfg.SOLVER.BEGIN_EPOCH
if cfg.TRAIN.RESUME_MODEL_DIR:
begin_epoch = load_checkpoint(exe, fluid.default_main_program())
# Load pretrained model
elif os.path.exists(cfg.TRAIN.PRETRAINED_MODEL_DIR):
load_pretrained_weights(exe, fluid.default_main_program(),
cfg.TRAIN.PRETRAINED_MODEL_DIR)
else:
print_info(
'Pretrained model dir {} not exists, training from scratch...'.
format(cfg.TRAIN.PRETRAINED_MODEL_DIR))
#fetch_list = [avg_loss.name, lr.name]
fetch_list = [
loss.name, 'teacher_' + teacher_loss.name, distill_loss.name, lr.name
]
if args.debug:
# Fetch more variable info and use streaming confusion matrix to
# calculate IoU results if in debug mode
np.set_printoptions(
precision=4, suppress=True, linewidth=160, floatmode="fixed")
fetch_list.extend([pred.name, grts.name, masks.name])
cm = ConfusionMatrix(cfg.DATASET.NUM_CLASSES, streaming=True)
if args.use_vdl:
if not args.vdl_log_dir:
print_info("Please specify the log directory by --vdl_log_dir.")
exit(1)
from visualdl import LogWriter
log_writer = LogWriter(args.vdl_log_dir)
# trainer_id = int(os.getenv("PADDLE_TRAINER_ID", 0))
# num_trainers = int(os.environ.get('PADDLE_TRAINERS_NUM', 1))
step = 0
all_step = cfg.DATASET.TRAIN_TOTAL_IMAGES // cfg.BATCH_SIZE
if cfg.DATASET.TRAIN_TOTAL_IMAGES % cfg.BATCH_SIZE and drop_last != True:
all_step += 1
all_step *= (cfg.SOLVER.NUM_EPOCHS - begin_epoch + 1)
avg_loss = 0.0
avg_t_loss = 0.0
avg_d_loss = 0.0
best_mIoU = 0.0
timer = Timer()
timer.start()
if begin_epoch > cfg.SOLVER.NUM_EPOCHS:
raise ValueError(
("begin epoch[{}] is larger than cfg.SOLVER.NUM_EPOCHS[{}]").format(
begin_epoch, cfg.SOLVER.NUM_EPOCHS))
if args.use_mpio:
print_info("Use multiprocess reader")
else:
print_info("Use multi-thread reader")
for epoch in range(begin_epoch, cfg.SOLVER.NUM_EPOCHS + 1):
data_loader.start()
while True:
try:
if args.debug:
# Print category IoU and accuracy to check whether the
# traning process is corresponed to expectation
loss, lr, pred, grts, masks = exe.run(
program=compiled_train_prog,
fetch_list=fetch_list,
return_numpy=True)
cm.calculate(pred, grts, masks)
avg_loss += np.mean(np.array(loss))
step += 1
if step % args.log_steps == 0:
speed = args.log_steps / timer.elapsed_time()
avg_loss /= args.log_steps
category_acc, mean_acc = cm.accuracy()
category_iou, mean_iou = cm.mean_iou()
print_info((
"epoch={} step={} lr={:.5f} loss={:.4f} acc={:.5f} mIoU={:.5f} step/sec={:.3f} | ETA {}"
).format(epoch, step, lr[0], avg_loss, mean_acc,
mean_iou, speed,
calculate_eta(all_step - step, speed)))
print_info("Category IoU: ", category_iou)
print_info("Category Acc: ", category_acc)
if args.use_vdl:
log_writer.add_scalar('Train/mean_iou', mean_iou,
step)
log_writer.add_scalar('Train/mean_acc', mean_acc,
step)
log_writer.add_scalar('Train/loss', avg_loss, step)
log_writer.add_scalar('Train/lr', lr[0], step)
log_writer.add_scalar('Train/step/sec', speed, step)
sys.stdout.flush()
avg_loss = 0.0
cm.zero_matrix()
timer.restart()
else:
# If not in debug mode, avoid unnessary log and calculate
loss, t_loss, d_loss, lr = exe.run(
program=compiled_train_prog,
fetch_list=fetch_list,
return_numpy=True)
avg_loss += np.mean(np.array(loss))
avg_t_loss += np.mean(np.array(t_loss))
avg_d_loss += np.mean(np.array(d_loss))
step += 1
if step % args.log_steps == 0 and cfg.TRAINER_ID == 0:
avg_loss /= args.log_steps
avg_t_loss /= args.log_steps
avg_d_loss /= args.log_steps
speed = args.log_steps / timer.elapsed_time()
print((
"epoch={} step={} lr={:.5f} loss={:.4f} teacher loss={:.4f} distill loss={:.4f} step/sec={:.3f} | ETA {}"
).format(epoch, step, lr[0], avg_loss, avg_t_loss,
avg_d_loss, speed,
calculate_eta(all_step - step, speed)))
if args.use_vdl:
log_writer.add_scalar('Train/loss', avg_loss, step)
log_writer.add_scalar('Train/lr', lr[0], step)
log_writer.add_scalar('Train/speed', speed, step)
sys.stdout.flush()
avg_loss = 0.0
avg_t_loss = 0.0
avg_d_loss = 0.0
timer.restart()
except fluid.core.EOFException:
data_loader.reset()
break
except Exception as e:
print(e)
if (epoch % cfg.TRAIN.SNAPSHOT_EPOCH == 0
or epoch == cfg.SOLVER.NUM_EPOCHS) and cfg.TRAINER_ID == 0:
ckpt_dir = save_checkpoint(fluid.default_main_program(), epoch)
if args.do_eval:
print("Evaluation start")
_, mean_iou, _, mean_acc = evaluate(
cfg=cfg,
ckpt_dir=ckpt_dir,
use_gpu=args.use_gpu,
use_mpio=args.use_mpio)
if args.use_vdl:
log_writer.add_scalar('Evaluate/mean_iou', mean_iou, step)
log_writer.add_scalar('Evaluate/mean_acc', mean_acc, step)
if mean_iou > best_mIoU:
best_mIoU = mean_iou
update_best_model(ckpt_dir)
print_info("Save best model {} to {}, mIoU = {:.4f}".format(
ckpt_dir,
os.path.join(cfg.TRAIN.MODEL_SAVE_DIR, 'best_model'),
mean_iou))
# Use VisualDL to visualize results
if args.use_vdl and cfg.DATASET.VIS_FILE_LIST is not None:
visualize(
cfg=cfg,
use_gpu=args.use_gpu,
vis_file_list=cfg.DATASET.VIS_FILE_LIST,
vis_dir="visual",
ckpt_dir=ckpt_dir,
log_writer=log_writer)
if cfg.TRAINER_ID == 0:
ckpt_dir = save_checkpoint(fluid.default_main_program(), epoch)
# save final model
if cfg.TRAINER_ID == 0:
save_checkpoint(fluid.default_main_program(), 'final')
def query(model_name, experiment_name, inputs, target):
MODEL_NAME = experiment_name
description = model_builder.load_description(MODEL_PATH, MODEL_NAME)
net = model_builder.build_model(model_name, description)
net.load(MODEL_PATH, MODEL_NAME, description)
# net.model.summary()
# print net.model.get_layer(name="embedding_2").get_weights()[0]
print net.role_vocabulary
print("unk_word_id", net.unk_word_id)
print("missing_word_id", net.missing_word_id)
# net.set_0_bias()
net.model.summary()
propbank_map = {
"subj" : "A0",
"obj" : "A1",
"ARG0" : "A0",
"ARG1" : "A1",
"ARG2" : "A2",
}
# tr_map = {
# "A0": numpy.asarray([[net.role_vocabulary["A0"]]], dtype=numpy.int64),
# "A1": numpy.asarray([[net.role_vocabulary["A1"]]], dtype=numpy.int64),
# "A2": numpy.asarray([[net.role_vocabulary["<UNKNOWN>"]]], dtype=numpy.int64)
# }
# net.word_vocabulary["<NOTHING>"] = net.missing_word_id
# net.role_vocabulary["<UNKNOWN>"] = net.unk_role_id
reverse_vocabulary = utils.get_reverse_map(net.word_vocabulary)
reverse_role_vocabulary = utils.get_reverse_map(net.role_vocabulary)
print reverse_role_vocabulary
raw_words = dict((reverse_role_vocabulary[r], reverse_vocabulary[net.missing_word_id]) for r in net.role_vocabulary.values())
# print raw_words
raw_words.update(inputs)
# print raw_words
# print len(raw_words)
assert len(raw_words) == len(net.role_vocabulary)
# print repr(raw_words)
# n = int(sys.argv[3])
t_r = [net.role_vocabulary.get(r, net.unk_role_id) for r in target.keys()]
t_w = [net.word_vocabulary.get(w, net.unk_word_id) for w in target.values()]
input_roles_words = {}
for r, w in raw_words.items():
input_roles_words[net.role_vocabulary[r]] = utils.input_word_index(net.word_vocabulary, w, net.unk_word_id, warn_unk=True)
print input_roles_words, t_r
input_roles_words.pop(t_r[0])
# default_roles_words = dict((r, net.missing_word_id) for r in (net.role_vocabulary.values()))
# default_roles_words.update(input_roles_words)
# input_roles_words = default_roles_words
x_w_i = numpy.asarray([input_roles_words.values()], dtype=numpy.int64)
x_r_i = numpy.asarray([input_roles_words.keys()], dtype=numpy.int64)
y_w_i = numpy.asarray(t_w, dtype=numpy.int64)
y_r_i = numpy.asarray(t_r, dtype=numpy.int64)
topN=20
predicted_word_indices = net.top_words(x_w_i, x_r_i, y_w_i, y_r_i, topN)
# print predicted_word_indices
# print len(predicted_word_indices)
print(x_w_i, x_r_i, y_w_i, y_r_i)
p_w = net.p_words(x_w_i, x_r_i, y_w_i, y_r_i, batch_size=1, verbose=0)[0]
print ('p_t_w: ', p_w)
resultlist = predicted_word_indices
# print resultlist
for i, t_w_i in enumerate(resultlist):
t_w = net.word_vocabulary.get(t_w_i, net.unk_word_id)
y_w_i = numpy.asarray([t_w_i], dtype=numpy.int64)
p = net.p_words(x_w_i, x_r_i, y_w_i, y_r_i, batch_size=1, verbose=0)[0]
n = numpy.round(p / 0.005)
fb = numpy.floor(n)
hb = n % 2
print u"{:<5} {:7.6f} {:<20} ".format(i+1, float(p), reverse_vocabulary[int(t_w_i)]) + u"\u2588" * int(fb) + u"\u258C" * int(hb)
def _prepare_model(self, config):
print('[Logging] preparing model...')
self.model = build_model(self._model_name, config['models'], self._cuda, self.data_utils)
self.lr = config['lr']
def eval_bicknell_switch(model_name,
experiment_name,
evaluation,
model=None,
print_result=True,
switch_test=False):
MODEL_NAME = experiment_name
if model:
net = model
else:
description = model_builder.load_description(MODEL_PATH, MODEL_NAME)
net = model_builder.build_model(model_name, description)
net.load(MODEL_PATH, MODEL_NAME, description)
bias = net.set_0_bias()
if print_result:
print net.role_vocabulary
eval_data_file = os.path.join(RF_EVAL_PATH, evaluation + '.txt')
result_file = os.path.join(MODEL_PATH,
MODEL_NAME + '_' + evaluation + '.txt')
probs = []
baseline = []
oov_count = 0
if print_result:
print eval_data_file
print "=" * 60
dataset = numpy.genfromtxt(eval_data_file,
dtype=str,
delimiter='\t',
usecols=[0, 1, 2, 3, 4])
samples = []
i = 0
while True:
d = dataset[i]
d2 = dataset[i + 1]
A0 = d[0][:-2]
V = d[1][:-2]
assert d2[0][:-2] == A0
assert d2[1][:-2] == V
if d[3] == 'yes':
assert d2[3] == 'no'
A1_correct = d[2][:-2]
A1_incorrect = d2[2][:-2]
b_correct = d[4]
b_incorrect = d2[4]
else:
assert d[3] == 'no'
A1_correct = d2[2][:-2]
A1_incorrect = d[2][:-2]
b_correct = d2[4]
b_incorrect = d[4]
if A1_correct not in net.word_vocabulary or A1_incorrect not in net.word_vocabulary:
if A1_correct not in net.word_vocabulary and print_result:
print "%s MISSING FROM VOCABULARY. SKIPPING..." % A1_correct
if A1_incorrect not in net.word_vocabulary and print_result:
print "%s MISSING FROM VOCABULARY. SKIPPING..." % A1_incorrect
else:
roles = net.role_vocabulary.values()
del roles[net.unk_role_id]
input_roles_words = dict((r, net.missing_word_id) for r in (roles))
input_roles_words[
net.role_vocabulary["A0"]] = utils.input_word_index(
net.word_vocabulary, A0, net.unk_word_id, warn_unk=True)
input_roles_words[
net.role_vocabulary["V"]] = utils.input_word_index(
net.word_vocabulary, V, net.unk_word_id, warn_unk=True)
sample = (
numpy.asarray(
[input_roles_words.values(),
input_roles_words.values()],
dtype=numpy.int64), # x_w_i
numpy.asarray(
[input_roles_words.keys(),
input_roles_words.keys()],
dtype=numpy.int64), # x_r_i
numpy.asarray([
net.word_vocabulary[A1_correct],
net.word_vocabulary[A1_incorrect]
],
dtype=numpy.int64
), # y_i (1st is correct and 2nd is incorrect
numpy.asarray(
[net.role_vocabulary["A1"], net.role_vocabulary["A1"]],
dtype=numpy.int64), # y_r_i
[b_correct, b_incorrect], # bicknell scores
"\"" + A0 + " " + V + "\"", # context
[A1_correct, A1_incorrect])
samples.append(sample)
i += 2
if i > len(dataset) - 2:
break
num_samples = len(samples)
num_correct = 0
num_total = 0
if print_result:
print "context", "correct", "incorrect", "P(correct)", "P(incorrect)", "bicnell_correct", "bicnell_incorrect"
result_list = []
for x_w_i, x_r_i, y_w_i, y_r_i, bicknell, context, a1 in samples:
p = net.p_words(x_w_i, x_r_i, y_w_i, y_r_i)
p_correct = p[0]
p_incorrect = p[1]
if print_result:
print context, a1[0], a1[1], p_correct, p_incorrect, bicknell[
0], bicknell[1]
if p_correct > p_incorrect:
result_list.append(1)
else:
result_list.append(0)
num_correct += p_correct > p_incorrect
num_total += 1
assert num_total == num_samples
accuracy = float(num_correct) / float(num_samples)
if print_result:
print "Number of lines %d" % num_samples
print "Baseline Lenci11 is 43/64=0.671875"
print "Final score of theano model is %d/%d=%.6f" % (
num_correct, num_samples, accuracy)
print result_list
if switch_test and print_result:
print "\nSwitch A0/A1 TEST"
input_words = []
input_roles = []
for i in range(1):
roles = net.role_vocabulary.values()
print net.unk_role_id
roles.remove(net.unk_role_id)
input_role_word_pairs = dict(
(r, net.missing_word_id) for r in roles)
input_role_word_pairs[
net.role_vocabulary["V"]] = utils.input_word_index(
net.word_vocabulary, "buy", net.unk_word_id, warn_unk=True)
input_words.append(input_role_word_pairs.values())
input_roles.append(input_role_word_pairs.keys())
man = utils.input_word_index(net.word_vocabulary,
"man",
net.unk_word_id,
warn_unk=True)
car = utils.input_word_index(net.word_vocabulary,
"car",
net.unk_word_id,
warn_unk=True)
a1 = net.role_vocabulary["A1"]
a0 = net.role_vocabulary["A0"]
a0_test = (
numpy.asarray(input_words, dtype=numpy.int64),
numpy.asarray(input_roles, dtype=numpy.int64),
numpy.asarray([man, car], dtype=numpy.int64),
numpy.asarray([a0], dtype=numpy.int64),
)
x_w_i, x_r_i, y_w_i, y_r_i = a0_test
p0 = net.p_words(x_w_i, x_r_i, y_w_i, y_r_i)
print p0
a1_test = (
numpy.asarray(input_words, dtype=numpy.int64),
numpy.asarray(input_roles, dtype=numpy.int64),
numpy.asarray([man, car], dtype=numpy.int64),
numpy.asarray([a1], dtype=numpy.int64),
)
x_w_i, x_r_i, y_w_i, y_r_i = a1_test
p1 = net.p_words(x_w_i, x_r_i, y_w_i, y_r_i)
print p1
print "man buy", p0[0]
print "buy man", p1[0]
print "car buy", p0[1]
print "buy car", p1[1]
net.set_bias(bias)
return num_correct, num_samples, accuracy
type=str,
default="cifar10")
args = parser.parse_args()
#fixing errors which occur on HPC
try:
_create_unverified_https_context = ssl._create_unverified_context
except AttributeError:
# Legacy Python that doesn't verify HTTPS certificates by default
pass
else:
# Handle target environment that doesn't support HTTPS verification
ssl._create_default_https_context = _create_unverified_https_context
model = build_model(th=0.1,
dv=args.device_variation,
std_dev=args.device_std_dev,
add_zero_pad=(not args.dataset == "cifar10"))
opt = tf.keras.optimizers.SGD(learning_rate=1)
model.compile(optimizer=opt,
loss='categorical_crossentropy',
metrics=['accuracy'])
if ("mnist" in args.dataset):
model.build(input_shape=tf.TensorShape([None, 28, 28, 1]))
else:
model.build(input_shape=tf.TensorShape([None, 32, 32, 3]))
print(model.summary())
dataset, dataset_test = build_dataset(args.dataset)
loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=False)
base_path = os.path.join(
os.path.abspath("./results/"), args.prefix,
def main(opt, device_id, batch_queue=None, semaphore=None):
# NOTE: It's important that ``opt`` has been validated and updated
# at this point.
configure_process(opt, device_id)
init_logger(opt.log_file)
assert len(opt.accum_count) == len(opt.accum_steps), \
'Number of accum_count values must match number of accum_steps'
# Load checkpoint if we resume from a previous training.
if opt.train_from:
logger.info('Loading checkpoint from %s' % opt.train_from)
checkpoint = torch.load(opt.train_from,
map_location=lambda storage, loc: storage)
model_opt = ArgumentParser.ckpt_model_opts(checkpoint["opt"])
ArgumentParser.update_model_opts(model_opt)
ArgumentParser.validate_model_opts(model_opt)
logger.info('Loading vocab from checkpoint at %s.' % opt.train_from)
vocab = checkpoint['vocab']
else:
checkpoint = None
model_opt = opt
vocab = torch.load(opt.data + '.vocab.pt')
if opt.teacher_model_path:
logger.info('Loading teacher model from {path}'.format(
path=opt.teacher_model_path))
teacher_model_ckpt = torch.load(
opt.teacher_model_path, map_location=lambda storage, loc: storage)
teacher_model_opt = ArgumentParser.ckpt_model_opts(
teacher_model_ckpt['opt'])
ArgumentParser.update_model_opts(teacher_model_opt)
ArgumentParser.validate_model_opts(teacher_model_opt)
logger.info('Loading vocab from checkpoint at {path}'.format(
path=opt.teacher_model_path))
teacher_vocab = teacher_model_ckpt['vocab']
# check for code where vocab is saved instead of fields
# (in the future this will be done in a smarter way)
if old_style_vocab(vocab):
fields = load_old_vocab(vocab,
opt.model_type,
dynamic_dict=opt.copy_attn)
else:
fields = vocab
teacher_fields = teacher_vocab if opt.teacher_model_path else None
# patch for fields that may be missing in old data/model
# patch_fields(opt, fields)
# Report src and tgt vocab sizes, including for features
report_vocab_size(fields)
if teacher_fields is not None:
report_vocab_size(teacher_fields)
# Build model.
fields_opt = {"original": fields, "teacher": teacher_fields}
model = custom_builder.build_model(model_opt, opt, fields_opt, checkpoint)
# model = build_model(model_opt, opt, fields, checkpoint)
teacher_model = build_model(
teacher_model_opt, teacher_model_opt, teacher_fields,
teacher_model_ckpt) if opt.teacher_model_path else None
n_params, enc, dec = _tally_parameters(model)
logger.info('encoder: %d' % enc)
logger.info('decoder: %d' % dec)
logger.info('* number of parameters: %d' % n_params)
_check_save_model_path(opt)
if teacher_model is not None:
n_params, enc, dec = _tally_parameters(teacher_model)
logger.info('encoder: %d' % enc)
logger.info('decoder: %d' % dec)
logger.info('* number of parameters: %d' % n_params)
_check_save_model_path(teacher_model_opt)
# Build optimizer.
optim = Optimizer.from_opt(model, opt, checkpoint=checkpoint)
# Build model saver
# model_saver = build_model_saver(model_opt, opt, model, fields, optim)
model_saver = custom_model_saver.build_model_saver(model_opt, opt, model,
fields_opt, optim)
tgt_field = dict(teacher_fields)["tgt"].base_field if teacher_model is not None \
else dict(fields)["tgt"].base_field
sos_id = tgt_field.vocab.stoi[tgt_field.init_token]
if teacher_model is not None and opt.word_sampling:
sampler = Emulator(teacher_model,
teacher_fields,
device_id,
max_length=50,
random_sampling_topk=5)
else:
sampler = None
if teacher_model is not None:
trainer = build_trainer(opt,
device_id,
model,
teacher_fields,
optim,
model_saver,
teacher_model=teacher_model,
emulator=sampler)
else:
trainer = build_trainer(opt,
device_id,
model,
fields,
optim,
model_saver,
teacher_model=teacher_model,
emulator=sampler)
if batch_queue is None:
if len(opt.data_ids) > 1:
train_shards = []
for train_id in opt.data_ids:
shard_base = "train_" + train_id
train_shards.append(shard_base)
train_iter = build_dataset_iter_multiple(train_shards, fields, opt)
else:
if opt.data_ids[0] is not None:
shard_base = "train_" + opt.data_ids[0]
else:
shard_base = "train"
train_iter = build_dataset_iter(shard_base, fields, opt)
else:
assert semaphore is not None, \
"Using batch_queue requires semaphore as well"
def _train_iter():
while True:
batch = batch_queue.get()
semaphore.release()
yield batch
train_iter = _train_iter()
valid_iter = build_dataset_iter("valid", fields, opt, is_train=False)
if len(opt.gpu_ranks):
logger.info('Starting training on GPU: %s' % opt.gpu_ranks)
else:
logger.info('Starting training on CPU, could be very slow')
train_steps = opt.train_steps
if opt.single_pass and train_steps > 0:
logger.warning("Option single_pass is enabled, ignoring train_steps.")
train_steps = 0
trainer.train(train_iter,
train_steps,
sos_id=sos_id,
save_checkpoint_steps=opt.save_checkpoint_steps,
valid_iter=valid_iter,
valid_steps=opt.valid_steps)
if trainer.report_manager.tensorboard_writer is not None:
trainer.report_manager.tensorboard_writer.close()
def evaluate(model_name,
experiment_name,
test_name,
batch_size,
VR_SP_SRL=True,
bootstrapping=False,
majority_baseline=False):
MODEL_NAME = experiment_name
# repr_file = os.path.join(MODEL_PATH, 'confusionM_' + MODEL_NAME)
description = model_builder.load_description(MODEL_PATH, MODEL_NAME)
net = model_builder.build_model(model_name, description)
net.load(MODEL_PATH, MODEL_NAME, description)
n_roles = len(net.role_vocabulary)
reverse_word_vocabulary = utils.get_reverse_map(net.word_vocabulary)
reverse_role_vocabulary = utils.get_reverse_map(net.role_vocabulary)
# net.set_0_bias()
print net.role_vocabulary
print("unk_word_id", net.unk_word_id)
print("missing_word_id", net.missing_word_id)
net.model.summary()
# print net.model.metrics_names
test_sample_size = 0
with open(EVAL_PATH + test_name, 'r') as lines:
for l in lines:
test_sample_size += 1
print(test_sample_size)
test_steps = test_sample_size / float(batch_size)
# test_steps = test_sample_size
# # DEBUG
# test_steps = 10
print 'Testing ' + test_name + ' ...'
print 'VR_SP_SRL: ' + str(VR_SP_SRL)
test_start = time.clock()
# if re.search('NNRF_1e8', experiment_name) or re.search('MTRF_dev', experiment_name):
# test_gen = get_minibatch(DATA_PATH + "NN_test", net.unk_word_id, net.unk_role_id, net.missing_word_id,
# n_roles, random=False, batch_size=batch_size)
# else:
# test_gen = generator(DATA_PATH + "NN_test", model_name, net.unk_word_id, net.unk_role_id, net.missing_word_id,
# n_roles, random=False, batch_size=batch_size)
# # Test the model
# test_result = net.model.evaluate_generator(
# generator = test_gen,
# steps = test_steps,
# max_q_size = 1,
# workers = 1,
# pickle_safe = False
# )
# print ('test_result', test_result)
# Compute confusion matrix
metrics_names = net.model.metrics_names
result_dict = {(x, 0) for x in metrics_names}
batch_n = 0
confusionM = np.zeros((n_roles, n_roles), dtype='int32')
ppl_role_list = dict()
result_list = []
output_list = []
for ([i_w, i_r, t_w, t_r], _) in data_gen(EVAL_PATH + test_name,
model_name,
net,
batch_size,
VR_SP_SRL=VR_SP_SRL):
# zeros = np.zeros(t_r.shape)
result_role = net.predict_role(i_w, i_r, t_w, t_r, batch_size)
# word_emb, avg_emb, event_emb = net.avg_emb.predict([i_w, i_r, t_w, t_r], batch_size)
# print word_emb.shape, avg_emb.shape, event_emb.shape
# assert np.multiply(word_emb[0][0], avg_emb[0])[0] == event_emb[0][0][0]
# assert np.multiply(word_emb[0][0], avg_emb[0])[1] == event_emb[0][0][1]
# test role prediction of MTRF_dev, result: role prediction is useless
# print i_r
# print t_r.reshape(-1)
# print result_role
# result_word_likelihood = net.predict(i_w, i_r, t_w, t_r, batch_size)[0]
# neg_log_likelihoods = -np.log(result_word_likelihood)
# for i, row in enumerate(neg_log_likelihoods, start=0):
# target_word = t_w[i][0]
# target_role = t_r[i][0]
# neg_log_likelihood = row[target_word]
# ppl_role_list.setdefault(target_role, []).append(neg_log_likelihood)
# print i_w, i_r, t_w, t_r
for i, true_r in enumerate(t_r, start=0):
# if reverse_role_vocabulary.get(t_r[0][0], '<unknown>') == 'AM-LOC':
# print ("input words", [reverse_word_vocabulary.get(w, '<unknown>') for w in i_w[0]])
# print ("input roles", [reverse_role_vocabulary.get(r, '<unknown>') for r in i_r[0]])
# print ("target word", [reverse_word_vocabulary.get(w, '<unknown>') for w in t_w[0]])
# print ("target role", [reverse_role_vocabulary.get(r, '<unknown>') for r in t_r[0]])
# print ("predicted role", [reverse_role_vocabulary.get(result_role[i], '<unknown>') for r in t_r[0]])
# print ''
confusionM[true_r, result_role[i]] += 1
if true_r == result_role[i]:
result_list.append(1)
output_list.append((true_r, result_role[i]))
batch_n += 1
if batch_n % 100 == 0:
print(batch_n)
if batch_n >= test_steps:
break
# ppl_role = dict()
# for k, v in ppl_role_list.items():
# neg_log_likelihood_role = np.mean(np.array(v))
# ppl_role[k] = np.exp(neg_log_likelihood_role)
# obtain ZeroR baseline
print confusionM
majority = 1
if majority_baseline == True:
for i in range(7):
confusionM[i][majority] = confusionM[i][:].sum()
confusionM[i][majority - 1] = 0
confusionM[i][majority + 1:] = 0
print confusionM
dir_P, dir_R, dir_F1, precision, recall, F1 = stats(net, confusionM)
print "Dir: %.2f \t %.2f \t %.2f" % (dir_P, dir_R, dir_F1)
# np.savetxt('confusionM_' + experiment_name + '.' + test_name.strip('.dat') + '.csv', confusionM, delimiter = ',')
# np.savetxt('output_' + experiment_name + '.' + test_name.strip('.dat') + '.csv', output_list, delimiter = ',')
# with open(repr_file, 'w') as f_out:
# f_out.write('[')
# for i in range(n_roles):
# f_out.write('[')
# for j in range(n_roles):
# f_out.write(str(confusionM[i][j]) + ", ")
# f_out.write('] \n')
# f_out.write(']')
# print "Loss(neg_log_likelihood) by role: "
# for r in ppl_role.keys():
# print (reverse_role_vocabulary[r], np.log(ppl_role[r]))
print("Result by role: ")
for r in range(len(precision)):
print('%s: \t %.2f \t %.2f \t %.2f' %
(reverse_role_vocabulary[r], precision[r], recall[r], F1[r]))
test_end = time.clock()
print 'test time: %f, sps: %f' % (test_end - test_start, test_steps *
batch_size / (test_end - test_start))
if bootstrapping:
P_mean, P_std, R_mean, R_std, F1_mean, F1_std = bootstrap(
experiment_name, test_name, net, n_roles, output_list=output_list)
return P_mean, P_std, R_mean, R_std, F1_mean, F1_std
val_ds = dataset_builder_fn(config["dataset"]["val_file_name"])
test_ds = dataset_builder_fn(config["dataset"]["test_file_name"])
decayed_lr = tf.train.cosine_decay(
learning_rate=config["train"]["learning_rate"],
global_step=global_step,
decay_steps=config['train']['decay_step'],
alpha=config['train']['decay_alpha'])
optimizer = tf.train.AdamOptimizer(decayed_lr)
anchor_num_per_output = len(config["anchor"]["scales"]) * len(
config["anchor"]["aspect_ratio"])
od_model = model_builder.build_model(config["dataset"]["num_classes"],
anchor_num_per_output)
if config["save"]["load_model"]:
od_model.load_weights(config["save"]["model_dir"])
compute_loss_fn = functools.partial(
model_builder.compute_loss,
num_classes=config["dataset"]["num_classes"],
c_weight=config["train"]["classificaiton_loss_weight"],
r_weight=config["train"]["regression_loss_weight"],
neg_label_value=config["dataset"]["neg_label_value"],
ignore_label_value=config["dataset"]["ignore_label_value"],
negative_ratio=config["train"]["negative_ratio"])
train_loss_sum = 0
train_c_loss_sum = 0
def eval_MNR_LOC(model_name,
experiment_name,
evaluation,
model=None,
print_result=True,
skip_header=False):
MODEL_NAME = experiment_name
if model:
net = model
else:
description = model_builder.load_description(MODEL_PATH, MODEL_NAME)
net = model_builder.build_model(model_name, description)
net.load(MODEL_PATH, MODEL_NAME, description)
bias = net.set_0_bias()
if print_result:
print net.role_vocabulary
tr_map = {
"ARG2": "A2",
"ARG3": "A3",
"ARGM-MNR": "AM-MNR",
"ARGM-LOC": "AM-LOC",
}
if evaluation == "AM-MNR":
eval_data_file = os.path.join(RV_EVAL_PATH,
'McRaeInstr-fixed' + '.txt')
remove_suffix = False
elif evaluation == 'AM-LOC':
eval_data_file = os.path.join(RV_EVAL_PATH, 'McRaeLoc-fixed' + '.txt')
remove_suffix = True
else:
sys.exit('No such evaluation!!!')
result_file = os.path.join(MODEL_PATH,
MODEL_NAME + '_' + evaluation + '.txt')
probs = []
baseline = []
oov_count = 0
r_i = net.role_vocabulary["V"]
if print_result:
print eval_data_file, evaluation
print "=" * 60
with open(eval_data_file, 'r') as f, \
open(result_file, 'w') as f_out:
for i, line in enumerate(f):
if i == 0 and skip_header:
# print line.strip() + "\tP(instrument|verb)"
continue #skip header
line = line.strip()
w, tw, temp1, temp2 = line.split(
)[:4] # input word, target word, other stuff
w = w[:-2] if remove_suffix else w
tw = tw[:-2] if remove_suffix else tw
w = wnl.lemmatize(w.lower(), wn.VERB)
tw = wnl.lemmatize(tw.lower(), wn.NOUN)
w_i = net.word_vocabulary.get(w, net.unk_word_id)
tw_i = net.word_vocabulary.get(tw, net.unk_word_id)
if evaluation == "AM-MNR":
r = temp2
else:
r = temp1
# tr_i = net.role_vocabulary.get(evaluation, net.unk_role_id)
tr_i = net.role_vocabulary.get(tr_map[r], net.unk_role_id)
y_r_i = numpy.asarray([tr_i], dtype=numpy.int64)
if tw_i == net.unk_word_id:
oov_count += 1
print w, tw
f_out.write(line + "\tnan\n")
continue
b = float(line.split()[-1 if remove_suffix else -2])
baseline.append(b)
input_roles_words = dict(
(r, net.missing_word_id)
for r in (net.role_vocabulary.values() + [net.unk_role_id]))
input_roles_words[r_i] = w_i
input_roles_words.pop(tr_i, None)
x_w_i = numpy.asarray([input_roles_words.values()],
dtype=numpy.int64)
x_r_i = numpy.asarray([input_roles_words.keys()],
dtype=numpy.int64)
y_w_i = numpy.asarray([tw_i], dtype=numpy.int64)
p = net.p_words(x_w_i, x_r_i, y_w_i, y_r_i)
# pr = net.p_roles(x_w_i, x_r_i, y_w_i, y_r_i)
probs.append(p)
f_out.write(line + "\t%s\n" % p)
rho, p_value = spearmanr(baseline, probs)
rating = len(probs)
if print_result:
print "Spearman correlation: %f; 2-tailed p-value: %f" % (rho, p_value)
print "Num ratings: %d (%d out of vocabulary)" % (rating, oov_count)
net.set_bias(bias)
return rho, p_value, oov_count, probs, baseline
def pd_themfit(model_name,
experiment_name,
df,
predict_role='V',
input_roles='all_available_args',
function='filler_prob',
n=5,
debug=False):
""" Adds a column to a pandas df with a role filler probability.
For each row in the pandas df, calculates the probability that a particular role filler will fill a
particular role, given a set of input roles and fillers (from that row).
Keyword arguments:
model_name -- The name of the model
experiment_name -- The name of the model plus the name of the experiment, separated by '_'
df -- The pandas dataframe. Must include columns for all propbank labels in predict_role and input_roles
predict_role -- the target role (in propbank labels) for which the filler will be predicted (default: 'V')
input_roles -- the set of roles (in propbank labels) that should be used as inputs (default: 'all_args')
"""
possible_roles = set(
['A0', 'A1', 'AM-LOC', 'AM-TMP', 'AM-MNR', '<UNKNOWN>', 'V'])
try:
assert predict_role in df.columns
assert predict_role in possible_roles
if input_roles != 'all_available_args':
for r in input_roles:
assert r in df.columns
assert r in possible_roles
except:
print("NOT ALL ROLES ARE AVAILABLE AS DF COLUMNS")
MODEL_NAME = experiment_name
description = model_builder.load_description(MODEL_PATH, MODEL_NAME)
net = model_builder.build_model(model_name, description)
net.load(MODEL_PATH, MODEL_NAME, description)
bias = net.set_0_bias()
# net.model.summary()
# print net.model.get_layer(name="embedding_2").get_weights()[0]
# If no <UNKNOWN> role in the role vocabulary, add it.
if net.role_vocabulary.get("<UNKNOWN>", -1) == -1:
net.role_vocabulary["<UNKNOWN>"] = len(net.role_vocabulary) - 1
print("Role vocabulary", net.role_vocabulary)
print("unk_word_id", net.unk_word_id)
print("missing_word_id", net.missing_word_id)
reverse_vocabulary = utils.get_reverse_map(net.word_vocabulary)
reverse_role_vocabulary = utils.get_reverse_map(net.role_vocabulary)
print("Reverse role vocabulary", reverse_role_vocabulary)
raw_words = dict(
(reverse_role_vocabulary[r], reverse_vocabulary[net.missing_word_id])
for r in net.role_vocabulary.values())
if input_roles == 'all_available_args':
possible_roles.remove(predict_role)
input_roles = possible_roles.intersection(set(df.columns))
all_roles = input_roles
all_roles.add(predict_role)
df = df.apply(
lambda x: process_row(predict_role=predict_role,
role_fillers={i: x[i]
for i in all_roles},
model=net,
raw_word_list=raw_words,
function=function,
n=n,
debug=debug),
axis=1)
return df
def eval_greenberg(model_name,
experiment_name,
evaluation,
model=None,
print_result=True):
MODEL_NAME = experiment_name
if model:
net = model
else:
description = model_builder.load_description(MODEL_PATH, MODEL_NAME)
net = model_builder.build_model(model_name, description)
net.load(MODEL_PATH, MODEL_NAME, description)
bias = net.set_0_bias()
if print_result:
print(net.role_vocabulary) # Updated to python3 syntax (team1-change)
eval_data_file = os.path.join(RV_EVAL_PATH, evaluation + '.txt')
result_file = os.path.join(MODEL_PATH,
MODEL_NAME + '_' + evaluation + '.txt')
probs = []
baseline = []
oov_count = 0
tr = "A1"
r_i = net.role_vocabulary["V"]
tr_i = net.role_vocabulary["A1"]
y_r_i = numpy.asarray([tr_i], dtype=numpy.int64)
if print_result:
print(eval_data_file) # Updated to python3 syntax (team1-change)
print("=" * 60) # Updated to python3 syntax (team1-change)
with open(eval_data_file, 'r') as f, \
open(result_file, 'w') as f_out:
for line in f:
line = line.strip().lower()
w, tw = line.split()[:2] # input word, target word, other stuff
w = w[:-2].strip()
tw = tw[:-2].strip()
w = wnl.lemmatize(w, wn.VERB)
tw = wnl.lemmatize(tw, wn.NOUN)
# a hack to fix some words
# tw = word_fix.get(tw, tw)
w_i = net.word_vocabulary.get(w, net.unk_word_id)
tw_i = net.word_vocabulary.get(tw, net.unk_word_id)
if tw_i == net.unk_word_id:
oov_count += 1
print(w, tr, tw) # Updated to python3 syntax (team1-change)
f_out.write(line + "\tnan\n")
continue
b = float(line.split()[-1])
sample = dict(
(r, net.missing_word_id)
for r in (net.role_vocabulary.values() + [net.unk_role_id]))
sample[r_i] = w_i
sample.pop(tr_i, None)
x_w_i = numpy.asarray([sample.values()], dtype=numpy.int64)
x_r_i = numpy.asarray([sample.keys()], dtype=numpy.int64)
y_w_i = numpy.asarray([tw_i], dtype=numpy.int64)
p = net.p_words(x_w_i, x_r_i, y_w_i, y_r_i)
# pr = net.p_roles(x_w_i, x_r_i, y_w_i, y_r_i)
if tw_i == net.unk_word_id:
print('OOV: %s' % w, b, p)
baseline.append(b)
probs.append(p)
f_out.write(line + "\t%s\n" % p)
rho, p_value = spearmanr(baseline, probs)
if print_result:
print(
f"Spearman correlation of {evaluation}: {rho}; 2-tailed p-value: {p_value}"
) # Updated to python3 syntax and f-string (team1-change)
print(f"Num ratings: {len(probs)} ({oov_count} out of vocabulary)"
) # Updated to python3 syntax and f-string (team1-change)
net.set_bias(bias)
return rho, p_value, oov_count, probs, baseline
input_data_list, label_data_list = dp.get_all_data()
print(f"train data size: {len(input_data_list)}")
print(f"sample input_data shape: {input_data_list[0].shape}")
print(f"sample label data: {label_data_list[0]}")
valdp = ExistingDataProvider(val_imgdir, val_annotdir, model_input_size)
val_data = valdp.get_all_data()
print(f"validation data size: {len(val_data[0])}")
model = build_model()
# model.summary()
modeljson = model.to_json()
model_arch_save_path = os.path.join(ckpt_dirpath, "model_arch.json")
with open(model_arch_save_path,'w') as fd:
fd.write(modeljson)
print("start model compile")
# model.compile(optimizer = tf.optimizers.Adam(0.001), loss = "mse")
metric_list=[
tf.keras.losses.MSE
def main(config: DictConfig) -> None:
warnings.filterwarnings('ignore')
print(OmegaConf.to_yaml(config))
torch.manual_seed(config.train.seed)
torch.cuda.manual_seed_all(config.train.seed)
np.random.seed(config.train.seed)
random.seed(config.train.seed)
use_cuda = config.train.cuda and torch.cuda.is_available()
device = torch.device('cuda' if use_cuda else 'cpu')
char2id, id2char = load_label(config.train.label_path,
config.train.blank_id)
train_audio_paths, train_transcripts, valid_audio_paths, valid_transcripts = load_dataset(
config.train.dataset_path, config.train.mode)
train_dataset = SpectrogramDataset(
config.train.audio_path,
train_audio_paths,
train_transcripts,
config.audio.sampling_rate,
config.audio.n_mfcc
if config.audio.feature_extraction == 'mfcc' else config.audio.n_mel,
config.audio.frame_length,
config.audio.frame_stride,
config.audio.extension,
config.audio.feature_extraction,
config.audio.normalize,
config.audio.spec_augment,
config.audio.freq_mask_parameter,
config.audio.num_time_mask,
config.audio.num_freq_mask,
config.train.sos_id,
config.train.eos_id,
)
train_sampler = BucketingSampler(train_dataset,
batch_size=config.train.batch_size)
train_loader = AudioDataLoader(
train_dataset,
batch_sampler=train_sampler,
num_workers=config.train.num_workers,
)
valid_dataset = SpectrogramDataset(
config.train.audio_path,
valid_audio_paths,
valid_transcripts,
config.audio.sampling_rate,
config.audio.n_mfcc
if config.audio.feature_extraction == 'mfcc' else config.audio.n_mel,
config.audio.frame_length,
config.audio.frame_stride,
config.audio.extension,
config.audio.feature_extraction,
config.audio.normalize,
config.audio.spec_augment,
config.audio.freq_mask_parameter,
config.audio.num_time_mask,
config.audio.num_freq_mask,
config.train.sos_id,
config.train.eos_id,
)
valid_sampler = BucketingSampler(valid_dataset,
batch_size=config.train.batch_size)
valid_loader = AudioDataLoader(
valid_dataset,
batch_sampler=valid_sampler,
num_workers=config.train.num_workers,
)
model = build_model(config, device)
optimizer = optim.Adam(model.parameters(), lr=config.train.lr)
print('Start Train !!!')
for epoch in range(0, config.train.epochs):
train(config, model, device, train_loader, valid_loader, train_sampler,
optimizer, epoch, id2char, epoch)
if epoch % 2 == 0:
torch.save(
model,
os.path.join(os.getcwd(), config.train.model_save_path +
str(epoch) + '.pt'))
def eval_GS(model_name,
experiment_name,
eval_file_name,
model=None,
print_result=True,
verb_baseline=False):
MODEL_NAME = experiment_name
eval_file = os.path.join(EVAL_PATH, eval_file_name)
result_file = os.path.join(MODEL_PATH, MODEL_NAME + '_' + eval_file_name)
if model:
net = model
else:
description = model_builder.load_description(MODEL_PATH, MODEL_NAME)
net = model_builder.build_model(model_name, description)
net.load(MODEL_PATH, MODEL_NAME, description)
sent_layer = 'context_embedding'
sent_model = Model(inputs=net.model.input,
outputs=net.model.get_layer(sent_layer).output)
# if print_result:
# sent_model.summary()
n_input_length = len(net.role_vocabulary) - 1
print net.role_vocabulary
scores = []
similarities = []
original_sim_f = []
similarities_f = []
lo_similarities = []
hi_similarities = []
records = []
print("Embedding: " + experiment_name)
print("=" * 60)
print("\n")
print("sentence1\tsentence2\taverage_score\tembedding_cosine")
print("-" * 60)
with open(eval_file, 'r') as f, \
open(result_file, 'w') as f_out:
first = True
for line in f:
# skip header
if first:
first = False
continue
s = line.split()
sentence = " ".join(s[1:5])
score = float(s[5])
hilo = s[6].upper()
# verb subject object landmark
# A1 - object; A0 - subject
V1, A0, A1, V2 = sentence.split()
V1 = wnl.lemmatize(V1, wn.VERB)
A0 = wnl.lemmatize(A0, wn.NOUN)
A1 = wnl.lemmatize(A1, wn.NOUN)
V2 = wnl.lemmatize(V2, wn.VERB)
V1_i = net.word_vocabulary.get(V1, net.unk_word_id)
A0_i = net.word_vocabulary.get(A0, net.unk_word_id)
A1_i = net.word_vocabulary.get(A1, net.unk_word_id)
V2_i = net.word_vocabulary.get(V2, net.unk_word_id)
# if np.array([V1_i, A0_i, A1_i, V2_i]).any() == net.unk_word_id:
# print 'OOV: ', A0, A1, V1, V2
V_ri = net.role_vocabulary['V']
A0_ri = net.role_vocabulary['A0']
A1_ri = net.role_vocabulary['A1']
sent1_x = dict((r, net.missing_word_id)
for r in (net.role_vocabulary.values()))
sent2_x = dict((r, net.missing_word_id)
for r in (net.role_vocabulary.values()))
sent1_x.pop(n_input_length)
sent2_x.pop(n_input_length)
sent1_x[V_ri] = V1_i
sent2_x[V_ri] = V2_i
if not verb_baseline:
sent1_x[A0_ri] = A0_i
sent1_x[A1_ri] = A1_i
sent2_x[A0_ri] = A0_i
sent2_x[A1_ri] = A1_i
zeroA = np.array([0])
s1_w = np.array(sent1_x.values()).reshape((1, n_input_length))
s1_r = np.array(sent1_x.keys()).reshape((1, n_input_length))
s2_w = np.array(sent2_x.values()).reshape((1, n_input_length))
s2_r = np.array(sent2_x.keys()).reshape((1, n_input_length))
if re.search('NNRF', model_name):
sent1_emb = sent_model.predict([s1_w, s1_r, zeroA])
sent2_emb = sent_model.predict([s2_w, s2_r, zeroA])
else:
sent1_emb = sent_model.predict([s1_w, s1_r, zeroA, zeroA])
sent2_emb = sent_model.predict([s2_w, s2_r, zeroA, zeroA])
# Baseline
#sent1_emb = V1_i
#sent2_emb = V2_i
# Compositional
# sent1_emb = V1_i + A0_i + A1_i
# sent2_emb = V2_i + A0_i + A1_i
#sent1_emb = V1_i * A0_i * A1_i
#sent2_emb = V2_i * A0_i * A1_i
similarity = -(cosine(sent1_emb, sent2_emb) - 1.0
) # convert distance to similarity
if hilo == "HIGH":
hi_similarities.append(similarity)
elif hilo == "LOW":
lo_similarities.append(similarity)
else:
raise Exception("Unknown hilo value %s" % hilo)
if (V1, A0, A1, V2) not in records:
records.append((V1, A0, A1, V2))
# print "\"%s %s %s\"\t\"%s %s %s\"\t%.2f\t%.2f \n" % (A0, V1, A1, A0, V2, A1, score, similarity)
scores.append(score)
similarities.append(similarity)
f_out.write("\"%s %s %s\"\t\"%s %s %s\"\t %.2f \t %.2f \n" %
(A0, V1, A1, A0, V2, A1, score, similarity))
print("-" * 60)
correlation, pvalue = spearmanr(scores, similarities)
if print_result:
print("Total number of samples: %d" % len(scores)
) #Added paranthesis to the print statements (team1-change)
print("Spearman correlation: %.4f; 2-tailed p-value: %.10f" %
(correlation, pvalue)
) #Added paranthesis to the print statements (team1-change)
print("High: %.2f; Low: %.2f" %
(np.mean(hi_similarities), np.mean(lo_similarities))
) #Added paranthesis to the print statements (team1-change)
# import pylab
# pylab.scatter(scores, similarities)
# pylab.show()
return correlation
annotations_test = tools.load_annotations('test', opts)
# Create data loaders:
data_train = data_loader(annotations_train, opts)
data_val = data_loader(annotations_val, opts)
data_test = data_loader(annotations_test, opts)
# Create the computational graph, or import it:
if opts.restore_model:
# Load meta-graph:
checkpoint = tools.get_checkpoint(opts)
metamodel_file = checkpoint + '.meta'
saver = tf.train.import_meta_graph(metamodel_file)
else:
# Create network and loss:
gradients = model_builder.build_model(opts)
saver = tf.train.Saver()
# Start clock:
start = time.time()
with tf.Session(config=tf_config) as sess:
if opts.restore_model:
checkpoint = tools.get_checkpoint(opts)
saver.restore(sess, checkpoint)
else:
sess.run(tf.global_variables_initializer())
# Train:
if opts.train:
# batch_size = 1
steps_per_epoch = num_samples/batch_size
valid_steps = num_samples_valid/batch_size # should be this
epochs = 100
from model_builder import build_model
# model = build_model(input_length, line_length, "cnn", "malware_only")
# model = build_model(input_length, line_length, "convlstm", "malware_only")
# model = build_model(input_length, line_length, "minconvrnn", "malware_only")
# model = build_model(input_length, line_length, "distcnn", "malware_only")
model = build_model(input_length, line_length, "distcnn", "binary")
# model = build_model(input_length, line_length, "distcnn", "binary", extras=["noembed"])
# model = load_model(r"D:\Research\3_Kaggle\kaggle_networks\93. NOPOOL net based on 79\KaggleConv-15.hdf5",
# model = load_model(r".\networks\rnn binary final nets\addconv\KaggleConv-09.hdf5",
# custom_objects={'DecayingConvLSTM2D':MinConvRNN,
# 'window_size': window_size ,
# 'ELU': elu,
# }
# )
print("Compiling Model and Training")
print()
def eval_pado_mcrae(model_name, experiment_name, evaluation, model=None, print_result=True):
MODEL_NAME = experiment_name
if model:
net = model
else:
description = model_builder.load_description(MODEL_PATH, MODEL_NAME)
net = model_builder.build_model(model_name, description)
net.load(MODEL_PATH, MODEL_NAME, description)
bias = net.set_0_bias()
# net.model.summary()
# print net.model.get_layer(name="embedding_2").get_weights()[0]
# If no <UNKNOWN> role in the role vocabulary, add it.
if net.role_vocabulary.get("<UNKNOWN>", -1) == -1:
net.role_vocabulary["<UNKNOWN>"] = len(net.role_vocabulary) - 1
if print_result:
print net.role_vocabulary
print("unk_word_id", net.unk_word_id)
print("missing_word_id", net.missing_word_id)
propbank_map = {
"subj" : "A0",
"obj" : "A1",
"ARG0" : "A0",
"ARG1" : "A1",
}
tr_map = {
"A0": numpy.asarray([net.role_vocabulary["A0"]], dtype=numpy.int64),
"A1": numpy.asarray([net.role_vocabulary["A1"]], dtype=numpy.int64),
"<UNKNOWN>": numpy.asarray([net.role_vocabulary["<UNKNOWN>"]], dtype=numpy.int64)
}
if "A2" not in net.role_vocabulary.keys():
propbank_map["ARG2"] = "<UNKNOWN>"
tr_map["A2"] = numpy.asarray([net.role_vocabulary["<UNKNOWN>"]], dtype=numpy.int64)
else:
propbank_map["ARG2"] = "A2"
tr_map["A2"] = numpy.asarray([net.role_vocabulary["A2"]], dtype=numpy.int64)
fixed = False
if evaluation == "pado":
eval_data_file = os.path.join(EVAL_PATH, 'pado_plausibility_pb.txt')
elif evaluation == 'mcrae':
eval_data_file = os.path.join(EVAL_PATH, 'mcrae_agent_patient_more.txt')
else:
fixed = True
if evaluation == 'pado_fixed':
eval_data_file = os.path.join(RV_EVAL_PATH, 'Pado-AsadFixes.txt')
elif evaluation == 'mcrae_fixed':
eval_data_file = os.path.join(RV_EVAL_PATH, 'McRaeNN-fixed.txt')
else:
eval_data_file = os.path.join(COMP_EVAL_PATH, 'compare-pado.txt')
result_file = os.path.join(MODEL_PATH, MODEL_NAME + '_' + evaluation + '.txt')
r_i = net.role_vocabulary["V"]
probs = {}
baseline = {}
oov_count = {}
blist=[]
plist = []
if print_result:
print eval_data_file
print "="* 60
with open(eval_data_file, 'r') as f, \
open(result_file, 'w') as f_out:
for i, line in enumerate(f):
line = line.strip()
if line == "":
continue
w, tw, tr = line.split()[:3] # input word, target word, role
w = w[:-2] if fixed else w
tw = tw[:-2] if fixed else tw
w = wnl.lemmatize(w, wn.VERB)
tw = wnl.lemmatize(tw, wn.NOUN)
w_i = net.word_vocabulary.get(w, net.unk_word_id)
tw_i = net.word_vocabulary.get(tw, net.unk_word_id)
tr_i = net.role_vocabulary.get(propbank_map[tr], net.unk_role_id)
if tw_i == net.unk_word_id:
print w, tr, tw
oov_count[tr] = oov_count.get(tr, 0) + 1
f_out.write(line + "\tnan\n")
continue
b = float(line.split()[3])
baseline.setdefault(tr, []).append(b)
blist.append(b)
sample = dict((r, net.missing_word_id) for r in (net.role_vocabulary.values() + [net.unk_role_id]))
sample[r_i] = w_i
sample.pop(net.role_vocabulary[propbank_map[tr]], None)
x_w_i = numpy.asarray([sample.values()], dtype=numpy.int64)
x_r_i = numpy.asarray([sample.keys()], dtype=numpy.int64)
y_w_i = numpy.asarray([tw_i])
y_r_i = tr_map[propbank_map[tr]]
s = net.p_words(x_w_i, x_r_i, y_w_i, y_r_i)
# pr = net.p_roles(x_w_i, x_r_i, y_w_i, y_r_i)
plist.append(s)
probs.setdefault(tr, []).append(s)
f_out.write(line + "\t%s\n" % s)
result = dict()
for r, b in baseline.iteritems():
p = probs[r]
rho, p_value = spearmanr(b, p)
rating = len(p)
oov = oov_count.get(r, 0)
result[r] = round(rho, 4)
if print_result:
print "=" * 60
print "ROLE: %s" % r
print "-" * 60
print "Spearman correlation: %f; 2-tailed p-value: %f" % (rho, p_value)
print "Num ratings: %d (%d out of vocabulary)" % (rating, oov)
rho, p_value = spearmanr(blist, plist)
result['all'] = round(rho, 4)
if print_result:
print "Spearman correlation of %s: %f; 2-tailed p-value: %f" % (evaluation, rho, p_value)
net.set_bias(bias)
return result, plist, blist
def main():
batch_size = 5
num_epochs = 100
df_train, df_val, df_test = load_data()
class_map, class_index, class_name = form_class_map()
train_set = ImageDataset(df_train)
train_dataloader = DataLoader(train_set,
batch_size=batch_size,
shuffle=True,
num_workers=8)
val_set = ImageDataset(df_val)
val_dataloader = DataLoader(val_set,
batch_size=batch_size,
shuffle=False,
num_workers=8)
test_set = ImageDataset(df_test)
test_dataloader = DataLoader(test_set,
batch_size=batch_size,
shuffle=False,
num_workers=8)
model = build_model('UNet')
model.apply(init_weights)
model.to(DEVICE)
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(),
lr=1e-3,
weight_decay=1e-5)
n_params = calc_n_params(model)
print("No. of parameters:", n_params)
frame_rate = test_inf_speed(model, test_set, test_dataloader)
print("Frame rate: {:.2f} FPS (frame per second)".format(frame_rate))
df = pd.DataFrame(columns=class_name + ["Train", "Val"])
model.train()
for epoch in range(num_epochs):
model = train(model, train_dataloader, criterion, optimizer, epoch)
start = time.time()
val_loss, val_acc, val_iou = validate(model, val_dataloader, criterion,
class_index)
train_loss, train_acc, _ = validate(model, train_dataloader, criterion,
class_index)
end = time.time()
print('Validate: {:.1f}s'.format(end - start))
print(
'Train Loss: {:.4f}\tTrain Accuracy: {:.4f}\tVal Loss: {:.4f}\tVal Accuracy: {:.4f}'
.format(train_loss, train_acc, val_loss, val_acc))
df.loc[len(df.index)] = val_iou + [train_acc, val_acc]
print('Val IOU:')
for i in range(len(class_index)):
print('{}: {:.4f}\t'.format(class_name[i], val_iou[i]), end='')
if i % 6 == 5 or i == len(class_index) - 1:
print()
print('mIOU: {:.4f}'.format(np.mean(val_iou)))
df.to_csv("iou.csv", header=True, index=False, columns=class_name)
df.to_csv("acc.csv", header=True, index=False, columns=["Train", "Val"])
visualize_results(df, class_name)
test(model, test_dataloader, class_map)
torch.save(
{
'model_state_dict': model.state_dict(),
'optimizer_state_dict': optimizer.state_dict(),
}, "Model_" + str(num_epochs))
class_names_string = class_names_string + class_name + ", "
else:
class_names_string = class_names_string + class_name
num_classes = len(label_values)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess=tf.Session(config=config)
# Compute your softmax cross entropy loss
net_input = tf.placeholder(tf.float32,shape=[None,None,None,3])
net_output = tf.placeholder(tf.float32,shape=[None,None,None,num_classes])
network, init_fn = model_builder.build_model(args.model, frontend=args.frontend, net_input=net_input, num_classes=num_classes, is_training=True)
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=network, labels=net_output))
opt = tf.train.RMSPropOptimizer(learning_rate=0.0001, decay=0.995).minimize(loss, var_list=[var for var in tf.trainable_variables()])
saver=tf.train.Saver(max_to_keep=1000)
sess.run(tf.global_variables_initializer())
utils.count_params()
# If a pre-trained ResNet is required, load the weights.
# This must be done AFTER the variables are initialized with sess.run(tf.global_variables_initializer())
if init_fn is not None:
init_fn(sess)
def train(cfg):
startup_prog = fluid.Program()
train_prog = fluid.Program()
if args.enable_ce:
startup_prog.random_seed = 1000
train_prog.random_seed = 1000
drop_last = True
dataset = SegDataset(file_list=cfg.DATASET.TRAIN_FILE_LIST,
mode=ModelPhase.TRAIN,
shuffle=True,
data_dir=cfg.DATASET.DATA_DIR)
def data_generator():
if args.use_mpio:
data_gen = dataset.multiprocess_generator(
num_processes=cfg.DATALOADER.NUM_WORKERS,
max_queue_size=cfg.DATALOADER.BUF_SIZE)
else:
data_gen = dataset.generator()
batch_data = []
for b in data_gen:
batch_data.append(b)
if len(batch_data) == (cfg.BATCH_SIZE // cfg.NUM_TRAINERS):
for item in batch_data:
yield item[0], item[1], item[2]
batch_data = []
# If use sync batch norm strategy, drop last batch if number of samples
# in batch_data is less then cfg.BATCH_SIZE to avoid NCCL hang issues
if not cfg.TRAIN.SYNC_BATCH_NORM:
for item in batch_data:
yield item[0], item[1], item[2]
# Get device environment
# places = fluid.cuda_places() if args.use_gpu else fluid.cpu_places()
# place = places[0]
gpu_id = int(os.environ.get('FLAGS_selected_gpus', 0))
place = fluid.CUDAPlace(gpu_id) if args.use_gpu else fluid.CPUPlace()
places = fluid.cuda_places() if args.use_gpu else fluid.cpu_places()
# Get number of GPU
dev_count = cfg.NUM_TRAINERS if cfg.NUM_TRAINERS > 1 else len(places)
print_info("#Device count: {}".format(dev_count))
# Make sure BATCH_SIZE can divided by GPU cards
assert cfg.BATCH_SIZE % dev_count == 0, (
'BATCH_SIZE:{} not divisble by number of GPUs:{}'.format(
cfg.BATCH_SIZE, dev_count))
# If use multi-gpu training mode, batch data will allocated to each GPU evenly
batch_size_per_dev = cfg.BATCH_SIZE // dev_count
print_info("batch_size_per_dev: {}".format(batch_size_per_dev))
config_info = {'input_size': 769, 'output_size': 1, 'block_num': 7}
config = ([(cfg.SLIM.NAS_SPACE_NAME, config_info)])
factory = SearchSpaceFactory()
space = factory.get_search_space(config)
port = cfg.SLIM.NAS_PORT
server_address = (cfg.SLIM.NAS_ADDRESS, port)
sa_nas = SANAS(config,
server_addr=server_address,
search_steps=cfg.SLIM.NAS_SEARCH_STEPS,
is_server=cfg.SLIM.NAS_IS_SERVER)
for step in range(cfg.SLIM.NAS_SEARCH_STEPS):
arch = sa_nas.next_archs()[0]
start_prog = fluid.Program()
train_prog = fluid.Program()
data_loader, avg_loss, lr, pred, grts, masks = build_model(
train_prog, start_prog, arch=arch, phase=ModelPhase.TRAIN)
cur_flops = flops(train_prog)
print('current step:', step, 'flops:', cur_flops)
data_loader.set_sample_generator(data_generator,
batch_size=batch_size_per_dev,
drop_last=drop_last)
exe = fluid.Executor(place)
exe.run(start_prog)
exec_strategy = fluid.ExecutionStrategy()
# Clear temporary variables every 100 iteration
if args.use_gpu:
exec_strategy.num_threads = fluid.core.get_cuda_device_count()
exec_strategy.num_iteration_per_drop_scope = 100
build_strategy = fluid.BuildStrategy()
if cfg.NUM_TRAINERS > 1 and args.use_gpu:
dist_utils.prepare_for_multi_process(exe, build_strategy,
train_prog)
exec_strategy.num_threads = 1
if cfg.TRAIN.SYNC_BATCH_NORM and args.use_gpu:
if dev_count > 1:
# Apply sync batch norm strategy
print_info("Sync BatchNorm strategy is effective.")
build_strategy.sync_batch_norm = True
else:
print_info(
"Sync BatchNorm strategy will not be effective if GPU device"
" count <= 1")
compiled_train_prog = fluid.CompiledProgram(
train_prog).with_data_parallel(loss_name=avg_loss.name,
exec_strategy=exec_strategy,
build_strategy=build_strategy)
# Resume training
begin_epoch = cfg.SOLVER.BEGIN_EPOCH
if cfg.TRAIN.RESUME_MODEL_DIR:
begin_epoch = load_checkpoint(exe, train_prog)
# Load pretrained model
elif os.path.exists(cfg.TRAIN.PRETRAINED_MODEL_DIR):
print_info('Pretrained model dir: ',
cfg.TRAIN.PRETRAINED_MODEL_DIR)
load_vars = []
load_fail_vars = []
def var_shape_matched(var, shape):
"""
Check whehter persitable variable shape is match with current network
"""
var_exist = os.path.exists(
os.path.join(cfg.TRAIN.PRETRAINED_MODEL_DIR, var.name))
if var_exist:
var_shape = parse_shape_from_file(
os.path.join(cfg.TRAIN.PRETRAINED_MODEL_DIR, var.name))
return var_shape == shape
return False
for x in train_prog.list_vars():
if isinstance(x, fluid.framework.Parameter):
shape = tuple(fluid.global_scope().find_var(
x.name).get_tensor().shape())
if var_shape_matched(x, shape):
load_vars.append(x)
else:
load_fail_vars.append(x)
fluid.io.load_vars(exe,
dirname=cfg.TRAIN.PRETRAINED_MODEL_DIR,
vars=load_vars)
for var in load_vars:
print_info("Parameter[{}] loaded sucessfully!".format(
var.name))
for var in load_fail_vars:
print_info(
"Parameter[{}] don't exist or shape does not match current network, skip"
" to load it.".format(var.name))
print_info(
"{}/{} pretrained parameters loaded successfully!".format(
len(load_vars),
len(load_vars) + len(load_fail_vars)))
else:
print_info(
'Pretrained model dir {} not exists, training from scratch...'.
format(cfg.TRAIN.PRETRAINED_MODEL_DIR))
fetch_list = [avg_loss.name, lr.name]
global_step = 0
all_step = cfg.DATASET.TRAIN_TOTAL_IMAGES // cfg.BATCH_SIZE
if cfg.DATASET.TRAIN_TOTAL_IMAGES % cfg.BATCH_SIZE and drop_last != True:
all_step += 1
all_step *= (cfg.SOLVER.NUM_EPOCHS - begin_epoch + 1)
avg_loss = 0.0
timer = Timer()
timer.start()
if begin_epoch > cfg.SOLVER.NUM_EPOCHS:
raise ValueError(
("begin epoch[{}] is larger than cfg.SOLVER.NUM_EPOCHS[{}]"
).format(begin_epoch, cfg.SOLVER.NUM_EPOCHS))
if args.use_mpio:
print_info("Use multiprocess reader")
else:
print_info("Use multi-thread reader")
best_miou = 0.0
for epoch in range(begin_epoch, cfg.SOLVER.NUM_EPOCHS + 1):
data_loader.start()
while True:
try:
loss, lr = exe.run(program=compiled_train_prog,
fetch_list=fetch_list,
return_numpy=True)
avg_loss += np.mean(np.array(loss))
global_step += 1
if global_step % args.log_steps == 0 and cfg.TRAINER_ID == 0:
avg_loss /= args.log_steps
speed = args.log_steps / timer.elapsed_time()
print((
"epoch={} step={} lr={:.5f} loss={:.4f} step/sec={:.3f} | ETA {}"
).format(epoch, global_step, lr[0], avg_loss, speed,
calculate_eta(all_step - global_step, speed)))
sys.stdout.flush()
avg_loss = 0.0
timer.restart()
except fluid.core.EOFException:
data_loader.reset()
break
except Exception as e:
print(e)
if epoch > cfg.SLIM.NAS_START_EVAL_EPOCH:
ckpt_dir = save_checkpoint(exe, train_prog,
'{}_tmp'.format(port))
_, mean_iou, _, mean_acc = evaluate(cfg=cfg,
arch=arch,
ckpt_dir=ckpt_dir,
use_gpu=args.use_gpu,
use_mpio=args.use_mpio)
if best_miou < mean_iou:
print('search step {}, epoch {} best iou {}'.format(
step, epoch, mean_iou))
best_miou = mean_iou
sa_nas.reward(float(best_miou))
def evaluate(model_name, experiment_name, batch_size):
MODEL_NAME = experiment_name
repr_file = os.path.join(MODEL_PATH, 'confusionM_' + MODEL_NAME)
description = model_builder.load_description(MODEL_PATH, MODEL_NAME)
net = model_builder.build_model(model_name, description)
net.load(MODEL_PATH, MODEL_NAME, description)
n_roles = len(net.role_vocabulary)
print(net.role_vocabulary) #Added () to print (team1-change)
print("unk_word_id", net.unk_word_id)
print("missing_word_id", net.missing_word_id)
net.model.summary()
print(net.model.metrics_names) #Added () to print (team1-change)
reverse_role_vocabulary = utils.get_reverse_map(net.role_vocabulary)
test_sample_size = config.OCT_TEST_SIZE
test_steps = test_sample_size / batch_size
# # DEBUG
# test_steps = 10
print('Testing...') #Added () to print (team1-change)
test_start = time.process_time() #Changed from time.clock() (team1-change)
# Always use generator in Keras
if re.search('NAME_WHICH_YOU_NEED_OLD_BATCHER', experiment_name):
test_gen = get_minibatch(DATA_PATH + "NN_test",
net.unk_word_id,
net.unk_role_id,
net.missing_word_id,
n_roles,
random=False,
batch_size=batch_size)
else:
test_gen = generator(DATA_PATH + "NN_test",
model_name,
net.unk_word_id,
net.unk_role_id,
net.missing_word_id,
n_roles,
random=False,
batch_size=batch_size)
# Test the model
test_result = net.model.evaluate_generator(generator=test_gen,
steps=test_steps,
max_q_size=1,
workers=1,
pickle_safe=False)
print('test_result', test_result)
# Compute confusion matrix
metrics_names = net.model.metrics_names
result_dict = {(x, 0) for x in metrics_names}
batch_n = 0
confusionM = np.zeros((n_roles, n_roles), dtype='int32')
ppl_role_list = dict()
ppl_role = dict()
result_list = []
for ([i_w, i_r, t_w, t_r], _) in generator(DATA_PATH + "NN_test",
model_name,
net.unk_word_id,
net.unk_role_id,
net.missing_word_id,
n_roles,
random=False,
batch_size=batch_size):
result_role = net.predict_role(i_w, i_r, t_w, t_r, batch_size)
result_word_likelihood = net.predict(i_w, i_r, t_w, t_r, batch_size)[0]
neg_log_likelihoods = -np.log(result_word_likelihood)
for i, row in enumerate(neg_log_likelihoods, start=0):
target_word = t_w[i][0]
target_role = t_r[i][0]
neg_log_likelihood = row[target_word]
ppl_role_list.setdefault(target_role,
[]).append(neg_log_likelihood)
for i, true_r in enumerate(t_r, start=0):
confusionM[true_r, result_role[i]] += 1
if true_r == result_role[i]:
result_list.append(1)
batch_n += 1
print(batch_n) #Added () to print (team1-change)
if batch_n >= test_steps:
break
for k, v in ppl_role_list.items():
neg_log_likelihood_role = np.mean(np.array(v))
ppl_role[k] = np.exp(neg_log_likelihood_role)
print("Confusion Matrix: ") #Added () to print (team1-change)
print(" A0, A1, LOC, TMP, MNR, V, <UNKNOWN>"
) #Added () to print (team1-change)
print(confusionM) #Added () to print (team1-change)
np.savetxt('confusionM_' + experiment_name + '.csv',
confusionM,
delimiter=',')
np.savetxt('result_list_' + experiment_name + '.csv',
result_list,
delimiter=',')
stats(net, confusionM)
print("Loss(neg_log_likelihood) by role: "
) #Added () to print (team1-change)
for r in ppl_role.keys():
print(reverse_role_vocabulary[r], np.log(ppl_role[r]))
print("PPL by role: ") #Added () to print (team1-change)
for r in ppl_role.keys():
print(reverse_role_vocabulary[r], ppl_role[r])
with open(repr_file, 'w') as f_out:
f_out.write('[')
for i in range(n_roles):
f_out.write('[')
for j in range(n_roles):
f_out.write(str(confusionM[i][j]) + ", ")
f_out.write('] \n')
f_out.write(']')
test_end = time.process_time() #Changed from time.clock() (team1-change)
print('test time: %f, sps: %f' %
(test_end - test_start, test_steps * batch_size /
(test_end - test_start))) #Added () to print (team1-change)
def main():
parser = argparse.ArgumentParser(description="Run QA-CLEF-System")
parser.add_argument('--preprocess',action="store_true")
parser.add_argument('--train',action="store_true")
parser.add_argument('--answeronly',action='store_true')
parser.add_argument('--selftest',action='store_true')
parser.add_argument('--data',nargs = '+',default=[2011],type=int)
parser.add_argument('--test',nargs = '+',default=[2012],type=int)
parser.add_argument('--forcedownload',action='store_true')
parser.add_argument('--preprocessonly',action='store_true')
parser.add_argument('--ngram', type=int, default=3)
parser.add_argument('--threshold', type=float, default=0.5)
parser.add_argument('--report',action='store_true')
args = parser.parse_args()
process_args(args)
data = []
for edition in args.data + args.test:
_data = qacache.find_data(edition)
if args.preprocess or _data is None:
input_check([edition],args.forcedownload)
_data = input_parse([edition])
print >> sys.stderr, 'preprocessing ' + str(edition) + '-data'
_data = preprocessing.preprocess(_data)
qacache.store_preprocessed_data(edition,_data[0])
else:
print >> sys.stderr, str(edition) + '-data is found on cache/' + str(edition) + '-prerocessed.txt'
data.append(_data)
if args.preprocessonly:
print >> sys.stderr, 'Preprocess-only task is done.'
sys.exit(0)
# build-model
print >> sys.stderr, 'Building model...'
training_model = model_builder.build_model(data[:len(args.data)])
test_model = model_builder.build_model(data[-len(args.test):]) if len(args.test) != 0 and not args.selftest else []
# scoring
print >> sys.stderr, 'Unweighted Feature Scoring...'
training_model and scoring.score(training_model)
test_model and scoring.score(test_model)
# training
weight = qacache.stored_weight()
if args.train or weight is None:
print >> sys.stderr, 'Training...'
weight = train(training_model)
else:
print >> sys.stderr, 'Weight is found on cache/weight.txt'
# weighted_scoring
print >> sys.stderr, 'Weighted Feature Scoring...'
final = scoring.weighted_scoring(training_model if args.selftest else test_model, weight)
# answer selection
select_answer(final,args.threshold)
# evaluation
result = evaluate(final)
qacache.write_json(final,'final.txt',indent=True)
if args.report:
report(final, args.test if not args.selftest else args.data,weight)
print "Result: %f" % result
def evaluate(cfg, ckpt_dir=None, use_gpu=False, use_mpio=False, **kwargs):
np.set_printoptions(precision=5, suppress=True)
startup_prog = fluid.Program()
test_prog = fluid.Program()
dataset = SegDataset(
file_list=cfg.DATASET.VAL_FILE_LIST,
mode=ModelPhase.EVAL,
data_dir=cfg.DATASET.DATA_DIR)
def data_generator():
#TODO: check is batch reader compatitable with Windows
if use_mpio:
data_gen = dataset.multiprocess_generator(
num_processes=cfg.DATALOADER.NUM_WORKERS,
max_queue_size=cfg.DATALOADER.BUF_SIZE)
else:
data_gen = dataset.generator()
for b in data_gen:
yield b[0], b[1], b[2]
data_loader, avg_loss, pred, grts, masks = build_model(
test_prog, startup_prog, phase=ModelPhase.EVAL, arch=kwargs['arch'])
data_loader.set_sample_generator(
data_generator, drop_last=False, batch_size=cfg.BATCH_SIZE)
# Get device environment
places = fluid.cuda_places() if use_gpu else fluid.cpu_places()
place = places[0]
dev_count = len(places)
print("#Device count: {}".format(dev_count))
exe = fluid.Executor(place)
exe.run(startup_prog)
test_prog = test_prog.clone(for_test=True)
ckpt_dir = cfg.TEST.TEST_MODEL if not ckpt_dir else ckpt_dir
if not os.path.exists(ckpt_dir):
raise ValueError('The TEST.TEST_MODEL {} is not found'.format(ckpt_dir))
if ckpt_dir is not None:
print('load test model:', ckpt_dir)
try:
fluid.load(test_prog, os.path.join(ckpt_dir, 'model'), exe)
except:
fluid.io.load_params(exe, ckpt_dir, main_program=test_prog)
# Use streaming confusion matrix to calculate mean_iou
np.set_printoptions(
precision=4, suppress=True, linewidth=160, floatmode="fixed")
conf_mat = ConfusionMatrix(cfg.DATASET.NUM_CLASSES, streaming=True)
fetch_list = [avg_loss.name, pred.name, grts.name, masks.name]
num_images = 0
step = 0
all_step = cfg.DATASET.TEST_TOTAL_IMAGES // cfg.BATCH_SIZE + 1
timer = Timer()
timer.start()
data_loader.start()
while True:
try:
step += 1
loss, pred, grts, masks = exe.run(
test_prog, fetch_list=fetch_list, return_numpy=True)
loss = np.mean(np.array(loss))
num_images += pred.shape[0]
conf_mat.calculate(pred, grts, masks)
_, iou = conf_mat.mean_iou()
_, acc = conf_mat.accuracy()
speed = 1.0 / timer.elapsed_time()
print(
"[EVAL]step={} loss={:.5f} acc={:.4f} IoU={:.4f} step/sec={:.2f} | ETA {}"
.format(step, loss, acc, iou, speed,
calculate_eta(all_step - step, speed)))
timer.restart()
sys.stdout.flush()
except fluid.core.EOFException:
break
category_iou, avg_iou = conf_mat.mean_iou()
category_acc, avg_acc = conf_mat.accuracy()
print("[EVAL]#image={} acc={:.4f} IoU={:.4f}".format(
num_images, avg_acc, avg_iou))
print("[EVAL]Category IoU:", category_iou)
print("[EVAL]Category Acc:", category_acc)
print("[EVAL]Kappa:{:.4f}".format(conf_mat.kappa()))
return category_iou, avg_iou, category_acc, avg_acc
mean, variance = tf.nn.moments(inputs, axes=(0, 1, 2, 3), keep_dims=True, name="normalize_moments")
Gamma = tf.constant(1.0, name="scale_factor", shape=mean.shape, dtype=tf.float32)
Beta = tf.constant(0.0,name="offset_factor", shape=mean.shape, dtype=tf.float32)
data = tf.nn.batch_normalization(inputs, mean, variance, offset=Beta, scale=Gamma, variance_epsilon=1e-3)
'''
normilizing is must at first
'''
result = build_model(data=data,
labels=None,
model_name='r2plus1d',
model_depth=18,
num_labels=400,
num_channels=3,
crop_size=112,
clip_length=24,
loss_scale=1.0,
is_test = 1,
)
rgb_saver = tf.train.Saver()
global_init = tf.global_variables_initializer()
feed_dict={}
all_activations = {'final': []}
nodes = tf.get_collection('source_node')
feature = None
for var in nodes:
if 'final_avg/AvgPool3D:0' in var.name:
feature = var
print("Model -->", args.model)
print("Crop Height -->", args.crop_height)
print("Crop Width -->", args.crop_width)
print("Num Classes -->", num_classes)
print("Image -->", args.image)
# Initializing network
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
net_input = tf.placeholder(tf.float32, shape=[None, None, None, 3])
net_output = tf.placeholder(tf.float32, shape=[None, None, None, num_classes])
network, _ = model_builder.build_model(args.model,
net_input=net_input,
num_classes=num_classes,
is_training=False)
sess.run(tf.global_variables_initializer())
print('Loading model checkpoint weights')
saver = tf.train.Saver(max_to_keep=1000)
saver.restore(sess, args.checkpoint_path)
print("Testing image " + args.image)
loaded_image = utils.load_image(args.image)
resized_image = cv2.resize(loaded_image, (args.crop_width, args.crop_width))
input_image = np.expand_dims(np.float32(
resized_image[:args.crop_height, :args.crop_width]),
axis=0) / 255.0
from model_builder import build_model, regex_baseline
from model_tester import execute_test
import sys
import profile
logistic = NeuralLogistic(restarts = 2, regularization = 0.00)
nn = NeuralNetwork([(8, 'logistic'), (8, 'tanh'), (None, 'softmax')], 'maxent', regularization = 1e0, restarts = 30, max_iter = 10000, init_size = 1, step_size = 1e-2)
#nn = NeuralNetwork([(None, 'softmax')], 'maxent', include_offset = True, max_iter = 5000)
me = MixtureOfExperts([NeuralLogistic(regularization = 1e-1), NeuralLogistic(regularization = 1e-1), NeuralLogistic(regularization = 1e-1)],
NeuralLogistic(regularization = 1e-1), max_iter = 25)
#NeuralNetwork([(3, 'tanh'), (None, 'softmax')], 'maxent'))
adaboost = AdaBoostClassifier(n_estimators = 500)
mix_ex_args = {'experts' :[NeuralLogistic(regularization = 1e-1), NeuralLogistic(regularization = 1e-1), NeuralLogistic(regularization = 1e-1)],
'gate' : NeuralLogistic(regularization = 1e-1),
'max_iter': 15}
model = build_model(regex_baseline, method = 'me', model_args = mix_ex_args)
print execute_test(model, 25, 5)
