def find_params(start, stop, step, step_type, adjust, others):
"""
For working on model and finding better parameters.
:param start: float
:param stop: float
:param step: float
:param step_type: str
:param adjust: str
:param others: tuple(float, float)
:return: None
"""
numb_epochs = 25 # Change the epoch number for more or less epochs.
batch_size = 25 # Change the value if training set size smaller.
if step_type == 'lin':
multiplier1 = 1 / start
multiplier2 = 1 * multiplier1 / stop
nstart = int(start * multiplier1 * multiplier2)
nstop = int(stop * multiplier1 * multiplier2)
nstep = int(step * multiplier1 * multiplier2)
divider = multiplier1 * multiplier2
numbers = range(nstart, nstop, nstep)
numbers = np.array(numbers).astype(float)
numbers /= divider
numbers = list(numbers)
elif step_type == 'exp':
numbers = []
acc = start
while True:
numbers.append(acc)
acc /= step
if acc <= stop:
break
else:
raise Exception('Give a proper Step Type!')
if adjust == 'lr':
combs = itertools.product(numbers, [others[0]], [others[1]])
elif adjust == 'wd':
combs = itertools.product([others[0]], numbers, [others[1]])
elif adjust == 'mom':
combs = itertools.product([others[0]], [others[1]], numbers)
else:
raise Exception('Give a proper str for Adjust Type!')
for comb in combs:
train_eval(('./gray/', './color_64/', 'train.txt', 'valid.txt',
numb_epochs, batch_size, comb[0], comb[1], comb[2]))
directory = './outputs_' + str(batch_size) + '_' + str(comb[0]) + '_' + str(comb[1]) + '_' + \
str(comb[2]) + '/'
print('\n---------------------')
print('---------------------')
print('---------------------\n')
print('In directory: ' + directory)
print(
'Loss for ' + str(numb_epochs) +
' epochs in order from first epoch to last.'
# ' epochs in order of 1 for train sample and 1 for validation from first epoch to last.'
)
for i in range(numb_epochs):
# main((directory + 'output_train_' + str(i) + '.npy', 'train.txt'))
main((directory + 'output_valid_' + str(i) + '.npy', 'valid.txt'))
def main(train_file=tr_f,
tag=tg,
test_file=te_f,
out_file=output,
gold_file=g_f):
pos_tagger.main(train_file, tag, test_file)
evaluate.main(out_file, tag, gold_file)
def main():
print 'TRAINING'
train.main()
print 'PREDICTING'
predict.main()
print 'EVAlUATING'
evaluate.main()
def main():
print "TRAINING"
train.main()
print "PREDICTING"
predict.main()
print "EVAlUATING"
evaluate.main()
def test1(self, inp, test_num):
#単純反応
if inp != 0:
n = inp
else:
n = self.N
result = []
wait = 0
self.msg.setText(u'文字が表示されたら何かボタンを押してください.')
self.msg.alignHoriz = 'center'
self.msg.alignVert = 'center'
self.msg.pos = (0, 0.7)
self.test_ch1.pos = (0, -0.1)
self.eval.pos = (0.3, 0.2)
t = 3.0
startClock = core.Clock()
pygame.mixer.init()
pygame.mixer.music.load(start)
pygame.mixer.music.play(1)
while t >= 0.2:
t = 3.6 - startClock.getTime()
self.msg.draw()
self.counter.setText('%.0f' % (t))
self.counter.draw()
self.myWin.flip()
self.myWin.flip()
time.sleep(1.5)
pygame.mixer.music.stop()
time.sleep(0.01)
for i in range(n):
self.myWin.flip()
wait = random.uniform(1, 3)
time.sleep(wait)
self.test_ch1.setText(random.choice(upper))
event.clearEvents()
testClock = core.Clock()
while True:
self.test_ch1.draw()
self.myWin.flip()
key = event.getKeys()
if len(key) > 0:
result.append(testClock.getTime())
self.sound(1, 0, result[-1])
break
event.clearEvents()
self.myWin.flip()
self.msg.alignHoriz = 'left'
self.msg.alignVert = 'top'
self.msg.pos = (-0.7, 0.7)
self.eval.pos = (0.7, 0.2)
evaluate.main(self.myWin, test_num, result, False)
return result
def main(_):
vocab, rev_vocab = du.initialize_vocab()
embeddings = du.load_embeddings()
if not os.path.exists(FLAGS.log_dir):
os.makedirs(FLAGS.log_dir)
file_handler = logging.FileHandler(pjoin(FLAGS.log_dir, "log.txt"))
logging.getLogger().addHandler(file_handler)
print(vars(FLAGS))
with open(os.path.join(FLAGS.log_dir, "flags.json"), 'w') as fout:
json.dump(FLAGS.__flags, fout)
# ========= Load Dataset =========
# You can change this code to load dataset in your own way
dev_dirname = os.path.dirname(os.path.abspath(FLAGS.dev_path))
dev_filename = os.path.basename(FLAGS.dev_path)
contexts, questions, question_uuids, vocab, rev_vocab, embeddings = prepare_dev(
dev_dirname, dev_filename, vocab, rev_vocab, embeddings)
questions, questions_mask, questions_seq = du.padding(
du.cast_to_int(questions), FLAGS.max_question_size)
contexts, contexts_mask, contexts_seq = du.padding(
du.cast_to_int(contexts), FLAGS.max_document_size)
dataset = {
'q': questions,
'q_m': questions_mask,
'q_s': questions_seq,
'c': contexts,
'c_m': contexts_mask,
'c_s': contexts_seq,
'q_uuids': question_uuids
}
print("length of dev set: {}".format(len(dataset['q'])))
# ========= Model-specific =========
# You must change the following code to adjust to your model
print("embedding length %d" % len(embeddings))
print("rev_vocab length %d" % len(rev_vocab))
print("vocab length %d" % len(vocab))
model = du.choose_model(embeddings=embeddings)
with tf.Session() as sess:
du.restore_model(session=sess, run_id=FLAGS.run_id)
answers = generate_answers(sess, model, dataset, rev_vocab=rev_vocab)
# write to json file to root dir
with io.open('dev-prediction.json', 'w', encoding='utf-8') as f:
f.write(unicode(json.dumps(answers, ensure_ascii=False)))
evaluate.main(FLAGS.dev_path, 'dev-prediction.json')
def main():
print config.SCORE_TYPE
for experiment in config.EXPERIMENTS:
now = time.time()
print "\nEXPERIMENT: {}\n".format(experiment)
generate_scores.main(config.SHARDS[experiment], experiment)
rank_answers.main(experiment)
evaluate.main(experiment)
analysis.main(experiment)
lstring = 'experiment: {}\ntime = {} sec'
print lstring.format(str(experiment), str(time.time() - now))
print "Done!"
def plot_final(dirr, outname):
out = StringIO()
sys.stdout = out
li = []
for i in range(40): # Change 75 if there are different number of epochs.
main((dirr + 'output_train_' + str(i) + '.npy',
'train.txt')) # Change names for train and valid.
for line in out.getvalue().splitlines():
li.append(line[:8])
plt.plot(np.array(li))
plt.xlabel('Epoch No')
plt.ylabel('Success Rate of Validation Set')
plt.locator_params(nbins=8)
plt.savefig(outname)
plt.close()
def get_text(message):
config.text = message.text
if config.in_file and config.out_file and config.text:
evaluate.ins = UPLOAD_FOLDER + config.in_file
evaluate.outs = DOWNLOAD_FOLDER + config.out_file
evaluate.main()
main.make_artwork(evaluate.outs, evaluate.outs, config.text)
with open(DOWNLOAD_FOLDER + config.out_file, "rb") as photo:
bot.send_photo(message.chat.id, photo.read())
# os.system(f"rm {evaluate.ins} {evaluate.outs} {DOWNLOAD_FOLDER + config.out_file, config.text}") """for UNIX systems [MacOS, Linux]"""
config.in_file = None
config.out_file = None
config.text = None
evaluate.ins = None
evaluate.out = None
def upload_file():
if request.method == 'POST':
file = request.files['file']
if file and allowed_file(file.filename):
filename = secure_filename(file.filename)
file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
evaluate.ins = UPLOAD_FOLDER + '/' + filename
evaluate.outs = os.getcwd() + '/examples/thumbs/' + filename
evaluate.main()
return send_from_directory(
(app.config['UPLOAD_FOLDER'] + '/thumbs/'),
filename,
as_attachment=True)
return '''
def upload_image():
startTime = time.time()
received_file = request.files['input_image']
if received_file:
# im = Image.open(received_file)
# im.resize((256,256),Image.ANTIALIAS)
received_dirPath = './data/test/2015'
imageFileName = received_file.filename
filepath = os.path.join(received_dirPath, imageFileName)
if os.path.isfile(filepath):
os.remove(filepath)
print(str(filepath) + "removed!")
received_file.save(filepath)
size = (256, 256)
pri_image = Image.open(filepath)
pri_image.resize(size, Image.ANTIALIAS).save(filepath)
print('image file saved to %s' % filepath)
usedTime = time.time() - startTime
print('接收图片并保存，总共耗时%.2f秒' % usedTime)
startTime = time.time()
predict_latex = evaluate.main()
usedTime = time.time() - startTime
print('完成对接收图片的分类预测，总共耗时%.2f秒' % usedTime)
print(predict_latex)
return predict_latex
else:
return 'failed'
def getevaluate(weights):
Result_dict = getFeatureLocation_result.getFinal_Result(
VersionHis_result, Similarreports_result, Structure_result,
Reporter_result, weights)
MAP = evaluate.main(Aimresult, Result_dict)
return MAP
def get_var(mem, father, children, main_deep):
d1 = mem.__len__()
d2 = mem[0].__len__()
depth = father.depth + 1
player = "black"
if father.color == "black":
player = "white"
if depth == main_deep + 1 + 2:
children.append(father)
father.score = evaluate.main(father.matrix)
else:
if player == "black":
for e in range(0, d1):
for p in range(0, d2):
new_mem = copy_matrix(mem)
if new_mem[e][p] == 0:
new_mem[e][p] = 1
new_father1 = Node("black", depth, new_mem)
father.kids.append(new_father1)
new_father1.ancestor = father
get_var(new_mem, new_father1, children, main_deep)
if player == "white":
for e in range(0, d1):
for p in range(0, d2):
new_mem = copy_matrix(mem)
if new_mem[e][p] == 0:
new_mem[e][p] = 2
new_father2 = Node("white", depth, new_mem)
father.kids.append(new_father2)
new_father2.ancestor = father
get_var(new_mem, new_father2, children, main_deep)
def getevaluate(weights):
#APISrcfileScores_dict = getAPISrcfileScores.main(weights[:2])#weights：自然语言 与程序语言之间的权重关系，决定相似度分数
Result_dict = getRecmdAPI_result.getFinal_Result(APIdscpScors_dict , APISimilarReportsScores_dict , APISrcfileScores_dict , weights[2:])
Aimresult=getAimList()
MAP = evaluate.main(Aimresult,Result_dict)
return MAP
def test_integration(self):
base_folder = os.path.dirname(__file__)
input_dir = os.path.join(base_folder, 'integration_dir')
output_dir = base_folder
scores_file = os.path.join(base_folder, 'scores.txt')
evaluate.main(input_dir, output_dir)
self.assertTrue(os.path.isfile(scores_file))
with open(scores_file, 'r') as fp:
scores_text = fp.read()
os.remove(scores_file)
self.assert_regex(scores_text, 'score:[\\d.]+')
self.assert_regex(scores_text, 'avg_spatial:[\\d.]+')
self.assert_regex(scores_text, 'avg_label:[\\d.]+')
self.assert_regex(scores_text, 'avg_pPDQ:[\\d.]+')
self.assert_regex(scores_text, 'FPs:[\\d]+')
self.assert_regex(scores_text, 'FNs:[\\d]+')
self.assert_regex(scores_text, 'TPs:[\\d]+')
scores_data = read_scores_file(scores_text)
self.assertIn('score', scores_data)
self.assertGreaterEqual(scores_data['score'], 0)
self.assertLessEqual(scores_data['score'], 1)
self.assertIn('avg_spatial', scores_data)
self.assertGreaterEqual(scores_data['avg_spatial'], 0)
self.assertLessEqual(scores_data['avg_spatial'], 1)
self.assertIn('avg_label', scores_data)
self.assertGreaterEqual(scores_data['avg_label'], 0)
self.assertLessEqual(scores_data['avg_label'], 1)
self.assertIn('avg_pPDQ', scores_data)
self.assertGreaterEqual(scores_data['avg_pPDQ'], 0)
self.assertLessEqual(scores_data['avg_pPDQ'], 1)
self.assertIn('FPs', scores_data)
self.assertGreaterEqual(scores_data['FPs'], 0)
self.assertEqual(scores_data['FPs'], int(scores_data['FPs']))
self.assertIn('FNs', scores_data)
self.assertGreaterEqual(scores_data['FNs'], 0)
self.assertEqual(scores_data['FNs'], int(scores_data['FNs']))
self.assertIn('TPs', scores_data)
self.assertGreaterEqual(scores_data['TPs'], 0)
self.assertEqual(scores_data['TPs'], int(scores_data['TPs']))
def main():
parser = argparse.ArgumentParser(description='PyTorch Kaggle')
parser.add_argument('--jobtype', '-M', type=str, default='evaluate', help='what are you going to do on this function')
parser.add_argument('--setting', '-S', type=str, default='setting1', help='which setting file are you going to use for training.')
#parser.add_argument('--model', '-M', type=str, default='', help='')
args = parser.parse_args()
#cfg = setting.Config()
cfg = cfgs.get(args.setting)
if args.jobtype == 'preprocess':
dataset.main(cfg)
if args.jobtype == 'train':
train.main(cfg)
if args.jobtype == 'evaluate':
evaluate.main(cfg)
def train(loader, loader_val, val_vocab, model, crit, optimizer, lr_scheduler, opt):
model = nn.DataParallel(model)
max_scores = 0
for epoch in range(opt['epochs']):
model.train()
save_flag=True
lr_scheduler.step()
iteration = 0
for data in loader:
audio_conv4 = data['audio_conv4'].cuda()
audio_fc2 = data['audio_fc2'].cuda()
labels = data['labels'].cuda()
masks = data['masks'].cuda()
sem_feats = data['sem_feats'].cuda()
torch.cuda.synchronize()
optimizer.zero_grad()
seq_probs, _ = model(audio_conv4, audio_fc2, sem_feats, labels, 'train', opt=opt)
loss = crit(seq_probs, labels[:, 1:], masks[:, 1:])
loss.backward()
clip_grad_value_(model.parameters(), opt['grad_clip'])
optimizer.step()
train_loss = loss.item()
torch.cuda.synchronize()
iteration += 1
print("iter %d (epoch %d), train_loss = %.6f" % (iteration, epoch, train_loss))
'''
if epoch % opt["save_checkpoint_every"] == 0 and save_flag:
model_path = os.path.join(opt["checkpoint_path"], 'model_%d.pth' % (epoch))
model_info_path = os.path.join(opt["checkpoint_path"], 'model_score.txt')
torch.save(model.state_dict(), model_path)
print("model saved to %s" % (model_path))
with open(model_info_path, 'a') as f:
f.write("model_%d, loss: %.6f\n" % (epoch, train_loss))
save_flag=False
'''
scores = evaluate.main(loader_val, val_vocab, opt, model)
sc = scores['Bleu_4']
scores = sum([scores['Bleu_1'], scores['Bleu_2'], scores['Bleu_3'], scores['Bleu_4']])
if scores > max_scores:
max_scores = scores
print(scores)
model_path = os.path.join(opt["checkpoint_path"], 'model_%d.pth' % (epoch))
model_info_path = os.path.join(opt["checkpoint_path"], 'model_score.txt')
torch.save(model.state_dict(), model_path)
print("model saved to %s" % (model_path))
with open(model_info_path, 'a') as f:
f.write(f"model_%{epoch}, bleu4: {sc}\n")
def process_script(package, file_name, id_, type_, name, known_results,
path_to_results):
from evaluate import main
key = f"{id_}_{type_}"
if key not in known_results:
script_results = main(file_name=file_name, package=package)
known_results[key] = script_results
known_results[key]["name"] = name
with path_to_results.open("w") as f:
json.dump(dict(sorted(known_results.items())), f, indent=4)
print("#" * 100)
return known_results
def getevaluate(weights):
Similarreports_result = getSimilarityScores2Reports.getFinalResultsbyWeights(
All_result, weights[2:9], issuekey_file_num, issuekey_file_list)
Structure_result = getStrucCmptScors.getFinal_result(
All_result0, weights[9:])
Result_dict = getFeatureLocation_result.getFinal_Result(
Similarreports_result, Structure_result, weights[:2])
Aimresult = getAimList()
MAP = evaluate.main(Aimresult, Result_dict)
return MAP
def main():
commands = ['train', 'predict', 'evaluate']
help_msg = \
'''
Usage: cliner [OPTIONS] COMMAND [ARGS]...
Options:
--help Show this message and exit.
Commands:
%s
''' % '\n '.join(commands)
# Is argument correct?
if len(sys.argv
) < 2 or sys.argv[1] not in commands or sys.argv[1] == '--help':
sys.stderr.write('%s\n\n' % (help_msg))
exit(1)
# select appropriate sub-command
subcmd = sys.argv[1]
del sys.argv[1]
# Where to import code from
homedir = os.path.dirname(os.path.abspath(__file__))
codedir = os.path.join(homedir, 'code')
if codedir not in sys.path:
sys.path.append(codedir)
# Call appropriate sub-command
if subcmd == 'train':
import train
train.main()
elif subcmd == 'predict':
import predict
predict.main()
elif subcmd == 'evaluate':
import evaluate
evaluate.main()
def getevaluate(weights):
APISimilarReportsScores_dict = getSimilarityScores2Reports.getFinalAPIResultsbyWeights(
All_result, weights[5:], issuekey_UsedAPI_num, issuekey_UsedAPI_list)
#APISrcfileScores_dict = getAPISrcfileScores.main(weights[:2])#weights：自然语言 与程序语言之间的权重关系，决定相似度分数
Result_dict = getRecmdAPI_result.getFinal_Result(
APIdscpScors_dict, APISimilarReportsScores_dict, APISrcfileScores_dict,
weights[2:5])
Aimresult = getAimList()
MAP = evaluate.main(Aimresult, Result_dict)
return MAP
def main():
args = utils.parse_args(create_parser())
exp_dir = pathlib.Path(args.exp_dir)
evaluate.main(
list(
filter(None, [
"--data-dir",
str(
pathlib.Path(__file__).parent.absolute().joinpath(
f"data/{args.dataset}/json/preprocessed")),
"--eval-splits", "train", "dev", "test", "--processor-path",
str(exp_dir.joinpath("processor.pkl")), "--model-path",
str(exp_dir.joinpath("model.json")), "--ckpt-path",
str(exp_dir.joinpath("checkpoint-best.pth")), "--embed-type",
args.embed_type,
"--embed-path" if args.embed_path is not None else None,
args.embed_path, "--logging-config", args.logging_config,
"--save-dir",
str(exp_dir.joinpath("eval")),
"--overwrite" if args.overwrite else None,
"--seed" if args.seed is not None else None,
str(args.seed) if args.seed is not None else None
])))
def test(model, ema, args, data):
device = torch.device(
f"cuda:{args.gpu}" if torch.cuda.is_available() else "cpu")
criterion = nn.CrossEntropyLoss()
loss = 0
answers = dict()
model.eval()
backup_params = EMA(0)
for name, param in model.named_parameters():
if param.requires_grad:
backup_params.register(name, param.data)
param.data.copy_(ema.get(name))
with torch.set_grad_enabled(False):
for batch in iter(data.dev_iter):
p1, p2 = model(batch)
batch_loss = criterion(p1, batch.s_idx) + criterion(
p2, batch.e_idx)
loss += batch_loss.item()
# (batch, c_len, c_len)
batch_size, c_len = p1.size()
ls = nn.LogSoftmax(dim=1)
mask = (torch.ones(c_len, c_len) *
float('-inf')).to(device).tril(-1).unsqueeze(0).expand(
batch_size, -1, -1)
score = (ls(p1).unsqueeze(2) + ls(p2).unsqueeze(1)) + mask
score, s_idx = score.max(dim=1)
score, e_idx = score.max(dim=1)
s_idx = torch.gather(s_idx, 1, e_idx.view(-1, 1)).squeeze()
for i in range(batch_size):
id = batch.id[i]
answer = batch.c_word[0][i][s_idx[i]:e_idx[i] + 1]
answer = ' '.join(
[data.WORD.vocab.itos[idx] for idx in answer])
answers[id] = answer
for name, param in model.named_parameters():
if param.requires_grad:
param.data.copy_(backup_params.get(name))
#print(answers)
with open(args.prediction_file, 'w', encoding='utf-8') as f:
print(json.dumps(answers, indent=4), file=f)
results = evaluate.main(args, answers, data)
return loss / len(data.dev_iter), results['exact_match'], results['f1']
def train():
if c.load_file:
model.load(c.load_file)
fn_weights = np.array(nets.read_params()[0])
INN_weights = np.array(model.read_params()[0])
print("fn weights init",np.sum(np.abs(fn_weights)))
print("INN weights init",np.sum(np.abs(INN_weights)))
#try:
for i_epoch in range(-c.pre_low_lr, c.n_epochs):
trainINN(i_epoch)
old_fn_weights = fn_weights
old_INN_weights = INN_weights
fn_weights = np.array(nets.read_params()[0])
INN_weights = np.array(model.read_params()[0])
print("fn weights updated",np.sum(np.abs(fn_weights - old_fn_weights)))
print("INN weights updated",np.sum(np.abs(INN_weights - old_INN_weights)))
testINN(i_epoch)
model.scheduler_step()
#if i_epoch % 4 == 0:
# ht.show_year_error()
viz.make_step()
if i_epoch > 0 and (i_epoch % c.checkpoint_save_interval) == 0:
model.save(c.filename + '_checkpoint_%.4i' % (i_epoch * (1-c.checkpoint_save_overwrite)))
model.save(c.filename)
viz.close()
import evaluate
evaluate.main()
def process_script(id_, type_, name, known_results):
from evaluate import main
key = id_ + "_" + type_
if key not in known_results:
# current_dir = os.getcwd()
# os.chdir(str(pathlib.Path(current_dir) / "filmdb_evaluation"))
script_results = main()
# os.chdir(current_dir)
known_results[key] = script_results
with open(path_to_results, "w") as f:
json.dump(dict(sorted(known_results.items())), f, indent=4)
print("#" * 100)
elif known_results[key].get("name") is None:
known_results[key]["name"] = name
with open(path_to_results, "w") as f:
json.dump(dict(sorted(known_results.items())), f, indent=4)
def evaluate_all(data_path):
sum = 0
sum_time = 0
for i in range(5):
start_time = time.time()
output_csv_file_name = 'error_analysis{}.csv'.format(i)
dat = os.path.join(data_path, 'dat{}_'.format(i))
hensyukyori = main(dat + 'in',
dat + 'out',
dat + 'ref',
is_analysis=True,
output_csv_file_name=output_csv_file_name)
sum += hensyukyori
elapsed_time = time.time() - start_time
sum_time += elapsed_time
print(hensyukyori)
print('[INFO] ave of hensyukyori: {}'.format(sum / 5))
print('[INFO] ave time: {}'.format(sum_time / 5))
def test(model, args, data):
device = torch.device("cuda:{}".format(args.gpu) if torch.cuda.is_available() else "cpu")
criterion = nn.CrossEntropyLoss()
loss = 0
answers = dict()
model.eval()
with torch.set_grad_enabled(False):
count = 0
for batch in iter(data.dev_iter):
if count > 50:
break
count += 1
if list((batch.q_char).size())[2] <= 3:
continue
p1, p2 = model(batch)
batch_loss = criterion(p1, batch.s_idx) + criterion(p2, batch.e_idx)
loss += batch_loss.item()
# (batch, c_len, c_len)
batch_size, c_len = p1.size()
ls = nn.LogSoftmax(dim=1) # 对每一行进行log softmax
mask = (torch.ones(c_len, c_len) * float('-inf')).to(device).tril(-1).unsqueeze(0).expand(batch_size, -1, -1)
score = (ls(p1).unsqueeze(2) + ls(p2).unsqueeze(1)) + mask
score, s_idx = score.max(dim=1)
score, e_idx = score.max(dim=1)
s_idx = torch.gather(s_idx, 1, e_idx.view(-1, 1)).squeeze()
for i in range(batch_size):
id = batch.id[i]
answer = batch.c_word[0][i][s_idx.item():e_idx.item()+1]
answer = ' '.join([data.WORD.vocab.itos[idx] for idx in answer])
answers[id] = answer
# for name, param in model.named_parameters():
# if param.requires_grad:
# param.data.copy_(backup_params.get(name))
with open(args.prediction_file, 'w', encoding='utf-8') as f:
print(json.dumps(answers), file=f)
results = evaluate.main(args)
return loss, results['exact_match'], results['f1']
def test(model, args, data):
device = torch.device(
f"cuda:{args.gpu}" if torch.cuda.is_available() else "cpu")
criterion = nn.CrossEntropyLoss()
loss = 0
answers = dict()
model.eval()
for batch in iter(data.dev_iter):
with torch.no_grad():
p1, p2 = model(batch.c_char, batch.q_char, batch.c_word[0],
batch.q_word[0], batch.c_word[1], batch.q_word[1])
#p1, p2 = model(batch)
batch_loss = criterion(p1, batch.s_idx) + criterion(p2, batch.e_idx)
loss += batch_loss.item()
# (batch, c_len, c_len)
batch_size, c_len = p1.size()
ls = nn.LogSoftmax(dim=1)
mask = (torch.ones(c_len, c_len) *
float('-inf')).to(device).tril(-1).unsqueeze(0).expand(
batch_size, -1, -1)
score = (ls(p1).unsqueeze(2) + ls(p2).unsqueeze(1)) + mask
score, s_idx = score.max(dim=1)
score, e_idx = score.max(dim=1)
s_idx = torch.gather(s_idx, 1, e_idx.view(-1, 1)).squeeze()
for i in range(batch_size):
id = batch.id[i]
answer = batch.c_word[0][i][s_idx[i]:e_idx[i] + 1]
answer = ' '.join([data.WORD.vocab.itos[idx] for idx in answer])
answers[id] = answer
with open(args.prediction_file, 'w', encoding='utf-8') as f:
print(json.dumps(answers), file=f)
results = evaluate.main(args)
return loss, results['exact_match'], results['f1']
'''
Created on 4 mei 2013
@author: Erik Vandeputte
'''
import stap_4
import evaluate
for i in range(3):
stap_4.main()
evaluate.main()
def test2(self, inp, test_num):
#物理的照合
#名称称号
#カテゴリ照合
if inp != 0:
n = inp
else:
n = self.N
target = [u'が物理的に', u'の名称が', u'のカテゴリが']
result = []
wait = 0
self.msg.setText(u'左側に表示された文字と右側に表示された\
文字' + target[test_num - 2] + '同じならスペースを,\
異なるなら何も押さず待機してください.\
左側の文字は１０回ごとに変更されます')
self.msg.pos = (-0.9, 0.7)
t = 3.0
startClock = core.Clock()
pygame.mixer.init()
pygame.mixer.music.load(start)
pygame.mixer.music.play(1)
while t >= 0.2:
t = 3.6 - startClock.getTime()
self.counter.setText('%.0f' % (t))
self.msg.draw()
self.counter.draw()
self.myWin.flip()
self.myWin.flip()
time.sleep(1.5)
pygame.mixer.music.stop()
time.sleep(0.01)
i = 0
error = 0
is_error = False
left_char = ''
right_char = ''
while i < n:
left_char, right_char, is_correct = self.get_char(
test_num, i, left_char, is_error)
self.test_ch1.setText(left_char)
self.test_ch2.setText(right_char)
self.test_ch1.draw()
if i % 10 == 0 and i != 0 and is_error == False:
self.change.draw()
self.myWin.flip()
time.sleep(1.5)
self.test_ch1.setText(left_char)
self.test_ch1.draw()
self.myWin.flip()
wait = random.uniform(1, 1.5)
self.test_ch1.draw()
self.test_ch2.draw()
time.sleep(wait)
self.myWin.flip()
event.clearEvents()
testClock = core.Clock()
while True:
key = event.getKeys()
if 'space' in key and is_correct == True:
result.append(testClock.getTime())
self.sound(2, 0, result[-1])
is_error = False
i += 1
break
elif 'space' in key and is_correct == False:
self.sound(2, 1, 0)
is_error = True
error += 1
break
elif testClock.getTime() > 1.5 and is_correct == True:
self.sound(2, 1, 0)
is_error = True
error += 1
break
elif testClock.getTime() > 1.5 and is_correct == False:
is_error = True
break
self.msg.pos = (-0.7, 0.7)
if inp != 0:
return
elif error / n < 0.3:
evaluate.main(self.myWin, test_num, result, False)
return result
else:
evaluate.main(self.myWin, test_num, result, True)
return
scale_num_rects = [];
scale_recall = [];
for idx in range(0, len(scale_range)-1):
cur_annolist = copy.deepcopy(annolist);
cur_det_annolist = copy.deepcopy(det_annolist);
cur_num_rects = filter_by_scale_annolist(cur_annolist, scale_range[idx], scale_range[idx+1]);
#saveXML("/afs/cs.stanford.edu/u/andriluka/code/cartrack_hossein/sail-car-log/car_tracking/tmp2/" + str(math.floor(scale_range[idx])) + "_" + str(math.floor(scale_range[idx+1]))a + ".al", cur_annolist);
scale_num_rects.append(cur_num_rects);
#filter_by_scale_annolist(cur_det_annolist, scale_range[idx], scale_range[idx+1]);
#recalls, precs = evaluate.main(cur_annolist, cur_det_annolist);
recalls, precs = evaluate.main(cur_annolist, det_annolist);
scale_recall.append(recalls[-1]);
#print "scale range: (%.2f, %.2f), recall: %.2f, precision: %.2f\n\n" % (scale_range[idx], scale_range[idx+1], recalls[-1], precs[-1]);
recalls, precs = evaluate.main(annolist, det_annolist);
print "recall: %.2f, precision: %.2f" % (recalls[-1], precs[-1]);
print scale_range
print scale_num_rects
print scale_recall
elif opts.do_clip_path:
for a in annolist:
n_found = 0;
def cw_tree_attack_targeted():
cw = CarliniL2_qa(debug=args.debugging)
criterion = nn.CrossEntropyLoss()
loss = 0
tot = 0
adv_loss = 0
targeted_success = 0
untargeted_success = 0
adv_text = []
answers = dict()
adv_answers = dict()
# model.eval()
embed = torch.load(args.word_vector)
device = torch.device("cuda:0" if args.cuda else "cpu")
vocab = Vocab(filename=args.dictionary,
data=[PAD_WORD, UNK_WORD, EOS_WORD, SOS_WORD])
generator = Generator(args.test_data, vocab=vocab, embed=embed)
transfered_embedding = torch.load('bidaf_transfered_embedding.pth')
transfer_emb = torch.nn.Embedding.from_pretrained(transfered_embedding).to(
device)
seqback = WrappedSeqback(embed,
device,
attack=True,
seqback_model=generator.seqback_model,
vocab=vocab,
transfer_emb=transfer_emb)
treelstm = generator.tree_model
generator.load_state_dict(torch.load(args.load_ae))
backup_params = EMA(0)
for name, param in model.named_parameters():
if param.requires_grad:
backup_params.register(name, param.data)
param.data.copy_(ema.get(name))
class TreeModel(nn.Module):
def __init__(self):
super(TreeModel, self).__init__()
self.inputs = None
def forward(self, hidden):
self.embedding = seqback(hidden)
return model(batch, perturbed=self.embedding)
def set_temp(self, temp):
seqback.temp = temp
def get_embedding(self):
return self.embedding
def get_seqback(self):
return seqback
tree_model = TreeModel()
for batch in tqdm(iter(data.dev_iter), total=1000):
p1, p2 = model(batch)
orig_answer, orig_s_idx, orig_e_idx = write_to_ans(
p1, p2, batch, answers)
batch_loss = criterion(p1, batch.s_idx) + criterion(p2, batch.e_idx)
loss += batch_loss.item()
append_info = append_input(batch, vocab)
batch_add_start = append_info['add_start']
batch_add_end = append_info['add_end']
batch_start_target = torch.LongTensor(
append_info['target_start']).to(device)
batch_end_target = torch.LongTensor(
append_info['target_end']).to(device)
add_sents = append_info['append_sent']
input_embedding = model.word_emb(batch.c_word[0])
append_info['tree'] = [generator.get_tree(append_info['tree'])]
seqback.sentences = input_embedding.clone().detach()
seqback.batch_trees = append_info['tree']
seqback.batch_add_sent = append_info['ae_sent']
seqback.start = append_info['add_start']
seqback.end = append_info['add_end']
seqback.adv_sent = []
batch_tree_embedding = []
for bi, append_sent in enumerate(append_info['ae_sent']):
seqback.target_start = append_info['target_start'][
0] - append_info['add_start'][0]
seqback.target_end = append_info['target_end'][0] - append_info[
'add_start'][0]
sentences = [
torch.tensor(append_sent, dtype=torch.long, device=device)
]
seqback.target = sentences[0][seqback.
target_start:seqback.target_end + 1]
trees = [append_info['tree'][bi]]
tree_embedding = treelstm(sentences, trees)[0][0].detach()
batch_tree_embedding.append(tree_embedding)
hidden = torch.cat(batch_tree_embedding, dim=0)
cw.batch_info = append_info
cw.num_classes = append_info['tot_length']
adv_hidden = cw.run(tree_model,
hidden, (batch_start_target, batch_end_target),
input_token=input_embedding)
seqback.adv_sent = []
# re-test
for bi, (add_start,
add_end) in enumerate(zip(batch_add_start, batch_add_end)):
if bi in cw.o_best_sent:
ae_words = cw.o_best_sent[bi]
bidaf_tokens = bidaf_convert_to_idx(ae_words)
batch.c_word[0].data[bi, add_start:add_end] = torch.LongTensor(
bidaf_tokens)
p1, p2 = model(batch)
adv_answer, adv_s_idx, adv_e_idx = write_to_ans(
p1, p2, batch, adv_answers)
batch_loss = criterion(p1, batch.s_idx) + criterion(p2, batch.e_idx)
adv_loss += batch_loss.item()
for bi, (start_target, end_target) in enumerate(
zip(batch_start_target, batch_end_target)):
start_output = adv_s_idx
end_output = adv_e_idx
targeted_success += int(
compare(start_output, start_target.item(), end_output,
end_target.item()))
untargeted_success += int(
compare_untargeted(start_output, start_target.item(),
end_output, end_target.item()))
for i in range(len(add_sents)):
logger.info(("orig:", transform(add_sents[i])))
try:
logger.info(("adv:", cw.o_best_sent[i]))
adv_text.append({
'adv_text': cw.o_best_sent[i],
'qas_id': batch.id[i],
'adv_predict': (orig_s_idx, orig_e_idx),
'orig_predict': (adv_s_idx, adv_e_idx),
'Orig answer:': orig_answer,
'Adv answer:': adv_answer
})
joblib.dump(adv_text, root_dir + '/adv_text.pkl')
except:
adv_text.append({
'adv_text': transform(add_sents[i]),
'qas_id': batch.id[i],
'adv_predict': (orig_s_idx, orig_e_idx),
'orig_predict': (adv_s_idx, adv_e_idx),
'Orig answer:': orig_answer,
'Adv answer:': adv_answer
})
joblib.dump(adv_text, root_dir + '/adv_text.pkl')
continue
# for batch size = 1
tot += 1
logger.info(("orig predict", (orig_s_idx, orig_e_idx)))
logger.info(("adv append predict", (adv_s_idx, adv_e_idx)))
logger.info(("targeted successful rate:", targeted_success))
logger.info(("untargetd successful rate:", untargeted_success))
logger.info(("Orig answer:", orig_answer))
logger.info(("Adv answer:", adv_answer))
logger.info(("tot:", tot))
for name, param in model.named_parameters():
if param.requires_grad:
param.data.copy_(backup_params.get(name))
with open(options.prediction_file, 'w', encoding='utf-8') as f:
print(json.dumps(answers), file=f)
with open(options.prediction_file + '_adv.json', 'w',
encoding='utf-8') as f:
print(json.dumps(adv_answers), file=f)
results = evaluate.main(options)
logger.info(tot)
logger.info(("adv loss, results['exact_match'], results['f1']", loss,
results['exact_match'], results['f1']))
return loss, results['exact_match'], results['f1']
def cw_random_word_attack():
cw = CarliniL2_untargeted_qa(debug=args.debugging)
criterion = nn.CrossEntropyLoss()
loss = 0
adv_loss = 0
targeted_success = 0
untargeted_success = 0
adv_text = []
answers = dict()
adv_answers = dict()
backup_params = EMA(0)
for name, param in model.named_parameters():
if param.requires_grad:
backup_params.register(name, param.data)
param.data.copy_(ema.get(name))
tot = 0
for batch in tqdm(iter(data.dev_iter), total=1000):
p1, p2 = model(batch)
orig_answer, orig_s_idx, orig_e_idx = write_to_ans(
p1, p2, batch, answers)
batch_loss = criterion(p1, batch.s_idx) + criterion(p2, batch.e_idx)
loss += batch_loss.item()
append_info = append_random_input(batch)
allow_idxs = append_info['allow_idx']
batch_start_target = torch.LongTensor([0]).to(device)
batch_end_target = torch.LongTensor([0]).to(device)
input_embedding = model.word_emb(batch.c_word[0])
cw_mask = np.zeros(input_embedding.shape).astype(np.float32)
cw_mask = torch.from_numpy(cw_mask).float().to(device)
for bi, allow_idx in enumerate(allow_idxs):
cw_mask[bi, np.array(allow_idx)] = 1
cw.wv = model.word_emb.weight
cw.inputs = batch
cw.mask = cw_mask
cw.batch_info = append_info
cw.num_classes = append_info['tot_length']
# print(transform(to_list(batch.c_word[0][0])))
cw.run(model, input_embedding, (batch_start_target, batch_end_target))
# re-test
for bi, allow_idx in enumerate(allow_idxs):
if bi in cw.o_best_sent:
for i, idx in enumerate(allow_idx):
batch.c_word[0].data[bi, idx] = cw.o_best_sent[bi][i]
p1, p2 = model(batch)
adv_answer, adv_s_idx, adv_e_idx = write_to_ans(
p1, p2, batch, adv_answers)
batch_loss = criterion(p1, batch.s_idx) + criterion(p2, batch.e_idx)
adv_loss += batch_loss.item()
for bi, (start_target, end_target) in enumerate(
zip(batch_start_target, batch_end_target)):
start_output = adv_s_idx
end_output = adv_e_idx
targeted_success += int(
compare(start_output, start_target.item(), end_output,
end_target.item()))
untargeted_success += int(
compare_untargeted(start_output, start_target.item(),
end_output, end_target.item()))
for i in range(len(allow_idxs)):
try:
logger.info(("adv:", transform(cw.o_best_sent[i])))
adv_text.append({
'added_text':
transform(cw.o_best_sent[i]),
'adv_text':
transform(to_list(batch.c_word[0][0])),
'qas_id':
batch.id[i],
'adv_predict': (orig_s_idx, orig_e_idx),
'orig_predict': (adv_s_idx, adv_e_idx),
'Orig answer:':
orig_answer,
'Adv answer:':
adv_answer
})
joblib.dump(adv_text, root_dir + '/adv_text.pkl')
except:
adv_text.append({
'adv_text':
transform(to_list(batch.c_word[0][0])),
'qas_id':
batch.id[i],
'adv_predict': (orig_s_idx, orig_e_idx),
'orig_predict': (adv_s_idx, adv_e_idx),
'Orig answer:':
orig_answer,
'Adv answer:':
adv_answer
})
joblib.dump(adv_text, root_dir + '/adv_text.pkl')
continue
# for batch size = 1
tot += 1
logger.info(("orig predict", (orig_s_idx, orig_e_idx)))
logger.info(("adv append predict", (adv_s_idx, adv_e_idx)))
logger.info(("targeted successful rate:", targeted_success))
logger.info(("untargetd successful rate:", untargeted_success))
logger.info(("Orig answer:", orig_answer))
logger.info(("Adv answer:", adv_answer))
logger.info(("tot:", tot))
for name, param in model.named_parameters():
if param.requires_grad:
param.data.copy_(backup_params.get(name))
with open(options.prediction_file, 'w', encoding='utf-8') as f:
print(json.dumps(answers), file=f)
with open(options.prediction_file + '_adv.json', 'w',
encoding='utf-8') as f:
print(json.dumps(adv_answers), file=f)
results = evaluate.main(options)
logger.info(tot)
logger.info(("adv loss, results['exact_match'], results['f1']", loss,
results['exact_match'], results['f1']))
return loss, results['exact_match'], results['f1']
out_pred = []
out_emb = []
out_target = []
for xr, y in test_iter:
x, x_m = batch.prepare_data(xr, chardict, n_tokens=n_char)
p = predict(x, x_m)
e = encode(x, x_m)
ranks = np.argsort(p)[:, ::-1]
for idx, item in enumerate(xr):
out_data.append(item)
out_pred.append(ranks[idx, :])
out_emb.append(e[idx, :])
out_target.append(y[idx])
# Save
print ("Saving...")
with open("%s/data.pkl" % save_path, "w") as f:
pkl.dump(out_data, f)
with open("%s/predictions.npy" % save_path, "w") as f:
np.save(f, np.asarray(out_pred))
with open("%s/embeddings.npy" % save_path, "w") as f:
np.save(f, np.asarray(out_emb))
with open("%s/targets.pkl" % save_path, "w") as f:
pkl.dump(out_target, f)
if __name__ == "__main__":
main(sys.argv[1:])
evaluate.main(sys.argv[3], sys.argv[2])
def main():
parser = argparse.ArgumentParser(description="CliRel (Clinical Relation) \
extractor- trains a classifier able to \
determine the type of relation between \
two medical concepts in a sentence.")
# Add arguments here
parser.add_argument("--train", nargs=3,
metavar=("train_dir", "model_file", "model_type"), type=str,
help="Directory should contain three subdirs (txt, \
concept, rel) containing .txt, .con, .rel files. \
Will train a classifier on this data. \
Trained model will be written to specified model file.\n \
Current model types:[svm-spt, svm-insert, svm-suffix]",
default=None)
parser.add_argument("--predict", nargs=3,
metavar=("test_dir", "model_file", "results_dir"), type=str,
help="Directory contains concept and text files \
that the specified (or default) model will predict. \
Resulting relation files will be written to \
the specified results directory.",
default=None)
parser.add_argument("--evaluate", nargs=3,
metavar=("test_dir", "gold_dir", "eval_file"), type=str,
help="Evaluate the relation files in the test directory \
in comparison with those in the gold directory. The \
results will be written to the evaluation file.",
default=None)
parser.add_argument("--verbose", action="store_true",
default=False, help="Show debugging info.")
args = parser.parse_args()
if not args.predict and not args.train and not args.evaluate:
sys.stderr.write("ERROR: No valid flag specified.\n")
parser.print_help()
sys.exit(1)
if args.train:
checkDir(args.train[0])
checkDir(os.path.dirname(args.train[1]))
if (os.path.isdir(args.train[1])):
sys.stderr.write("ERROR: Model expected to be a file, %s is a directory\n"
% args.train[1])
sys.exit(1)
train.main(args.train[0], args.train[1], args.train[2], args.verbose)
if args.predict:
checkDir(args.predict[0])
checkFile(args.predict[1])
checkDir(args.predict[2])
predict.main(args.predict[0], args.predict[1], args.predict[2], args.verbose)
if args.evaluate:
checkDir(args.evaluate[0])
checkDir(args.evaluate[1])
checkDir(os.path.dirname(args.evaluate[2]))
if (os.path.isdir(args.evaluate[2])):
sys.stderr.write("ERROR: eval_file expected to be a file, %s is a \
directory\n" % args.evaluate[2])
sys.exit(1)
evaluate.main(args.evaluate[0], args.evaluate[1], args.evaluate[2], args.verbose)
def main(train_file = tr_f,tag=tg,test_file=te_f,out_file = output,gold_file = g_f): pos_tagger.main(train_file,tag,test_file) evaluate.main(out_file,tag,gold_file)
