word_size, pos_embedding_tensor = utils.initialize_embedding(
feature_file, embLen)
doc_size, type_size, feature_list, label_list, type_list = utils.load_corpus(
test_file)
# doc_size_test, _, feature_list_test, label_list_test, type_list_test = utils.load_corpus(test_file)
# doc_size_dev, _, feature_list_dev, label_list_dev, type_list_dev = utils.load_corpus(dev_file)
f1_mean = 0
precision_mean = 0
recall_mean = 0
cdev_f1_mean = 0
for i in range(100):
nocluster = noCluster.noCluster(embLen, word_size, type_size, drop_prob)
nocluster.load_state_dict(
torch.load('./dumped_models/ffnn_dump_' + str(dataset) + '.pth'))
nocluster.freeze_params()
# optimizer = utils.sgd(nocluster.parameters(), lr=0.025)
optimizer = optim.SGD(filter(lambda p: p.requires_grad,
nocluster.parameters()),
lr=0.05)
scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer,
mode='max',
factor=0.5,
patience=10)
# f = filter(lambda p: p.requires_grad, nocluster.parameters())
# for item in list(f):
# print(item)
test_file = './data/intermediate/' + dataset + '/rm/test.data'
feature_file = './data/intermediate/' + dataset + '/rm/feature.txt'
type_file = './data/intermediate/' + dataset + '/rm/type.txt'
type_file_test = './data/intermediate/' + dataset + '/rm/type_test.txt'
none_ind = utils.get_none_id(type_file)
print("None id:", none_ind)
label_distribution = utils.get_distribution(type_file)
label_distribution_test = utils.get_distribution(type_file_test)
word_size, pos_embedding_tensor = utils.initialize_embedding(
feature_file, embLen)
_, type_size, _, _, _ = utils.load_corpus(train_file)
nocluster = noCluster.noCluster(embLen, word_size, type_size, drop_prob,
label_distribution, label_distribution_test)
nocluster.load_state_dict(
torch.load('./dumped_models/ffnn_dump_' + '_'.join(sys.argv[1:7]) +
'.pth'))
torch.cuda.set_device(0)
nocluster.cuda()
if_cuda = True
packer = pack.repack(repack_ratio, 20, if_cuda)
print('in the order of: train, dev, test...\n')
for file in [train_file, dev_file, test_file]:
doc_size_test, _, feature_list_test, label_list_test, type_list_test = utils.load_corpus(
file)
opt_new = vars(args)
# set random seed
SEED = opt_new['seed']
print('Using Random Seed: '+str(SEED))
torch.manual_seed(SEED)
np.random.seed(SEED)
random.seed(SEED)
# load model
save_path = os.path.join(opt_new['save_dir'], opt_new['info'])
save_filename = os.path.join(save_path, 'best_model.pth')
loaded = torch.load(save_filename)
opt = loaded['config']
nocluster = noCluster.noCluster(opt)
nocluster.load_state_dict(loaded['model'], strict=False)
torch.cuda.set_device(0)
nocluster.cuda()
nocluster.eval()
data_dir = os.path.join(opt['data_dir'], opt['dataset'], 'rm')
dev_file = os.path.join(data_dir, 'dev.data')
test_file = os.path.join(data_dir, 'test.data')
feature_file = os.path.join(data_dir, 'feature.txt')
type_file = os.path.join(data_dir, 'type.txt')
type_file_test = os.path.join(data_dir, 'type_test.txt')
none_ind = nocluster.opt['none_ind']