def train(args):
"""
This function trains the models
:param args: the command line arguments defining the desired actions
"""
# load data
train_data_all, dev_data_all, _ = load(args.data_dir,
cachedir=args.cachedir,
override_cache=args.override_cache,
text_only=(args.model.lower()
in ["bi-lstm", "bert"]),
include_tfidf=args.include_tfidf,
balanced=args.balanced)
train_data, train_labels = train_data_all.X, train_data_all.y
dev_data, dev_labels = dev_data_all.X, dev_data_all.y
# Build model
apx = get_appendix(args.include_tfidf, args.balanced)
if args.model.lower() == "simple-ff":
model = FeedForward(args.ff_hunits, train_data.shape[1])
train_pytorch(args,
model,
train_data,
train_labels,
dev_data,
dev_labels,
save_model_path=f"models/simple-ff{apx}.torch")
elif args.model.lower() == "bi-lstm":
model = BiLSTM(epochs=args.num_epochs,
batch_size=args.batch_size,
max_seq_len=args.max_seq_len)
model.train(train_data, train_labels, dev_data, dev_labels)
elif args.model.lower() == "logreg":
model = LogisticRegression()
model.train(train_data,
train_labels,
dev_data,
dev_labels,
save_model_path=f"models/logreg{apx}.pkl")
elif args.model.lower() == "majority-vote":
model = MajorityVote()
model.train(train_labels, dev_labels)
elif args.model.lower() == "bert":
model = Bert(epochs=args.num_epochs,
batch_size=args.batch_size,
max_seq_len=args.max_seq_len,
learning_rate=args.learning_rate)
model.train(train_data,
train_labels,
dev_data,
dev_labels,
save_model_path=f"models/bert.pkl")
elif args.model.lower() == "svm":
model = SVM()
model.train(train_data,
train_labels,
save_model_path=f"models/svm{apx}.sav")
else:
raise Exception("Unknown model type passed in!")
def main(args):
exp_info = exp_config.Experiment(args.dataset)
paths = exp_info.paths
args.paths = paths
args.metadata = exp_info.metadata
np.random.seed(args.seed)
torch.manual_seed(args.seed)
args.batch_size = 1
feature_size, train_loader, val_loader, test_loader, all_loader = exp_info.get_dataset(
args, save=True)
label_num = exp_info.get_label_num(args)
hidden_size = 256
hidden_layers = 2
if args.model == 'lstm':
parsing_model = lstm_model.BiLSTM(feature_size, hidden_size,
hidden_layers, label_num)
else:
parsing_model = mlp_model.MLP(feature_size, hidden_size, label_num)
parsing_model = torch.nn.DataParallel(parsing_model)
prev = args.subsample
args.subsample = 1
args.save_path = os.path.join(paths.inter_root, 'likelihood', args.task,
args.model)
args.resume = os.path.join(
paths.checkpoint_root,
'detection_{}_{}_e{}_lr{}_b{}_lrd{}_s{}_do{}'.format(
args.task, args.model, args.epochs, args.lr, args.using_batch_size,
args.lr_decay, 1 if not args.subsample else args.subsample,
args.dropout_rate))
args.subsample = prev
logutils.load_checkpoint(args, parsing_model)
validate(test_loader, parsing_model, args=args)
def get_model(cfg):
num_classes = int(cfg['n_classes'])
input_size = int(cfg['data_dim'])
if cfg['model'] == 'blstm':
classifier = BiLSTM(input_size,
n_classes=num_classes,
embedding_size=128,
hidden_size=256,
dropout=cfg['dropout'])
if cfg['model'] == 'sdec':
classifier = DECSeq(
input_size,
int(cfg['embedding_size']),
num_classes,
#dropout=cfg['dropout'],
k=5,
aggr='max',
pool_op=cfg['pool_op'])
if cfg['model'] == 'dec':
classifier = DEC(
input_size,
int(cfg['embedding_size']),
num_classes,
#dropout=cfg['dropout'],
k=5,
aggr='max',
pool_op=cfg['pool_op'])
elif cfg['model'] == 'pn_geom':
classifier = PNptg2(input_size,
int(cfg['embedding_size']),
num_classes,
same_size=cfg['same_size'])
return classifier
def get_model(cfg):
num_classes = int(cfg['n_classes'])
input_size = int(cfg['data_dim'])
if cfg['model'] == 'blstm':
classifier = BiLSTM(input_size,
n_classes=num_classes,
embedding_size=128,
hidden_size=256,
dropout=cfg['dropout'])
if cfg['model'] == 'dec_ori':
classifier = DGCNNSeq(input_size,
int(cfg['embedding_size']),
num_classes,
batch_size=int(cfg['batch_size']),
k=5,
fov=1,
dropout=0.5)
if cfg['model'] == 'dec':
classifier = DECSeq(
input_size,
int(cfg['embedding_size']),
num_classes,
dropout=cfg['dropout'],
#fov=3,
k=int(cfg['k_dec']),
# k=5,
aggr='max',
#pool_op=global_max_pool)
pool_op=cfg['pool_op'])
#bn=True)
if cfg['model'] == 'nnc':
classifier = NNC(input_size,
int(cfg['embedding_size']),
num_classes,
batch_size=int(cfg['batch_size']),
pool_op=cfg['pool_op'],
same_size=cfg['same_size'])
if cfg['model'] == 'gcn':
classifier = GCNConvNetBN(input_size,
int(cfg['embedding_size']),
num_classes,
pool_op=cfg['pool_op'],
same_size=cfg['same_size'])
if cfg['model'] == 'gat':
classifier = GAT(input_size,
num_classes)
elif cfg['model'] == 'pn_geom':
classifier = PNptg2(input_size,
int(cfg['embedding_size']),
num_classes,
#batch_size=int(cfg['batch_size']),
same_size=cfg['same_size'])
return classifier
def get_model(cfg):
num_classes = int(cfg['n_classes'])
input_size = int(cfg['data_dim'])
n_gf = int(cfg['num_gf'])
if cfg['model'] == 'blstm':
classifier = BiLSTM(input_size,
n_classes=num_classes,
embedding_size=128,
hidden_size=256)
if cfg['model'] == 'dec_ori':
classifier = DGCNNSeq(input_size,
int(cfg['embedding_size']),
num_classes,
batch_size=int(cfg['batch_size']),
k=3,
fov=1,
dropout=0.5)
if cfg['model'] == 'dec':
classifier = DECSeq(
input_size,
int(cfg['embedding_size']),
num_classes,
#fov=3,
batch_size=int(cfg['batch_size']),
k=7,
aggr='max',
pool_op=global_max_pool,
same_size=cfg['same_size'])
#bn=True)
if cfg['model'] == 'nnc':
classifier = NNC(input_size,
int(cfg['embedding_size']),
num_classes,
batch_size=int(cfg['batch_size']),
pool_op=global_max_pool,
same_size=cfg['same_size'])
if cfg['model'] == 'gcn':
classifier = GCNConvNet(input_size,
int(cfg['embedding_size']),
num_classes,
batch_size=int(cfg['batch_size']),
pool_op=global_max_pool,
same_size=cfg['same_size'])
elif cfg['model'] == 'pn_geom':
classifier = PNptg(input_size,
int(cfg['embedding_size']),
num_classes,
batch_size=int(cfg['batch_size']),
pool_op=global_max_pool,
same_size=cfg['same_size'])
return classifier
def main():
# save file
config.mulu = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
config.save_dir = os.path.join(config.save_direction, config.mulu)
if not os.path.isdir(config.save_dir):
os.makedirs(config.save_dir)
# get iter
create_alphabet = None
train_iter, dev_iter, test_iter, create_alphabet = load_Data(config)
config.embed_num = create_alphabet.word_alphabet.vocab_size
config.class_num = create_alphabet.label_alphabet.vocab_size
config.paddingId = create_alphabet.word_paddingId
config.label_paddingId = create_alphabet.label_paddingId
config.create_alphabet = create_alphabet
print("embed_num : {}, class_num : {}".format(config.embed_num, config.class_num))
print("PaddingID {}".format(config.paddingId))
if config.pretrained_embed:
print("Using Pre_Trained Embedding.")
pretrain_embed = load_pretrained_emb_zeros(path=config.pretrained_embed_file,
text_field_words_dict=create_alphabet.word_alphabet.id2words,
pad=paddingkey)
config.pretrained_weight = pretrain_embed
model = None
if config.model_BiLstm is True:
print("loading model.....")
model = BiLSTM(config)
print(model)
if config.use_cuda is True:
model = model.cuda()
print("Training Start......")
train.train(train_iter=train_iter, dev_iter=dev_iter, test_iter=test_iter, model=model, config=config)
def main(args):
exp_info = exp_config.Experiment(args.dataset)
paths = exp_info.paths
args.paths = paths
args.save_path = os.path.join(paths.inter_root, 'likelihood', args.task,
args.model)
args.resume = os.path.join(
paths.checkpoint_root,
'detection_{}_{}_e{}_lr{}_b{}_lrd{}_s{}_do{}'.format(
args.task, args.model, args.epochs, args.lr, args.batch_size,
args.lr_decay, 1 if not args.subsample else args.subsample,
args.dropout_rate))
np.random.seed(args.seed)
torch.manual_seed(args.seed)
feature_size, train_loader, val_loader, test_loader, all_loader = exp_info.get_dataset(
args, save=True)
label_num = exp_info.get_label_num(args)
criterion = torch.nn.CrossEntropyLoss()
hidden_size = 256
hidden_layers = 2
if args.model == 'lstm':
model = lstm_model.BiLSTM(feature_size,
hidden_size,
hidden_layers,
label_num,
dropout_rate=args.dropout_rate)
else:
model = mlp_model.MLP(feature_size,
hidden_size,
label_num,
dropout_rate=args.dropout_rate)
optimizer = optim.Adam(model.parameters(), lr=args.lr)
scheduler = lr_scheduler.StepLR(optimizer, args.lr_freq, args.lr_decay)
model = torch.nn.DataParallel(model)
if args.cuda:
criterion = criterion.cuda()
model = model.cuda()
if args.resume:
logutils.load_checkpoint_epoch(args, model, args.trained_epochs,
optimizer, scheduler)
best_prec = 0.0
if not args.eval:
best_dict = None
for epoch in tqdm(range(args.start_epoch, args.epochs),
desc='Epochs Loop'):
train(train_loader, model, criterion, optimizer, epoch, args)
prec = validate(test_loader, model, args)
best_prec = max(prec, best_prec)
is_best = (best_prec == prec)
tqdm.write('Best precision: {:.03f}'.format(best_prec))
scheduler.step()
if is_best or (epoch + 1) % args.save_interval == 0:
current_dict = {
'epoch': epoch + 1,
'state_dict': copy.deepcopy(model.state_dict()),
'best_prec': prec,
'optimizer': copy.deepcopy(optimizer.state_dict()),
'scheduler': copy.deepcopy(scheduler.state_dict())
}
if is_best or (not best_dict):
best_dict = copy.deepcopy(current_dict)
logutils.save_checkpoint(best_dict, is_best, args)
if (epoch + 1) % args.save_interval == 0:
logutils.save_checkpoint_epoch(best_dict, epoch + 1, args)
else:
validate(test_loader, model, args, test=True)
if args.save:
validate(all_loader, model, args, test=True, save=True)
def train():
print("Training the model.")
print("Split method:", SPLIT_METHOD)
print("Sequence Length:", SEQ_LENGTH)
model_path = get_model_path(SPLIT_METHOD, SEQ_LENGTH, FEAT_MODEL, FEAT_NUM)
print("Model path/name:", model_path)
n_epochs = 200
between_name = get_between_name(SPLIT_METHOD, SEQ_LENGTH, FEAT_MODEL,
FEAT_NUM)
X_tr = np.load(os.path.join(VARS_DIR, "X_" + between_name + "_train.npy"))
Y_tr = np.load(os.path.join(VARS_DIR, "Y_" + between_name + "_train.npy"))
X_val = np.load(os.path.join(VARS_DIR, "X_" + between_name + "_val.npy"))
Y_val = np.load(os.path.join(VARS_DIR, "Y_" + between_name + "_val.npy"))
X_test = np.load(os.path.join(VARS_DIR, "X_" + between_name + "_test.npy"))
Y_test = np.load(os.path.join(VARS_DIR, "Y_" + between_name + "_test.npy"))
print(X_tr.shape, Y_tr.shape)
print(X_val.shape, Y_val.shape)
print(X_test.shape, Y_test.shape)
X_tr, X_val, X_test = normalize([X_tr, X_val, X_test])
if SEQ_LENGTH > 1:
model = BiLSTM(FEAT_NUM, 256, nb_classes=NB_CLASS).to(DEVICE)
else:
model = SalakhNet(input_size=FEAT_NUM, nb_class=NB_CLASS).to(DEVICE)
load = True
if load and os.path.exists(model_path):
model.load_state_dict(torch.load(model_path))
print("Model Loaded")
optimizer = optim.Adam(model.parameters(), lr=0.001)
device = next(model.parameters()).device
loss_fn = nn.CrossEntropyLoss()
with torch.no_grad():
model.eval()
X_val = torch.Tensor(X_val).to(device)
X_test = torch.Tensor(X_test).to(device)
preds = model(X_val).log_softmax(dim=1).cpu().numpy().argmax(axis=1)
best_val_acc = np.sum(preds == Y_val) / len(preds) * 100
preds = model(X_test).log_softmax(dim=1).cpu().numpy().argmax(axis=1)
test_acc = np.sum(preds == Y_test) / len(preds) * 100
for epoch in range(1, n_epochs + 1):
model.train()
losses = []
n_batches = math.ceil(len(Y_tr) / get_batch_size(SEQ_LENGTH))
for batch_idx in range(n_batches):
optimizer.zero_grad()
s = batch_idx * get_batch_size(SEQ_LENGTH)
e = min(len(Y_tr), (batch_idx + 1) * get_batch_size(SEQ_LENGTH))
X_batch, Y_batch = torch.Tensor(
X_tr[s:e]).to(device), torch.LongTensor(Y_tr[s:e]).to(device)
preds = model(X_batch)
loss = loss_fn(preds, Y_batch)
losses.append(loss.item())
loss.backward()
optimizer.step()
# print("Train Loss:", np.mean(losses))
with torch.no_grad():
model.eval()
preds = model(X_val).log_softmax(dim=1).cpu().numpy().argmax(
axis=1)
val_acc = np.sum(preds == Y_val) / len(preds) * 100
if val_acc > best_val_acc:
best_val_acc = val_acc
preds = model(X_test).log_softmax(dim=1).cpu().numpy().argmax(
axis=1)
test_acc = np.sum(preds == Y_test) / len(preds) * 100
torch.save(model.state_dict(), model_path)
print("Val ACC: %.2f" % best_val_acc, "Test Acc: %.2f" % test_acc)
train_loader = DataLoader(dataset=dataset_train, **config["training"])
test_loader = DataLoader(dataset=dataset_test, **config["test"])
print(f"Total samples train: {len(dataset_train)}, "
f"Number of batches: {len(train_loader)}"
f"\nTotal samples test: {len(dataset_test)}, "
f"Number of batches: {len(test_loader)}")
# Define model
n_features = len(config["data"]["features"])
n_classes = 4
if config["data"]["all_labels"]:
n_classes = 9
print(f"Num classes: {n_classes}\n")
model = BiLSTM(n_features, n_classes, **config["network"]).to(device)
# Define loss and optimizer
criterion = nn.CrossEntropyLoss()
lr = config["optimizer"]["learning_rate"]
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
if args.resume:
model.load_state_dict(checkpoint["model_state_dict"])
optimizer.load_state_dict(checkpoint["optimizer_state_dict"])
def decentralized_coordinate(coords):
decentralized_coords = coords - torch.min(coords, axis=0).values
return decentralized_coords
def train(loader, log_interval, max_batches=None):
np.random.seed(args.seed)
import dynet_config
dynet_config.set(random_seed=args.seed)
from train_funcs import *
dyparams = dy.DynetParams()
dyparams.set_autobatch(True)
from models import BiLSTM, LSTMAtt
model = dy.ParameterCollection()
model_params = model
W2I, vecs = pickle.load(
open("models/" + args.pretrained_model + "/Model.meta", "rb")), None
# Create the base bilstm network, which will be pretrained.
base_network = BiLSTM(1, 300, 150, W2I, model, vecs)
# Load the pretrained bilstm network
base_network.load_model("models/" + args.pretrained_model + "/Model")
# Add attention weights to the pretrained bilstm network.
network = LSTMAtt(base_network, model, 300, 150)
if not os.path.exists(model_path):
os.makedirs(model_path)
os.makedirs(model_path + "/Results")
command_line_file = open(model_path + "/Command_Lines.txt", 'w')
for arg in sys.argv:
command_line_file.write(arg + "\n")
command_line_file.close()
else:
print "Model Folder Already exists."
sys.exit(1)
network.save_meta(model_path + "/Model.meta")
def training(train_loader, test_loader, opt):
net_list = [
LSTM.LSTM(opt),
BiLSTM.BiLSTM(opt),
OriTextCNN.OriTextCNN(opt),
DilaTextCNN.DilaTextCNN(opt),
TextCNNInc.TextCNNInc(opt)
]
net = net_list[4]
best_acc = 0
# best_top_accs = []
best_acc_loss = 0
NUM_TRAIN = len(train_loader) * opt.BATCH_SIZE
PRE_EPOCH = 0
NUM_TRAIN_PER_EPOCH = len(train_loader)
NET_PREFIX = opt.NET_SAVE_PATH + net.model_name + "/"
print('==> Loading Model ...')
temp_model_name = "temp_model.dat"
model_name = "best_model.dat"
if not os.path.exists(NET_PREFIX):
os.mkdir(NET_PREFIX)
if not os.path.exists('./source/log/' + net.model_name):
os.mkdir('./source/log/' + net.model_name)
if os.path.exists(NET_PREFIX + temp_model_name) and opt.RE_TRAIN == False:
try:
net, PRE_EPOCH, best_acc = net.load(NET_PREFIX + temp_model_name)
print("Load existing model: %s" % (NET_PREFIX + temp_model_name))
except:
pass
if opt.USE_CUDA: net.cuda()
criterion = nn.CrossEntropyLoss()
if opt.OPTIMIZER == 'Adam':
optimizer = torch.optim.Adam(net.parameters(), lr=opt.LEARNING_RATE)
elif opt.OPTIMIZER == 'SGD':
optimizer = torch.optim.SGD(net.parameters(), lr=opt.LEARNING_RATE)
elif opt.OPTIMIZER == 'RMSP':
optimizer = torch.optim.RMSprop(net.parameters(), lr=opt.LEARNING_RATE)
else:
raise NameError("This optimizer isn't defined")
train_recorder = {'loss': [], 'acc': []}
test_recorder = {'loss': [], 'acc': []}
t = datetime.datetime.now().strftime("%m%d_%H:%M:%S")
log_file_name = "%s_%s.txt" % (net.model_name, t)
for epoch in range(opt.NUM_EPOCHS):
train_loss = 0
train_acc = 0
# Start training
net.train()
for i, data in tqdm(enumerate(train_loader),
desc="Training",
total=NUM_TRAIN_PER_EPOCH,
leave=False,
unit='b'):
inputs, labels, sent = data
if opt.USE_CUDA:
inputs, labels = Variable(inputs.cuda()), Variable(
labels.cuda())
else:
inputs, labels = Variable(inputs), Variable(labels)
# zero the parameter gradients
optimizer.zero_grad()
# forward + backward + optimize
outputs = net(inputs)
loss = criterion(outputs, labels)
loss.backward()
optimizer.step()
# Do statistics for training
train_loss += loss.data[0]
_, predicts = torch.max(outputs, 1)
num_correct = (predicts == labels).sum()
train_acc += num_correct.data[0]
# Start testing
ave_test_loss, ave_test_acc, topnacc = testing(test_loader, net, opt)
ave_train_loss = float(train_loss) / NUM_TRAIN
ave_train_acc = float(train_acc) / NUM_TRAIN
# Do recording for each epoch
train_recorder['loss'].append(ave_train_loss)
train_recorder['acc'].append(ave_train_acc)
test_recorder['loss'].append(ave_test_loss)
test_recorder['acc'].append(ave_test_acc)
# Write log to files
with open('./source/log/' + net.model_name + '/' + log_file_name,
'w+') as fp:
json.dump(
{
'train_recorder': train_recorder,
'test_recorder': test_recorder
}, fp)
# Output results
print(
'Epoch [%d/%d], Train Loss: %.4f, Train Acc: %.4f, Test Loss: %.4f, Test Acc: %.4f'
% (epoch + 1 + PRE_EPOCH, opt.NUM_EPOCHS + PRE_EPOCH,
ave_train_loss, ave_train_acc, ave_test_loss, ave_test_acc))
if ave_test_acc > best_acc:
best_acc = ave_test_acc
best_top_accs = topnacc
best_acc_loss = ave_test_loss
net.save((epoch + PRE_EPOCH), best_acc, model_name)
# Save a temp model
net.save((epoch + PRE_EPOCH), best_acc, temp_model_name)
summary_info = {
'total_epoch': (epoch + PRE_EPOCH),
'best_acc': best_acc,
'best_acc_loss': best_acc_loss,
'ave_test_acc': ave_test_acc,
'ave_test_loss': ave_test_loss,
'ave_train_acc': ave_train_acc,
'ave_train_loss': ave_train_loss,
'best_top_accs': best_top_accs
}
write_summary(net, opt, summary_info)
print(
'==> Training Finished. Current model is %s. The highest test acc is %.4f'
% (net.model_name, best_acc))
return net
from models import BiLSTM
from utils import aspects_sent_convert, read_file_lines, write_file_lines
from import_dy import dy
from random_inits import np
# Use a sentence level classifier to automatically label sentences.
if __name__ == "__main__":
label_files = sys.argv[1].split(',')
lines = list()
for label_file in label_files:
lines += read_file_lines(label_file)
model = dy.ParameterCollection()
W2I = pickle.load(open("models/SentenceLevelBiLSTM" + "/Model.meta", "rb"))
network = BiLSTM(1, 300, 150, W2I, model)
network.load_model("models/SentenceLevelBiLSTM/Model")
new_samples = list()
for i in range(0, len(lines), 1000):
print str(i) + " Sentences Labeled"
samples = aspects_sent_convert(lines[i:i + 1000])
for sample in samples:
dy.renew_cg()
sent = sample.sent
h = network(sent[0])
dist = network.classify(h)
prediction = np.argmax(dist.npvalue())
sample_str = sent[0] + u"<->" + str(prediction)
if opt.USE_CHAR: opt.CHAR_SIZE = len(vocab_dict)
else: opt.VOCAB_SIZE = len(vocab_dict)
opt.NUM_CLASSES = len(title)
# trainData = prep.load_cls_data(data_dict, title, train=True)
testData = prep.load_cls_data(test_dict, title, train=False)
trainDataSet = BalancedData(data_dict,title,opt,vocab_dict=vocab_dict)
# trainDataSet = BeibeiClassification(trainData[:32], vocab_dict=vocab_dict, char=True)
testDataSet = BeibeiClassification(testData[:] , vocab_dict=vocab_dict, char=True)
train_loader = DataLoader(dataset=trainDataSet, batch_size=opt.BATCH_SIZE,
shuffle=True, num_workers=opt.NUM_WORKERS, drop_last=True)
test_loader = DataLoader(dataset=testDataSet, batch_size=opt.BATCH_SIZE,
shuffle=True, num_workers=opt.NUM_WORKERS, drop_last=True)
if opt.IS_TRAINING:
net = training(train_loader, test_loader, opt)
else:
if opt.ENSEMBLE_TEST:
net_list = [OriTextCNN.OriTextCNN(opt),BiLSTM.BiLSTM(opt)]
for i, _ in enumerate(net_list):
net_list[i], *_ = net_list[i].load(opt.NET_SAVE_PATH+ net_list[i].model_name + "/best_model.dat")
ave_test_loss, ave_test_acc, _ = ensemble_testing(test_loader, net_list, opt)
else:
net = LSTM.LSTM(opt)
net, *_ = net.load(opt.NET_SAVE_PATH + net.model_name + "/best_model.dat")
print('==> Now testing model: %s '%(net.model_name))
ave_test_loss, ave_test_acc = testing(test_loader, net, opt)
print( 'Test Loss: %.4f, Test Acc: %.4f'%(ave_test_loss, ave_test_acc))
train_loader, dev_loader, test_loader = (DataLoader(
dataset=dataset,
batch_size=args.batch_size,
collate_fn=utils.collate_fn,
shuffle=dataset.train) for i, dataset in enumerate(dataests))
print("Building Model...")
if args.model == "cnn":
model = CNN.Model(vocab_size=len(vocab),
embedding_size=args.embedding_size,
hidden_size=args.hidden_size,
filter_sizes=[3, 4, 5],
dropout=args.dropout)
elif args.model == "bilstm":
model = BiLSTM.Model(vocab_size=len(vocab),
embedding_size=args.embedding_size,
hidden_size=args.hidden_size,
dropout=args.dropout)
else:
raise ValueError(
"Model should be either cnn or bilstm, {} is invalid.".format(
args.model))
if torch.cuda.is_available():
model = model.cuda()
trainer = Trainer(model)
best_acc = 0
train_list = []
dev_list = []
for i in range(args.epochs):
from . import config as cfg
from models import BiLSTM
from data import liar_plus
model_name = 'BiDirectional LSTM'
model = BiLSTM(liar_plus.text.vocab.vectors,
lstm_layer=2,
padding_idx=liar_plus.text.vocab.stoi[liar_plus.text.pad_token],
hidden_dim=128).to(cfg.device)
)
os.makedirs(ckpt_prediction_folder, exist_ok=True)
# Load model config
with open(path_config, "r") as f:
config = yaml.safe_load(f)
# Load model
n_features = len(config["data"]["features"])
n_classes = 4
if config["data"]["all_labels"]:
n_classes = 9
print(f"Num classes: {n_classes}\n")
print("Loading model..", end=" ", flush=True)
model = BiLSTM(n_features, n_classes, **config["network"]).to(device)
model.load_state_dict(checkpoint["model_state_dict"])
model.eval()
print("DONE")
for path_ply in args.files:
path_ply = os.path.join(args.prefix_path, path_ply)
print(f"\nProcessing file: {path_ply}")
print("* Preparing dataloader..", end=" ", flush=True)
dataset = AerialPointDataset(path_ply, **config["data"])
loader = DataLoader(
dataset=dataset,
batch_size=args.batch_size,
num_workers=args.num_workers,
shuffle=False,
)
def main():
# save file
mulu = datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
args.mulu = mulu
args.save_dir = os.path.join(args.save_dir, mulu)
if not os.path.isdir(args.save_dir):
os.makedirs(args.save_dir)
# get iter
train_iter, dev_iter, test_iter, create_alphabet = load_Conll2003_NER(args)
args.embed_num = create_alphabet.word_alphabet.vocab_size
args.class_num = create_alphabet.label_alphabet.vocab_size
args.paddingId = create_alphabet.word_paddingId
args.create_alphabet = create_alphabet
print("embed_num : {}, class_num : {}".format(args.embed_num,
args.class_num))
print("PaddingID {}".format(args.paddingId))
if args.word_Embedding:
print("Using Pre_Trained Embedding.")
pretrain_embed = load_pretrained_emb_avg(
path=args.word_Embedding_Path,
text_field_words_dict=create_alphabet.word_alphabet.id2words,
pad=paddingkey)
args.pretrained_weight = pretrain_embed
# print params
show_params()
model = None
if args.BiLSTM is True:
print("loading BiLSTM model.....")
model = BiLSTM(args)
shutil.copy("./models/BiLSTM.py", args.save_dir)
print(model)
elif args.BiLSTM_CRF is True:
print("loading BiLSTM_CRF model......")
bilstm = BiLSTM(args)
crf = CRF(args)
model = BiLSTM_CRF(BiLSTM=bilstm, CRF=crf, args=args)
print(model)
if args.use_cuda is True:
print("Using Cuda To Speed Up......")
model = model.cuda()
if os.path.exists("./Test_Result.txt"):
os.remove("./Test_Result.txt")
print("Training Start......")
if args.BiLSTM is True:
train_conll2003.train(train_iter=train_iter,
dev_iter=dev_iter,
test_iter=test_iter,
model=model,
args=args)
elif args.BiLSTM_CRF is True:
train_conll2003_CRF.train(train_iter=train_iter,
dev_iter=dev_iter,
test_iter=test_iter,
model=model,
args=args)
def test(args):
"""
This function tests our models
:param args: the command line arguments with the desired actions
"""
_, _, test_data_all = load(args.data_dir,
cachedir=args.cachedir,
override_cache=args.override_cache,
text_only=(args.model.lower()
in ["bi-lstm", "bert"]),
include_tfidf=args.include_tfidf,
balanced=args.balanced)
test_data, test_labels = test_data_all.X, test_data_all.y
apx = get_appendix(args.include_tfidf, args.balanced)
if args.model.lower() == "simple-ff":
preds = test_pytorch(
test_data,
test_labels,
load_model_path=f"models/simple-ff{apx}.torch",
predictions_file=f"preds/simple-ff-preds{apx}.txt")
elif args.model.lower() == "bi-lstm":
model = BiLSTM(load_model_path="models/bilstm.keras",
tokenizer_path='models/bilstm-tokenizer.json')
preds = model.test(test_data, y_test=test_labels)
elif args.model.lower() == "logreg":
model = LogisticRegression(load_model_path=f"models/logreg{apx}.pkl")
preds = model.test(
test_data,
test_labels,
save_predictions_path=f"preds/logreg-preds{apx}.txt")
elif args.model.lower() == "majority-vote":
model = MajorityVote(load_model_path="models/majority-class.txt")
preds = model.test(test_labels)
elif args.model.lower() == "bert":
model = Bert(load_model_path="models/bert.pkl")
preds = model.test(test_data,
test_labels,
save_predictions_path="preds/bert-preds.txt")
elif args.model.lower() == "svm":
model = SVM(load_model_path=f"models/svm{apx}.sav")
preds = model.test(test_data,
save_predictions_path=f"preds/svm-preds{apx}.txt")
else:
raise Exception("Unknown model type passed in!")
metrics = classification_report(test_labels, preds, output_dict=True)
pprint(metrics)
with open(f"scores/{args.model.lower()}{apx}.json", "w") as fout:
json.dump(metrics, fout, indent=4)
def run(*args, **kwargs):
parser = argparse.ArgumentParser()
parser.add_argument('-train_file', type=str, default='./data/train.csv')
parser.add_argument('-dev_file', type=str, default='./data/dev.csv')
parser.add_argument('-test_file', type=str, default='./data/test.csv')
parser.add_argument('-save_path', type=str, default='./model.pkl')
parser.add_argument('-model',
type=str,
default=kwargs['model'],
help="[cnn, bilstm]")
parser.add_argument('-batch_size', type=int, default=kwargs['batch_size'])
parser.add_argument('-embedding_size', type=int, default=128)
parser.add_argument('-hidden_size', type=int, default=128)
parser.add_argument('-learning_rate', type=float, default=1e-3)
parser.add_argument('-dropout', type=float, default=0.5)
parser.add_argument('-epochs', type=int, default=20)
parser.add_argument('-seed', type=int, default=1)
args = parser.parse_args()
print(args)
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)
print("Loading Data...")
datasets, vocab = utils.build_dataset(args.train_file, args.dev_file,
args.test_file)
train_loader, dev_loader, test_loader = (DataLoader(
dataset=dataset,
batch_size=args.batch_size,
collate_fn=utils.collate_fn,
shuffle=dataset.train) for i, dataset in enumerate(datasets))
print("Building Model...")
if args.model == "cnn":
model = CNN.Model(vocab_size=len(vocab),
embedding_size=args.embedding_size,
hidden_size=args.hidden_size,
filter_sizes=[3, 4, 5],
dropout=args.dropout)
elif args.model == "bilstm":
model = BiLSTM.Model(vocab_size=len(vocab),
embedding_size=args.embedding_size,
hidden_size=args.hidden_size,
dropout=args.dropout)
if torch.cuda.is_available():
model = model.cuda()
trainer = Trainer(model, args.learning_rate)
train_loss_list = list()
dev_loss_list = list()
best_acc = 0
for i in range(args.epochs):
print("Epoch: {} ################################".format(i))
train_loss, train_acc = trainer.train(train_loader)
dev_loss, dev_acc = trainer.evaluate(dev_loader)
train_loss_list.append(train_loss)
dev_loss_list.append(dev_loss)
print("Train Loss: {:.4f} Acc: {:.4f}".format(train_loss, train_acc))
print("Dev Loss: {:.4f} Acc: {:.4f}".format(dev_loss, dev_acc))
if dev_acc > best_acc:
best_acc = dev_acc
trainer.save(args.save_path)
print("###########################################")
trainer.load(args.save_path)
test_loss, test_acc = trainer.evaluate(test_loader)
print("Test Loss: {:.4f} Acc: {:.4f}".format(test_loss, test_acc))
return train_loss_list, dev_loss_list
model_path = "models/" + args.model_path
random.seed(args.seed)
np.random.seed(args.seed)
import dynet_config
dynet_config.set(random_seed=args.seed)
from import_dy import dy
dyparams = dy.DynetParams()
dyparams.set_autobatch(True)
model = dy.ParameterCollection()
from models import BiLSTM
from train_funcs import *
network = BiLSTM(1, 300, 150, W2I, model, vecs)
if args.train is not None:
train = aspects_sent_convert(read_file_lines(args.train))
if not os.path.exists(model_path):
os.makedirs(model_path)
os.makedirs(model_path + "/Results")
command_line_file = open(model_path + "/Command_Lines.txt", 'w')
for arg in sys.argv:
command_line_file.write(arg + "\n")
command_line_file.close()
else:
print "Model Folder Already exists."
sys.exit(1)
network.save_meta(model_path + "/Model.meta")
trainer = dy.AdamTrainer(model)
PAD, UNK)
test_tag_id_lists, test_tag_ids = pad(test_tag_lists, test_lengths, max_len,
tag2id, PAD, UNK)
train_vocab_size = len(train_word_id_lists)
dev_vocab_size = len(dev_word_id_lists)
test_vocab_size = len(test_word_id_lists)
embedding_dim = args.embedding
hidden_dim = args.hidden
num_layers = args.num_layers
dropout = args.dropout
# tag
out_dim = len(tag2id) + 2
model = BiLSTM(train_vocab_size, embedding_dim, num_layers, hidden_dim,
dropout, out_dim)
optimizer = optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
batch_size = args.batch_size
if args.cuda:
model.cuda()
train_word_id_lists = train_word_id_lists.cuda()
train_tag_id_lists = train_tag_id_lists.cuda()
dev_word_id_lists = dev_word_id_lists.cuda()
dev_tag_id_lists = dev_tag_id_lists.cuda()
test_word_id_lists = test_word_id_lists.cuda()
test_tag_id_lists = test_tag_id_lists.cuda()