type=int,
default=1,
help="number of batch_size")
parser.add_argument(
"--multi_segment",
type=bool,
default=False,
help="whether to use multiple segment_labels for entity types")
parser.add_argument("--sep_label",
type=bool,
default=False,
help="whether to insert <sep>")
args = parser.parse_args()
print("Loading Vocab", args.vocab_path)
vocab = WordVocab.load_vocab(args.vocab_path)
print("Vocab Size: ", len(vocab))
print("Loading Train Dataset", args.train_dataset)
train_dataset = BERTDataset(args.train_dataset,
vocab,
seq_len=args.seq_len,
corpus_lines=args.corpus_lines,
on_memory=args.on_memory,
multi_segment=args.multi_segment,
sep=args.sep_label)
print("Creating Dataloader")
train_data_loader = DataLoader(train_dataset,
batch_size=args.batch_size,
num_workers=args.num_workers)
epochs = 10
num_workers = 1
with_cuda = False
log_freq = 20
corpus_lines = None
cuda_devices = None
on_memory = True
lr = 1e-4
adam_weight_decay = 0.01
adam_beta1 = 0.9
adam_beta2 = 0.999
print("Loading Vocab", vocab_path)
vocab = WordVocab.load_vocab(vocab_path)
print("Vocab Size: ", len(vocab))
print("Loading Train Dataset", train_dataset)
train_dataset = BERTDataset(train_dataset,
vocab,
seq_len=seq_len,
corpus_lines=corpus_lines,
on_memory=on_memory)
print("Loading Test Dataset", test_dataset)
test_dataset = BERTDataset(test_dataset, vocab, seq_len=seq_len, on_memory=on_memory) \
if test_dataset is not None else None
print("Creating Dataloader")
train_data_loader = DataLoader(train_dataset,
def train():
parser = argparse.ArgumentParser()
parser.add_argument("-c",
"--train_dataset",
required=True,
type=str,
help="train dataset for train bert")
parser.add_argument("-t",
"--test_dataset",
type=str,
default=None,
help="test set for evaluate train set")
parser.add_argument("-v",
"--vocab_path",
required=True,
type=str,
help="built vocab model path with bert-vocab")
parser.add_argument("-o",
"--output_path",
required=True,
type=str,
help="ex)output/bert.model")
parser.add_argument("-bt",
"--bert_path",
type=str,
help='path of pretrained bert')
parser.add_argument("-hs",
"--hidden",
type=int,
default=256,
help="hidden size of transformer model")
parser.add_argument("-l",
"--layers",
type=int,
default=8,
help="number of layers")
parser.add_argument("-a",
"--attn_heads",
type=int,
default=8,
help="number of attention heads")
parser.add_argument("-s",
"--seq_len",
type=int,
default=100,
help="maximum sequence len")
parser.add_argument("-b",
"--batch_size",
type=int,
default=64,
help="number of batch_size")
parser.add_argument("-e",
"--epochs",
type=int,
default=10,
help="number of epochs")
parser.add_argument("-w",
"--num_workers",
type=int,
default=4,
help="dataloader worker size")
parser.add_argument("--with_cuda",
type=bool,
default=True,
help="training with CUDA: true, or false")
parser.add_argument("--log_freq",
type=int,
default=10,
help="printing loss every n iter: setting n")
parser.add_argument("--cuda_devices",
type=int,
nargs='+',
default=None,
help="CUDA device ids")
parser.add_argument("--lr",
type=float,
default=1e-3,
help="learning rate of adam")
parser.add_argument("--adam_weight_decay",
type=float,
default=0.01,
help="weight_decay of adam")
parser.add_argument("--adam_beta1",
type=float,
default=0.9,
help="adam first beta value")
parser.add_argument("--adam_beta2",
type=float,
default=0.999,
help="adam first beta value")
d = vars(parser.parse_args())
args = Option(d)
print("Loading Vocab", args.vocab_path)
vocab = WordVocab.load_vocab(args.vocab_path)
print("Vocab Size: ", len(vocab))
print("Loading Train Dataset", args.train_dataset)
train_dataset = Dataset(args.train_dataset, vocab, seq_len=args.seq_len)
print("Loading Test Dataset", args.test_dataset)
test_dataset = Dataset(args.test_dataset, vocab, seq_len=args.seq_len) \
if args.test_dataset is not None else None
print("Creating Dataloader")
train_data_loader = DataLoader(train_dataset,
batch_size=args.batch_size,
num_workers=args.num_workers)
test_data_loader = DataLoader(test_dataset, batch_size=args.batch_size, num_workers=args.num_workers) \
if test_dataset is not None else None
print("Building model")
bert = torch.load(args.bert_path)
model = Readout(bert.hidden, args.hidden)
#model = bert_fc(args.bert_path, args.hidden)
print("Creating Trainer")
trainer = Trainer(args,
bert,
model,
train_dataloader=train_data_loader,
test_dataloader=test_data_loader)
print("Training Start")
for epoch in range(args.epochs):
trainer.train(epoch)
trainer.save(epoch, args.output_path)
if test_data_loader is not None:
trainer.test(epoch)