# with open(os.path.join(TEMP_DIRECTORY, "lm_train.txt"), 'w') as f:
# for item in lm_train:
# f.write("%s\n" % item)
#
# with open(os.path.join(TEMP_DIRECTORY, "lm_test.txt"), 'w') as f:
# for item in lm_test:
# f.write("%s\n" % item)
#
# model = LanguageModelingModel("auto", MODEL_NAME, args=language_modeling_args, use_cuda=torch.cuda.is_available())
# model.train_model(os.path.join(TEMP_DIRECTORY, "lm_train.txt"), eval_file=os.path.join(TEMP_DIRECTORY, "lm_test.txt"))
# MODEL_NAME = language_modeling_args["best_model_dir"]
# Train the model
print("Started Training")
train['labels'] = encode(train["labels"])
dev['labels'] = encode(dev["labels"])
dev_sentences = dev['text'].tolist()
dev_preds = np.zeros((len(dev), args["n_fold"]))
if args["evaluate_during_training"]:
for i in range(args["n_fold"]):
if os.path.exists(args['output_dir']) and os.path.isdir(
args['output_dir']):
shutil.rmtree(args['output_dir'])
print("Started Fold {}".format(i))
model = ClassificationModel(
MODEL_TYPE,
MODEL_NAME,
args=args,
with open(os.path.join(TEMP_DIRECTORY, "lm_test.txt"), 'w') as f:
for item in lm_test:
f.write("%s\n" % item)
model = LanguageModelingModel(MODEL_TYPE,
MODEL_NAME,
args=language_modeling_args)
model.train_model(os.path.join(TEMP_DIRECTORY, "lm_train.txt"),
eval_file=os.path.join(TEMP_DIRECTORY, "lm_test.txt"))
MODEL_NAME = language_modeling_args["best_model_dir"]
# Train the model
print("Started Training")
train['labels'] = encode(train["labels"])
test['labels'] = encode(test["labels"])
test_sentences = test['text'].tolist()
test_preds = np.zeros((len(test), args["n_fold"]))
if args["evaluate_during_training"]:
for i in range(args["n_fold"]):
if os.path.exists(args['output_dir']) and os.path.isdir(
args['output_dir']):
shutil.rmtree(args['output_dir'])
print("Started Fold {}".format(i))
model = ClassificationModel(
MODEL_TYPE,
MODEL_NAME,
args=args,
