# with open(os.path.join(TEMP_DIRECTORY, "lm_train.txt"), 'w') as f: # for item in lm_train: # f.write("%s\n" % item) # # with open(os.path.join(TEMP_DIRECTORY, "lm_test.txt"), 'w') as f: # for item in lm_test: # f.write("%s\n" % item) # # model = LanguageModelingModel("auto", MODEL_NAME, args=language_modeling_args, use_cuda=torch.cuda.is_available()) # model.train_model(os.path.join(TEMP_DIRECTORY, "lm_train.txt"), eval_file=os.path.join(TEMP_DIRECTORY, "lm_test.txt")) # MODEL_NAME = language_modeling_args["best_model_dir"] # Train the model print("Started Training") train['labels'] = encode(train["labels"]) dev['labels'] = encode(dev["labels"]) dev_sentences = dev['text'].tolist() dev_preds = np.zeros((len(dev), args["n_fold"])) if args["evaluate_during_training"]: for i in range(args["n_fold"]): if os.path.exists(args['output_dir']) and os.path.isdir( args['output_dir']): shutil.rmtree(args['output_dir']) print("Started Fold {}".format(i)) model = ClassificationModel( MODEL_TYPE, MODEL_NAME, args=args,
with open(os.path.join(TEMP_DIRECTORY, "lm_test.txt"), 'w') as f: for item in lm_test: f.write("%s\n" % item) model = LanguageModelingModel(MODEL_TYPE, MODEL_NAME, args=language_modeling_args) model.train_model(os.path.join(TEMP_DIRECTORY, "lm_train.txt"), eval_file=os.path.join(TEMP_DIRECTORY, "lm_test.txt")) MODEL_NAME = language_modeling_args["best_model_dir"] # Train the model print("Started Training") train['labels'] = encode(train["labels"]) test['labels'] = encode(test["labels"]) test_sentences = test['text'].tolist() test_preds = np.zeros((len(test), args["n_fold"])) if args["evaluate_during_training"]: for i in range(args["n_fold"]): if os.path.exists(args['output_dir']) and os.path.isdir( args['output_dir']): shutil.rmtree(args['output_dir']) print("Started Fold {}".format(i)) model = ClassificationModel( MODEL_TYPE, MODEL_NAME, args=args,