def load_dataset(data_path):
train_path = os.path.join(data_path, "train.npz")
test_path = os.path.join(data_path, "test.npz")
dictionary_path = os.path.join(data_path, "word_index.pkl")
# Load the data
toxic = ToxicData(train_path, test_path, dictionary_path)
train_ids, train_dataset = toxic.load_train(mode="sup")
return train_dataset
# And create the generators
testgen = DatasetGenerator(test_data,
batch_size=args.test_batch_size,
shuffle=False)
# Initialize the model
model = load_model(args.model)
model.load_state(MODEL_FILE)
# Get the predictions
predictions = model.predict_generator(testgen,
testgen.steps_per_epoch,
verbose=1)
ToxicData.save_submission(SUBMISSION_FILE, ids, predictions)
if __name__ == "__main__":
# Create the paths for the data
train_path = os.path.join(args.data, "train.npz")
test_path = os.path.join(args.data, "test.npz")
dictionary_path = os.path.join(args.data, "word_index.pkl")
# Load the data
toxic = ToxicData(train_path, test_path, dictionary_path)
if args.train:
train(toxic)
if args.test:
test(toxic)
ToxicData.save_submission(submission_file, ids, predictions)
if __name__ == "__main__":
# Create the paths for the data
train_path = os.path.join(args.data, "train.npz")
test_path = os.path.join(args.data, "test.npz")
dictionary_path = os.path.join(args.data, "word_index.pkl")
if args.use_augmented:
augmented_path = os.path.join(args.data, "train_*.npz")
else:
augmented_path = ""
# Load the data
toxic = ToxicData(train_path,
test_path,
dictionary_path,
augmented_path=augmented_path,
original_prob=args.original_prob,
fixed_len=args.fixed_len)
model = None
if args.train:
if args.kfold:
model = kfold(toxic)
else:
model = train(toxic)
if args.test:
test(toxic, model=model)