root_data_path = "E:/Deep Learning/deep-text-corrector-master/deep-text-corrector-master/"
train_path = os.path.join(root_data_path, "movie_dialog_train.txt")
val_path = os.path.join(root_data_path, "movie_dialog_val.txt")
test_path = os.path.join(root_data_path, "movie_dialog_test.txt")
model_path = os.path.join(root_data_path, "movie_dialog_model")
config = DefaultMovieDialogConfig()
#data_reader = MovieDialogReader(config, train_path)
#train(data_reader, train_path, val_path, model_path)
data_reader = MovieDialogReader(config,
train_path,
dropout_prob=0.25,
replacement_prob=0.25,
dataset_copies=1)
corrective_tokens = get_corrective_tokens(data_reader, train_path)
import pickle
with open(os.path.join(root_data_path, "corrective_tokens.pickle"), "wb") as f:
pickle.dump(corrective_tokens, f)
import pickle
with open(os.path.join(root_data_path, "token_to_id.pickle"), "wb") as f:
pickle.dump(data_reader.token_to_id, f)
sess = tf.InteractiveSession()
model = create_model(sess, True, model_path, config=config)
decoded = decode_sentence(sess,
model,
data_reader,
json.loads(lines[0]),
corrective_tokens=corrective_tokens)
corrective_tokens_filename = "corrective_tokens.pickle"
corrective_tokens_path = os.path.join(ROOT_DATA_PATH,
corrective_tokens_filename)
download(s3_client,
corrective_tokens_filename,
local_path=corrective_tokens_path)
token_to_id_filename = "token_to_id.pickle"
token_to_id_path = os.path.join(ROOT_DATA_PATH, token_to_id_filename)
download(s3_client, token_to_id_filename, local_path=token_to_id_path)
# Load model.
config = DefaultMovieDialogConfig()
sess = tf.Session()
print("Loading model")
model = create_model(sess, True, MODEL_PATH, config=config)
print("Loaded model")
with open(corrective_tokens_path) as f:
corrective_tokens = pickle.load(f)
with open(token_to_id_path) as f:
token_to_id = pickle.load(f)
data_reader = MovieDialogReader(config, token_to_id=token_to_id)
print("Done initializing.")
def process_event(event, context):
print("Received event: " + json.dumps(event, indent=2))
outputs = decode_sentence(sess,
model,