def main(_):
options = get_options_from_flags()
data_dir = options.data_dir
download_dir = options.download_dir
# 디렉토리 생성
# data_dir = Pretrained 된 Word-Embedding Vector를 다운로드 받을 디렉토리
# download_dir = SQuAD DataSet를 다운로드 받을 디렉토리
for d in [data_dir, download_dir]:
# 디렉토리를 재귀적으로 생성하며 디렉토리가 이미 존재하면 예외를 발생시키지 않음.
os.makedirs(d, exist_ok=True)
# 데이터 다운로드 단계
# 첫째, GloVe vectors를 https://nlp.stanford.edu/projects/glove/ 로 부터 다운로드 받음.
# 둘째, SQuAD Dataset 다운로드 받음
if options.word_embedding_model_type == "fasttext":
download_fasttext_data(download_dir)
else:
download_data(download_dir)
#
if options.word_embedding_model_type == "fasttext":
split_vocab_and_embedding_fasttext(options, data_dir, download_dir)
else:
split_vocab_and_embedding(options, data_dir, download_dir)
if options.exobrain_korean_dataset:
DataParser(data_dir,
download_dir).create_train_data_exobrain_korean(options)
else:
DataParser(data_dir, download_dir).create_train_data(options)
if options.use_cove_vectors:
save_cove_weights(options)
maybe_upload_data_files_to_s3(options)
def main(_):
options = get_options_from_flags()
options.num_gpus = 0
with tf.Session() as sess:
squad_data = SquadData(options)
squad_data.setup_with_tf_session(sess)
print("TRAIN")
_print_ds(squad_data.vocab, squad_data.train_ds)
print("DEV")
_print_ds(squad_data.vocab, squad_data.dev_ds)
def main(_):
tf.set_random_seed(0)
np.random.seed(0)
options = get_options_from_flags()
inputs = np.random.randint(low=0, high=30, size=(1, 1), dtype=np.int64)
embeddings = embedding_util \
.load_word_embeddings_including_unk_and_padding(options)
torch_output = compute_torch_values(inputs, embeddings)
tf_cudnn_output = compute_tf_cudnn_values(inputs, embeddings, options)
tf_output = compute_tf_values(inputs, embeddings, options)
assert_tensors_equal(torch_output, tf_output, "Torch output", "TF output")
assert_tensors_equal(tf_cudnn_output, tf_output, "cuDNN TF output",
"TF output")
def main(_):
options = get_options_from_flags()
update_remote_options(options)
maybe_download_data_files_from_s3(options)
Evaluator(options).evaluate()
def main(_):
options = get_options_from_flags()
maybe_download_data_files_from_s3(options)
Evaluator(options).predict()
def main(_):
options = get_options_from_flags()
maybe_download_data_files_from_s3(options)
Trainer(options).train()