def main(_): options = get_options_from_flags() data_dir = options.data_dir download_dir = options.download_dir # 디렉토리 생성 # data_dir = Pretrained 된 Word-Embedding Vector를 다운로드 받을 디렉토리 # download_dir = SQuAD DataSet를 다운로드 받을 디렉토리 for d in [data_dir, download_dir]: # 디렉토리를 재귀적으로 생성하며 디렉토리가 이미 존재하면 예외를 발생시키지 않음. os.makedirs(d, exist_ok=True) # 데이터 다운로드 단계 # 첫째, GloVe vectors를 https://nlp.stanford.edu/projects/glove/ 로 부터 다운로드 받음. # 둘째, SQuAD Dataset 다운로드 받음 if options.word_embedding_model_type == "fasttext": download_fasttext_data(download_dir) else: download_data(download_dir) # if options.word_embedding_model_type == "fasttext": split_vocab_and_embedding_fasttext(options, data_dir, download_dir) else: split_vocab_and_embedding(options, data_dir, download_dir) if options.exobrain_korean_dataset: DataParser(data_dir, download_dir).create_train_data_exobrain_korean(options) else: DataParser(data_dir, download_dir).create_train_data(options) if options.use_cove_vectors: save_cove_weights(options) maybe_upload_data_files_to_s3(options)
def main(_): options = get_options_from_flags() options.num_gpus = 0 with tf.Session() as sess: squad_data = SquadData(options) squad_data.setup_with_tf_session(sess) print("TRAIN") _print_ds(squad_data.vocab, squad_data.train_ds) print("DEV") _print_ds(squad_data.vocab, squad_data.dev_ds)
def main(_): tf.set_random_seed(0) np.random.seed(0) options = get_options_from_flags() inputs = np.random.randint(low=0, high=30, size=(1, 1), dtype=np.int64) embeddings = embedding_util \ .load_word_embeddings_including_unk_and_padding(options) torch_output = compute_torch_values(inputs, embeddings) tf_cudnn_output = compute_tf_cudnn_values(inputs, embeddings, options) tf_output = compute_tf_values(inputs, embeddings, options) assert_tensors_equal(torch_output, tf_output, "Torch output", "TF output") assert_tensors_equal(tf_cudnn_output, tf_output, "cuDNN TF output", "TF output")
def main(_): options = get_options_from_flags() update_remote_options(options) maybe_download_data_files_from_s3(options) Evaluator(options).evaluate()
def main(_): options = get_options_from_flags() maybe_download_data_files_from_s3(options) Evaluator(options).predict()
def main(_): options = get_options_from_flags() maybe_download_data_files_from_s3(options) Trainer(options).train()