def build_web_corpus(n_processes, sets_to_build, source_dir, target_dir): sets_to_build_dict = {} if 'verified' in sets_to_build: sets_to_build_dict['verified'] = join(source_dir, "verified-web-dev.json") if 'dev' in sets_to_build: sets_to_build_dict['dev'] = join(source_dir, "web-dev.json") if 'train' in sets_to_build: sets_to_build_dict['train'] = join(source_dir, "web-train.json") if 'test' in sets_to_build: sets_to_build_dict['test'] = join(source_dir, "web-test-without-answers.json") #dict( # verified=join(TRIVIA_QA, "qa", "verified-web-dev.json"), # dev=join(TRIVIA_QA, "qa", "web-dev.json"), # train=join(TRIVIA_QA, "qa", "web-train.json"), # test=join(TRIVIA_QA, "qa", "web-test-without-answers.json") #) build_dataset("web", NltkAndPunctTokenizer(), sets_to_build_dict, FastNormalizedAnswerDetector(), n_processes, out_dir=target_dir)
def build_web_corpus(n_processes): build_dataset( "web", NltkAndPunctTokenizer(), dict(verified=join(TRIVIA_QA, "qa", "verified-web-dev.json"), dev=join(TRIVIA_QA, "qa", "web-dev.json"), train=join(TRIVIA_QA, "qa", "web-train.json"), test=join(TRIVIA_QA, "qa", "web-test-without-answers.json")), FastNormalizedAnswerDetector(), n_processes)
def build_wiki_corpus(n_processes): build_dataset( "wiki", NltkAndPunctTokenizer(), dict( verified=join(TRIVIA_QA, "qa", "verified-wikipedia-dev.json"), dev=join(TRIVIA_QA, "qa", "wikipedia-dev.json"), train=join(TRIVIA_QA, "qa", "wikipedia-train.json"), ), FastNormalizedAnswerDetector(), n_processes)
def build_sample_corpus(n_processes): build_dataset("web-sample", NltkAndPunctTokenizer(), dict( dev=join(TRIVIA_QA, "qa", "web-dev.json"), train=join(TRIVIA_QA, "qa", "web-train.json"), ), FastNormalizedAnswerDetector(), n_processes, sample=1000)
def build_unfiltered_corpus(n_processes): build_dataset("web-open", NltkAndPunctTokenizer(), dict(dev=join(TRIVIA_QA_UNFILTERED, "unfiltered-web-dev.json"), train=join(TRIVIA_QA_UNFILTERED, "unfiltered-web-train.json"), test=join(TRIVIA_QA_UNFILTERED, "unfiltered-web-test-without-answers.json")), answer_detector=FastNormalizedAnswerDetector(), n_process=n_processes)
def build_unfiltered_corpus(n_processes, sets_to_build, source_dir, target_dir): sets_to_build_dict = {} if 'dev' in sets_to_build: sets_to_build_dict['dev'] = join(source_dir, "unfiltered-web-dev.json") if 'train' in sets_to_build: sets_to_build_dict['train'] = join(source_dir, "unfiltered-web-train.json") if 'test' in sets_to_build: sets_to_build_dict['test'] = join( source_dir, "unfiltered-web-test-without-answers.json") build_dataset("web-open", NltkAndPunctTokenizer(), sets_to_build_dict, answer_detector=FastNormalizedAnswerDetector(), n_process=n_processes, out_dir=target_dir)