def main(): parser = argparse.ArgumentParser("Preprocess SQuAD data") #basedir = join(expanduser("~"), "data", "squad") basedir = join(expanduser("~"), "azayats", "data", "squad") parser.add_argument("--train_file", default=join(basedir, "train-v1.1.json")) parser.add_argument("--dev_file", default=join(basedir, "dev-v1.1.json")) if not exists(config.CORPUS_DIR): mkdir(config.CORPUS_DIR) target_dir = join(config.CORPUS_DIR, SquadCorpus.NAME) if exists(target_dir) and len(listdir(target_dir)) > 0: raise ValueError("Files already exist in " + target_dir) args = parser.parse_args() tokenzier = NltkAndPunctTokenizer() print("Parsing train...") train = list(parse_squad_data(args.train_file, "train", tokenzier)) print("Parsing dev...") dev = list(parse_squad_data(args.dev_file, "dev", tokenzier)) print("Saving...") SquadCorpus.make_corpus(train, dev) print("Done")
def main(): #Namespace(directory= 'C:/Users/boidiyv/document-qa-master',dump=False, fake=False, verbose=False) parser = argparse.ArgumentParser("Preprocess SQuAD data") #parser = argparse.ArgumentParser() parser.add_argument('--document-qa/docqa/squad', type=Path) parser.add_argument("--train_file", default=config.SQUAD_TRAIN) parser.add_argument("--dev_file", default=config.SQUAD_DEV) #parser.add_argument("--document-qa-master",type=lambda p: Path(p).absolute(),default=Path(__file__).absolute().parent / "document-qa-master",help="Path to the data directory" ) if not exists(config.CORPUS_DIR): mkdir(config.CORPUS_DIR) target_dir = join(config.CORPUS_DIR, SquadCorpus.NAME) if exists(target_dir) and len(listdir(target_dir)) > 0: raise ValueError("Files already exist in " + target_dir) args = parser.parse_args('') tokenzier = NltkAndPunctTokenizer() print("Parsing train...") train = list(parse_squad_data(args.train_file, "train", tokenzier)) print(train) print("Parsing dev...") dev = list(parse_squad_data(args.dev_file, "dev", tokenzier)) print("Saving...") SquadCorpus.make_corpus(train, dev) print("Done")
def main(): parser = argparse.ArgumentParser("Preprocess SQuAD data") parser.add_argument("--train_file", default=config.SQUAD_TRAIN) parser.add_argument("--dev_file", default=config.SQUAD_DEV) parser.add_argument("--weighted-questions", action='store_true') if not exists(config.CORPUS_DIR): mkdir(config.CORPUS_DIR) target_dir = join(config.CORPUS_DIR, SquadCorpus.NAME) if exists(target_dir) and len(listdir(target_dir)) > 0: raise ValueError("Files already exist in " + target_dir) args = parser.parse_known_args()[0] tokenizer = NltkAndPunctTokenizer() print("Parsing train...") train = list( parse_squad_data(args.train_file, "train", tokenizer, weighted_samples=args.weighted_questions)) print("Parsing dev...") dev = list(parse_squad_data(args.dev_file, "dev", tokenizer)) print("Saving...") SquadCorpus.make_corpus(train, dev) print("Done")