build_data.untar(dpath, fname)
#########################
# Distractor personas
#########################
fname = 'train_sorted_50_personas.json'
gd_id = '1SGFdJqyNYeepKFqwMLv4Ym717QQTtpi8'
build_data.download_from_google_drive(gd_id,
os.path.join(dpath, fname))
fname = 'valid_sorted_50_personas.json'
gd_id = '1A7oVKmjJ1EZTh6-3Gio4XQo81QgnTGGi'
build_data.download_from_google_drive(gd_id,
os.path.join(dpath, fname))
fname = 'dnli_sorted_50_personas.json'
gd_id = '1wlIkVcBZoGQd3rbI7XWNhuq4rvw9FyoP'
build_data.download_from_google_drive(gd_id,
os.path.join(dpath, fname))
print("Data has been placed in " + dpath)
build_data.mark_done(dpath, version)
def make_path(opt, fname):
return os.path.join(opt['datapath'], FOLDER_NAME, fname)
if __name__ == '__main__':
opt = params.ParlaiParser().parse_args(print_args=False)
build(opt)
build_data.untar(dpath, fname_data)
# next download the wordstats files
fname_wordstats = 'wordstats_v1.tar.gz'
build_data.download(URL_ROOT + fname_wordstats, dpath, fname_wordstats)
build_data.untar(dpath, fname_wordstats)
# next download the evaluation logs
fname_evallogs = 'evaluationlogs_v1.tar.gz'
build_data.download(URL_ROOT + fname_evallogs, dpath, fname_evallogs)
build_data.untar(dpath, fname_evallogs)
# and the reproducible logs.
# for more info see https://github.com/facebookresearch/ParlAI/issues/2855
fname_evallogs = 'evaluation_logs_reproducible_v1.tar.gz'
build_data.download(URL_ROOT + fname_evallogs, dpath, fname_evallogs)
build_data.untar(dpath, fname_evallogs)
print("Data has been placed in " + dpath)
build_data.mark_done(dpath, version)
def make_path(opt, fname):
return os.path.join(opt['datapath'], FOLDER_NAME, fname)
if __name__ == '__main__':
opt = params.ParlaiParser().parse_args()
build(opt)
