Exemple #1
0
def index_all():
    index_file = os.path.join(GPFS_STORAGE, localnode.name + "_index")
    writer = open(index_file, 'w')
    for i in localnode.indices():
        print("Indexing file: gram2_%s.processed" % str(i))
        index_processed_file(i, writer)
    writer.close()
    print("done")
Exemple #2
0
def process_all():
    for index in localnode.indices():
        filename = "gram2_" + str(index)
        zip_file = os.path.join(LOCAL_STORAGE, filename + ".csv.zip")
        out_file = os.path.join(GPFS_STORAGE, filename + ".processed")
        if os.path.isfile(zip_file) and not os.path.isfile(out_file):
            process_zip(zip_file, out_file)
        elif not os.path.isfile(zip_file):
            print("Need to download the zip file first.")
        else:
            print("Already processed.")
Exemple #3
0
def download_all(force=False):
    for i in localnode.indices():
        download_2gram_by_index(i, force=force)