def index_all():
index_file = os.path.join(GPFS_STORAGE, localnode.name + "_index")
writer = open(index_file, 'w')
for i in localnode.indices():
print("Indexing file: gram2_%s.processed" % str(i))
index_processed_file(i, writer)
writer.close()
print("done")
def process_all():
for index in localnode.indices():
filename = "gram2_" + str(index)
zip_file = os.path.join(LOCAL_STORAGE, filename + ".csv.zip")
out_file = os.path.join(GPFS_STORAGE, filename + ".processed")
if os.path.isfile(zip_file) and not os.path.isfile(out_file):
process_zip(zip_file, out_file)
elif not os.path.isfile(zip_file):
print("Need to download the zip file first.")
else:
print("Already processed.")
def download_all(force=False):
for i in localnode.indices():
download_2gram_by_index(i, force=force)