print("uri_map", uri_map)
# wait while the file list gets downloaded
ready = False
while not ready:
ready = True
for async_result in async_results:
if not async_result.ready():
ready = False
break
if not ready:
time.sleep(1)
engine_files = dview.apply(s3downloader.getFiles)
s3downloader.addFiles("s3://hdfdata/ncep3/")
output = s3downloader.dump()
s3_uris = s3downloader.getFiles(state="PENDING")
print(s3_uris)
counts = dview.apply(s3downloader.start)
while any(v > 0 for v in counts):
time.sleep(1)
counts = dview.apply(s3downloader.update)
print("counts:", counts)
print("done!")
ncep3_files = ["s3://hdfdata/ncep3/GSSTF_NCEP.3.1987.07.01.he5",
"s3://hdfdata/ncep3/GSSTF_NCEP.3.1987.07.02.he5",
"s3://hdfdata/ncep3/GSSTF_NCEP.3.1987.07.03.he5",
"s3://hdfdata/ncep3/GSSTF_NCEP.3.1987.07.04.he5"]
s3downloader.init()
nspace = s3downloader.freespace()
print("free space:", nspace)
usedspace = s3downloader.usedspace()
print("used space:", usedspace)
print("preloaded files:")
downloads = s3downloader.getFiles(s3uri_prefix="s3://hdfdata/")
for download in downloads:
print(download)
print("clearing..")
s3downloader.clear()
usedspace = s3downloader.usedspace()
#print("used space", usedspace)
#s3download.addFiles(ncep3_files)
#s3download.addFiles("s3://hdfdata/ncep3/")
s3downloader.addFiles("s3://hdfdata/ncep3/")
output = s3downloader.dump()