コード例 #1
0
print("uri_map", uri_map)


# wait while the file list gets downloaded
ready = False
while not ready:
    ready = True
    for async_result in async_results:
        if not async_result.ready():
            ready = False
            break
    if not ready:
        time.sleep(1)

engine_files = dview.apply(s3downloader.getFiles)

s3downloader.addFiles("s3://hdfdata/ncep3/")
output = s3downloader.dump()

s3_uris = s3downloader.getFiles(state="PENDING")
print(s3_uris)

counts = dview.apply(s3downloader.start)
while any(v > 0 for v in counts):
    time.sleep(1)
    counts = dview.apply(s3downloader.update)
    print("counts:", counts)

print("done!")
コード例 #2
0
ncep3_files = ["s3://hdfdata/ncep3/GSSTF_NCEP.3.1987.07.01.he5",
"s3://hdfdata/ncep3/GSSTF_NCEP.3.1987.07.02.he5",
"s3://hdfdata/ncep3/GSSTF_NCEP.3.1987.07.03.he5",
"s3://hdfdata/ncep3/GSSTF_NCEP.3.1987.07.04.he5"]

s3downloader.init()


nspace = s3downloader.freespace()
print("free space:", nspace)
usedspace = s3downloader.usedspace()
print("used space:", usedspace)

print("preloaded files:")
downloads = s3downloader.getFiles(s3uri_prefix="s3://hdfdata/")
for download in downloads:
    print(download)

print("clearing..")
s3downloader.clear()

usedspace = s3downloader.usedspace()
#print("used space", usedspace)

#s3download.addFiles(ncep3_files)
#s3download.addFiles("s3://hdfdata/ncep3/")

s3downloader.addFiles("s3://hdfdata/ncep3/")
output = s3downloader.dump()