# Pywren "Hello World" to see if Pywren is working
import pywren_ibm_cloud as pywren
def hello(name):
return 'Hello {}!'.format(name)
pw = pywren.local_executor()
#pw = pywren.cloudrun_executor()
#pw = pywren.cloudrun_executor(log_level="DEBUG")
pw.call_async(hello, 'World')
print(pw.get_result())
def test_rechunker_local():
source_url = "data/source.zarr"
target_url = "data/target.zarr"
temp_url = "data/temp.zarr"
# delete
for url in (source_url, target_url, temp_url):
delete(url)
source_store = fsspec.get_mapper(source_url)
target_store = fsspec.get_mapper(target_url)
temp_store = fsspec.get_mapper(temp_url)
shape = (4000, 4000)
source_chunks = (400, 4000)
target_chunks = (4000, 400)
executor = PywrenExecutor(pywren.local_executor())
# create
# dask_chunks determine the unit of work - they are bigger than the chunks
# of the zarr file we are writing to
dask_chunks = (800, 4000)
zarr_chunks = source_chunks
arr = da.random.random(size=shape, chunks=dask_chunks)
itemsize = arr.dtype.itemsize
max_mem = str(itemsize * prod(dask_chunks))
plan = rechunk(arr, zarr_chunks, max_mem, source_store)
plan.execute(
scheduler=executor_scheduler,
executor=executor,
batch_size=100
)
z = zarr.open_array(source_store, mode="r")
assert z.shape == shape
assert z.chunks == source_chunks
assert z.nchunks == 10
assert z.dtype == float
# rechunk
max_mem = 25_600_000
source_array = zarr.open_array(source_store, mode="r")
plan = rechunk(
source_array, target_chunks, max_mem, target_store, temp_store=temp_store
)
plan.execute(
scheduler=executor_scheduler,
executor=executor,
batch_size=100
)
z = zarr.open_array(target_store, mode="r")
assert z.shape == shape
assert z.chunks == target_chunks
assert z.nchunks == 10
assert z.dtype == float
# delete everything
for url in (source_url, target_url, temp_url):
fs = fsspec.open(url).fs
if fs.exists(url):
fs.rm(url, recursive=True)
elif command == "create":
# create a random Zarr array for the source
# dask_chunks determine the unit of work - they are bigger than the chunks
# of the zarr file we are writing to
dask_chunks = (800, 4000)
zarr_chunks = source_chunks
executor = PywrenExecutor(pywren.local_executor())
arr = da.random.random(size=shape, chunks=dask_chunks)
itemsize = arr.dtype.itemsize
max_mem = str(itemsize * prod(dask_chunks))
plan = rechunk(arr, zarr_chunks, max_mem, source_store)
plan.execute(scheduler=executor_scheduler,
executor=executor,
batch_size=100)
elif command == "rechunk":
# rechunk the source to the target
max_mem = 25_600_000
def pywren_local_function_executor():
return pywren.local_executor(
# Minimal config needed to avoid Pywren error if ~/.pywren_config is missing
config={"pywren": {
"storage_bucket": "unused"
}})