if skip_existing and '.zmetadata' in annual_mapper:
return 'skipped'
ds_monthly = xr.open_zarr(monthly_mapper, consolidated=True).sel(time=tslice).chunk(chunks)
template = _annual(ds_monthly, compute=False).chunk(chunks)
ds_annual = ds_monthly.map_blocks(_annual, template=template)
annual_mapper.clear()
task = ds_annual.to_zarr(annual_mapper, mode='w', compute=False)
dask.compute(task, retries=task_retries)
zarr.consolidate_metadata(annual_mapper)
return 'done'
if __name__ == '__main__':
df = get_cmip_runs(comp=True, unique=True)
print(df)
split_df = df[df.scenario.str.contains('ssp')].reset_index()
df = df.reset_index()
with dask.config.set(scheduler='processes'):
with ProgressBar():
for i, row in split_df.iterrows():
print(f'processing {i+1} of {len(split_df)}')
print(row)
split_and_write(row.model, row.scenario, row.member, method)
with Client(threads_per_worker=1, memory_limit='28 G') as client:
print(client)
print(client.dashboard_link)
y_scen = y_scen.chunk(chunks)
print('y_scen:\n', y_scen)
if dry_run:
print('skipping write of ... dry_run=True')
return 'skipped'
else:
store.clear()
write = y_scen.to_zarr(store, compute=False, mode='w')
write.compute(retries=3)
zarr.consolidate_metadata(store)
return 'done'
if __name__ == '__main__':
with Client(threads_per_worker=1, memory_limit='14 G') as client:
print(client)
print(client.dashboard_link)
df = get_cmip_runs(comp=True, unique=True).reset_index()
print(df)
for i, row in df.iterrows():
print(f'quantile mapping {i+1} of {len(df)}')
print(row)
result = main(row.model, row.scenario, row.member)
if result == 'done':
client.restart()
print('skipping write of ... dry_run=True')
return 'skip'
else:
store.clear()
write = y_scen.to_zarr(store, compute=False, mode='w')
write.compute(retries=3)
zarr.consolidate_metadata(store)
return 'done'
if __name__ == '__main__':
# gateway = Gateway()
# with gateway.new_cluster(worker_cores=1, worker_memory=12) as cluster:
# client = cluster.get_client()
# cluster.adapt(minimum=20, maximum=375)
with Client(threads_per_worker=1, memory_limit='12 G') as client:
print(client)
print(client.dashboard_link)
df = get_cmip_runs()
print(df)
for i, row in df.iterrows():
print(f'bias correcting {i} of {len(df)}')
print(row)
result = main(row.model, row.scenario, row.member)
if result == 'done':
client.restart()