def main():
if not sys.argv[1:]:
sys.stderr.write("Usage: python %s <avro file>\n" % sys.argv[0])
sys.exit(1)
data = dask.bag.from_delayed(read_avro(sys.argv[1], blocksize=1024 * 1024))
print("Partitions: %s" % data.npartitions)
task = data.count()
with progress.ProgressBar():
count = task.compute()
print("Result: %s" % count)
def main():
if not sys.argv[1:]:
sys.stderr.write("Usage: python %s <query>\n" % sys.argv[0])
sys.exit(1)
search_body = {"query": {"match": {"_all": sys.argv[1]}}}
data = dask.bag.from_delayed(read_elasticsearch(search_body,
npartitions=8))
print("Partitions: %s" % data.npartitions)
task = data.count()
with progress.ProgressBar():
count = task.compute()
print("Result: %s" % count)
# This is a very basic example of using chunks to make computation easier.
# Due to the large number of dimensions, if you didn't chunk the dataset, you'd excede 16 GB of RAM. With chunks, you can run this in 23.1s while barely using any memory.
# More info here: example of using chunking for low-memory situations
import xarray as xr
import dask
from dask.diagnostics import progress
fpath = '/home/jqiu21/Documents/Internship/'
ds = xr.open_dataset(fpath + 'Tair-1979-JFM00.nc', chunks={'time': 1})
print(ds)
delayed_obj = ds.rename({
't': 'tair'
}).to_netcdf(fpath + 'Tair-1979-JFM00_redim.nc',
mode='w',
compute=False,
engine='netcdf4',
format='NETCDF4')
with progress.ProgressBar():
delayed_obj.compute()
def plot_verlauf_von(var, expts, start, end, table_id='Amon'):
query = dict(
experiment_id=expts,
table_id=table_id,
variable_id=[var],
member_id='r1i1p1f1',
)
col = intake.open_esm_datastore(
"https://storage.googleapis.com/cmip6/pangeo-cmip6.json")
col_subset = col.search(require_all_on=["source_id"], **query)
from collections import defaultdict
dsets = defaultdict(dict)
for group, df in col_subset.df.groupby(by=['source_id', 'experiment_id']):
dsets[group[0]][group[1]] = open_delayed(df)
dsets_ = dask.compute(dict(dsets))[0]
expt_da = xr.DataArray(expts,
dims='experiment_id',
name='experiment_id',
coords={'experiment_id': expts})
dsets_aligned = {}
print('Suchen und sortieren der angeforderten Daten.')
for k, v in dsets_.items():
expt_dsets = v.values()
if any([d is None for d in expt_dsets]):
print(f"Missing experiment for {k}")
continue
for ds in expt_dsets:
ds.coords['year'] = ds.time.dt.year
# workaround for
# https://github.com/pydata/xarray/issues/2237#issuecomment-620961663
dsets_ann_mean = [
v[expt].pipe(global_mean).swap_dims({
'time': 'year'
}).drop('time').coarsen(year=12).mean() for expt in expts
]
# align everything with the 4xCO2 experiment
dsets_aligned[k] = xr.concat(dsets_ann_mean, join='outer', dim=expt_da)
print('Berechnung des Globalenmittel von jedem einzelnen Modell.')
print('Dies kann ein paar Minuten dauern.')
with progress.ProgressBar():
dsets_aligned_ = dask.compute(dsets_aligned)[0]
source_ids = list(dsets_aligned_.keys())
source_da = xr.DataArray(source_ids,
dims='source_id',
name='source_id',
coords={'source_id': source_ids})
big_ds = xr.concat(
[ds.reset_coords(drop=True) for ds in dsets_aligned_.values()],
dim=source_da)
df_all = big_ds.sel(year=slice(start, end)).to_dataframe().reset_index()
print('Erstellung des Plots.')
sns.relplot(data=df_all,
x="year",
y=var,
hue='experiment_id',
kind="line",
ci="sd",
aspect=2)
return ()