def dask_load(sources, geobox, measurements, dask_chunks, skip_broken_datasets=False): def data_func(measurement): return make_dask_array(sources, geobox, measurement, skip_broken_datasets=skip_broken_datasets, dask_chunks=dask_chunks) return Datacube.create_storage(OrderedDict((dim, sources.coords[dim]) for dim in sources.dims), geobox, measurements, data_func)
def dask_load(sources, geobox, measurements, dask_chunks, skip_broken_datasets=False): def data_func(measurement): return make_dask_array(sources, geobox, measurement, skip_broken_datasets=skip_broken_datasets, dask_chunks=dask_chunks) return Datacube.create_storage(sources.coords, geobox, measurements, data_func)
def xr_load(sources, geobox, measurements, skip_broken_datasets=False, use_threads=False): mk_new = get_loader(sources) data = Datacube.create_storage(OrderedDict((dim, sources.coords[dim]) for dim in sources.dims), geobox, measurements) # TODO: re-add use_threads for index, datasets in np.ndenumerate(sources.values): for m in measurements: t_slice = data[m.name].values[index] fuse_measurement(t_slice, datasets, geobox, m, mk_new=mk_new, skip_broken_datasets=skip_broken_datasets) return data
def xr_load(sources, geobox, measurements, skip_broken_datasets=False, use_threads=False): mk_new = get_loader(sources) data = Datacube.create_storage(sources.coords, geobox, measurements) if use_threads: def work_load_data(index, datasets, m): t_slice = data[m.name].values[index] fuse_measurement(t_slice, datasets, geobox, m, mk_new=mk_new, skip_broken_datasets=skip_broken_datasets) futures = [] pool = ThreadPoolExecutor(cpu_count() * 2) for index, datasets in np.ndenumerate(sources.values): for m in measurements: futures.append(pool.submit(work_load_data, index, datasets, m)) wait(futures) else: for index, datasets in np.ndenumerate(sources.values): for m in measurements: t_slice = data[m.name].values[index] fuse_measurement(t_slice, datasets, geobox, m, mk_new=mk_new, skip_broken_datasets=skip_broken_datasets) return data