def register_with_dask(classes):
"""
Register classes with dask so that it can serialize the underlying
numpy arrays a bit faster
"""
try:
from distributed.protocol import register_generic
for c in classes:
register_generic(c)
except:
pass
# In[ ]:
#tag::serialize_class_with_numpy[]
class NumpyInfo:
def __init__(self, name: str, features: npt.ArrayLike):
self.name = name
self.features = features
i = NumpyInfo("boo", np.array(0))
numpybits = [i]
# Surprisingly this works, despite the implication that we would need to call register_generic
from distributed.protocol import register_generic
register_generic(NumpyInfo)
dask.compute(ok_fun(1))
#end::serialize_class_with_numpy[]
# In[ ]:
dask.visualize(ok_fun(1))
# In[ ]:
ok_fun(1).visualize()
# In[ ]:
ok_fun(1)
def to_fits(self):
if self.data is None or len(self.data.shape) == 0:
data = None
else:
data = self.data
if self.kind == "image":
return fits.ImageHDU(data, self.header)
elif self.kind == "bintable":
return fits.BinTableHDU(data, self.header)
elif self.kind == "primary":
return fits.PrimaryHDU(data, self.header)
else:
raise ValueError(f"Unknown kind: {self.kind}")
register_generic(DaskHDU)
def _check_ext(key, hdu, idx):
if "EXTNAME" in hdu.header and isinstance(
key, str) and hdu.header["EXTNAME"].lower() == key.lower():
return True
if idx == key:
return True
return False
class DaskHDUList:
"""Represent a list of FITS header-data units in a way that is
convenient for Dask to serialize and deserialize
"""
@dask_deserialize.register(RandomForestRegressor)
@cuda_deserialize.register(RandomForestRegressor)
def rfr_deserialize(header, frames):
return pickle_loads(header, frames)
@dask_serialize.register(RandomForestClassifier)
@cuda_serialize.register(RandomForestClassifier)
def rfc_serialize(rf):
return pickle_dumps(rf)
@dask_deserialize.register(RandomForestClassifier)
@cuda_deserialize.register(RandomForestClassifier)
def rfc_deserialize(header, frames):
return pickle_loads(header, frames)
register_generic(SparseCumlArray, 'cuda', cuda_serialize, cuda_deserialize)
register_generic(SparseCumlArray, 'dask', dask_serialize, dask_deserialize)
register_generic(cuml.Base, 'cuda', cuda_serialize, cuda_deserialize)
register_generic(cuml.Base, 'dask', dask_serialize, dask_deserialize)
register_generic(MultinomialNB, 'cuda', cuda_serialize, cuda_deserialize)
register_generic(MultinomialNB, 'dask', dask_serialize, dask_deserialize)
copyreg.pickle(cp.cusparse.MatDescriptor, serialize_mat_descriptor)
except ImportError:
# distributed is probably not installed on the system
pass