def test_load_nu(self):
distribution = Distribution.from_dim_data_per_rank(self.context,
nu_test_data)
da = self.context.load_npy(self.output_path, distribution)
for i in range(da.shape[0]):
for j in range(da.shape[1]):
self.assertEqual(da[i, j], self.expected[i, j])
def test_load_nu(self):
distribution = Distribution.from_dim_data_per_rank(self.context,
nu_test_data)
da = self.context.load_hdf5(self.output_path, distribution, key="test")
assert_array_equal(self.expected, da)
def from_localarrays(cls, key, context=None, targets=None, distribution=None,
dtype=None):
"""The caller has already created the LocalArray objects. `key` is
their name on the engines. This classmethod creates a DistArray that
refers to these LocalArrays.
Either a `context` or a `distribution` must also be provided. If
`context` is provided, a ``dim_data_per_rank`` will be pulled from
the existing ``LocalArray``\s and a ``Distribution`` will be created
from it. If `distribution` is provided, it should accurately
reflect the distribution of the existing ``LocalArray``\s.
If `dtype` is not provided, it will be fetched from the engines.
"""
def get_dim_datas_and_dtype(arr):
return (arr.dim_data, arr.dtype)
da = cls.__new__(cls)
da.key = key
if (context is None) == (distribution is None):
errmsg = "Must provide `context` or `distribution` but not both."
raise RuntimeError(errmsg)
# has context, get dist and dtype
elif (distribution is None) and (dtype is None):
res = context.apply(get_dim_datas_and_dtype, args=(key,),
targets=targets)
dim_datas = [i[0] for i in res]
dtypes = [i[1] for i in res]
da._dtype = dtypes[0]
da.distribution = Distribution.from_dim_data_per_rank(context,
dim_datas,
targets)
# has context and dtype, get dist
elif (distribution is None) and (dtype is not None):
da._dtype = dtype
dim_datas = context.apply(getattr, args=(key, 'dim_data'),
targets=targets)
da.distribution = Distribution.from_dim_data_per_rank(context,
dim_datas,
targets)
# has distribution, get dtype
elif (distribution is not None) and (dtype is None):
da.distribution = distribution
da._dtype = distribution.context.apply(getattr,
args=(key, 'dtype'),
targets=[0])[0]
# has distribution and dtype
elif (distribution is not None) and (dtype is not None):
da.distribution = distribution
da._dtype = dtype
# sanity check that I didn't miss any cases above, because this is a
# confusing function
else:
assert False
return da
def from_localarrays(cls,
key,
context=None,
targets=None,
distribution=None,
dtype=None):
"""The caller has already created the LocalArray objects. `key` is
their name on the engines. This classmethod creates a DistArray that
refers to these LocalArrays.
Either a `context` or a `distribution` must also be provided. If
`context` is provided, a ``dim_data_per_rank`` will be pulled from
the existing ``LocalArray``\s and a ``Distribution`` will be created
from it. If `distribution` is provided, it should accurately
reflect the distribution of the existing ``LocalArray``\s.
If `dtype` is not provided, it will be fetched from the engines.
"""
def get_dim_datas_and_dtype(arr):
return (arr.dim_data, arr.dtype)
da = cls.__new__(cls)
da.key = key
if (context is None) == (distribution is None):
errmsg = "Must provide `context` or `distribution` but not both."
raise RuntimeError(errmsg)
# has context, get dist and dtype
elif (distribution is None) and (dtype is None):
res = context.apply(get_dim_datas_and_dtype,
args=(key, ),
targets=targets)
dim_datas = [i[0] for i in res]
dtypes = [i[1] for i in res]
da._dtype = dtypes[0]
da.distribution = Distribution.from_dim_data_per_rank(
context, dim_datas, targets)
# has context and dtype, get dist
elif (distribution is None) and (dtype is not None):
da._dtype = dtype
dim_datas = context.apply(getattr,
args=(key, 'dim_data'),
targets=targets)
da.distribution = Distribution.from_dim_data_per_rank(
context, dim_datas, targets)
# has distribution, get dtype
elif (distribution is not None) and (dtype is None):
da.distribution = distribution
da._dtype = distribution.context.apply(getattr,
args=(key, 'dtype'),
targets=[0])[0]
# has distribution and dtype
elif (distribution is not None) and (dtype is not None):
da.distribution = distribution
da._dtype = dtype
# sanity check that I didn't miss any cases above, because this is a
# confusing function
else:
assert False
return da
