def _reduce(self, local_reduce_name, axes=None, dtype=None, out=None):
if any(0 in localshape for localshape in self.localshapes()):
raise NotImplementedError("Reduction not implemented for empty "
"LocalArrays")
if out is not None:
_raise_nie()
dtype = dtype or self.dtype
out_dist = self.distribution.reduce(axes=axes)
ddpr = out_dist.get_dim_data_per_rank()
def _local_reduce(local_name, larr, out_comm, ddpr, dtype, axes):
import distarray.localapi.localarray as la
local_reducer = getattr(la, local_name)
res = proxyize(la.local_reduction(out_comm, local_reducer, larr, # noqa
ddpr, dtype, axes))
return res
local_reduce_args = (local_reduce_name, self.key, out_dist.comm, ddpr,
dtype, normalize_reduction_axes(axes, self.ndim))
out_key = self.context.apply(_local_reduce, local_reduce_args,
targets=self.targets)[0]
return DistArray.from_localarrays(key=out_key, distribution=out_dist,
dtype=dtype)
def mean(self, axis=None, dtype=float, out=None):
if axis or out is not None:
_raise_nie()
elif dtype is not None:
dtype = np.dtype(dtype)
return dtype.type((np.divide(self.sum(dtype=dtype), self.size)))
else:
return np.divide(self.sum(dtype=dtype), self.size)
def std(self, axis=None, dtype=None, out=None):
if axis or out is not None:
_raise_nie()
elif dtype is not None:
dtype = np.dtype(dtype)
return dtype.type((math.sqrt(self.var())))
else:
return math.sqrt(self.var())
def std(self, axis=None, dtype=None, out=None):
if axis or out is not None:
_raise_nie()
keys = self.context._key_and_push(axis, dtype)
result_key = self.context._generate_key()
subs = (result_key, self.key) + keys
self.context._execute('%s = %s.std(%s,%s)' % subs)
result = self.context._pull0(result_key)
return result
def magic(major, minor, prefix=MAGIC_PREFIX):
"""Return the magic string for the given file format version.
Parameters
----------
major : int in [0, 255]
minor : int in [0, 255]
Returns
-------
magic : str
Raises
------
ValueError if the version cannot be formatted.
"""
if major < 0 or major > 255:
raise ValueError("Major version must be 0 <= major < 256.")
if minor < 0 or minor > 255:
raise ValueError("Minor version must be 0 <= minor < 256.")
if six.PY2:
return prefix + chr(major) + chr(minor)
elif six.PY3:
return prefix + bytes([major, minor])
else:
raise _raise_nie()
def view(self, dtype=None):
"""Return a new LocalArray whose underlying `ndarray` is a view on
`self.ndarray`.
Note
----
Currently unimplemented for ``dtype is not None``.
"""
if dtype is None:
new_da = self.__class__(distribution=self.distribution,
dtype=self.dtype,
buf=self.ndarray)
else:
_raise_nie()
# TODO: to implement this properly, a new dim_data will need to be
# generated that reflects the size and shape of the new dtype.
#new_da = self.__class__(distribution=self.distribution
# dtype=dtype,
# buf=self.ndarray)
return new_da
def _reduce(self, local_reduce_name, axes=None, dtype=None, out=None):
if any(0 in localshape for localshape in self.localshapes()):
raise NotImplementedError("Reduction not implemented for empty "
"LocalArrays")
if out is not None:
_raise_nie()
dtype = dtype or self.dtype
out_dist = self.distribution.reduce(axes=axes)
ddpr = out_dist.get_dim_data_per_rank()
def _local_reduce(local_name, larr, out_comm, ddpr, dtype, axes):
import distarray.localapi.localarray as la
local_reducer = getattr(la, local_name)
res = proxyize(
la.local_reduction(
out_comm,
local_reducer,
larr, # noqa
ddpr,
dtype,
axes))
return res
local_reduce_args = (local_reduce_name, self.key, out_dist.comm, ddpr,
dtype, normalize_reduction_axes(axes, self.ndim))
out_key = self.context.apply(_local_reduce,
local_reduce_args,
targets=self.targets)[0]
return DistArray.from_localarrays(key=out_key,
distribution=out_dist,
dtype=dtype)
def read_magic(fp):
"""Read the magic string to get the version of the file format.
Parameters
----------
fp : filelike object
Returns
-------
major : int
minor : int
"""
magic_str = _read_bytes(fp, MAGIC_LEN, "magic string")
if magic_str[:-2] != MAGIC_PREFIX:
msg = "the magic string is not correct; expected %r, got %r"
raise ValueError(msg % (MAGIC_PREFIX, magic_str[:-2]))
if six.PY2:
major, minor = map(ord, magic_str[-2:])
elif six.PY3:
major, minor = magic_str[-2:]
else:
raise _raise_nie()
return major, minor
def __itruediv__(self, other):
_raise_nie()
def __ifloordiv__(self, other):
_raise_nie()
def arange(start, stop=None, step=1, dtype=None, dist={0: 'b'},
grid_shape=None, comm=None):
_raise_nie()
def __ifloordiv__(self, other):
_raise_nie()
def __iand__(self, other):
_raise_nie()
def where(condition, x=None, y=None):
_raise_nie()
def __rdivmod__(self, other):
_raise_nie()
def arange(start, stop=None, step=1, dtype=None, dist={0:'b'},
grid_shape=None, comm=None):
_raise_nie()
def identity(n, dtype=np.intp):
_raise_nie()
def asdistarray(object, dtype=None, order=None):
_raise_nie()
def distarray(object, dtype=None, copy=True, order=None, subok=False, ndmin=0):
_raise_nie()
def __ixor__(self, other):
_raise_nie()
def any(self, axis=None, out=None):
_raise_nie()
def concatenate(seq, axis=0):
_raise_nie()
def __deepcopy__(self):
_raise_nie()
def correlate(x, y, mode='valid'):
_raise_nie()
def __imul__(self, other):
_raise_nie()
def convolve(x, y, mode='valid'):
_raise_nie()
def __ipow__(self, other, modulo=None):
_raise_nie()
def outer(a, b):
_raise_nie()
def localarray(object, dtype=None, copy=True, order=None, subok=False,
ndmin=0):
_raise_nie()
def inner(a, b):
_raise_nie()
def where(condition, x=None, y=None):
_raise_nie()
def vdot(a, b):
_raise_nie()
def __iand__(self, other):
_raise_nie()
def tensordot(a, b, axes=(-1,0)):
_raise_nie()
def var(self, axis=None, dtype=None, out=None):
if axis or out is not None:
_raise_nie()
mu = self.mean()
temp = (self - mu)**2
return temp.mean(dtype=dtype)
def cross(a, b, axisa=-1, axisb=-1, axisc=-1, axis=None):
_raise_nie()
def cumprod(self, axis=None, dtype=None, out=None):
_raise_nie()
def allclose(a, b, rtol=10e-5, atom=10e-8):
_raise_nie()
def __copy__(self):
_raise_nie()
def distarray2string(a):
_raise_nie()
def __nonzero__(self):
_raise_nie()
def average(a, axis=None, weights=None, returned=0):
_raise_nie()
def __isub__(self, other):
_raise_nie()
def cov(x, y=None, rowvar=1, bias=0):
_raise_nie()
def __itruediv__(self, other):
_raise_nie()
def corrcoef(x, y=None, rowvar=1, bias=0):
_raise_nie()
def __imod__(self, other):
_raise_nie()
def median(m):
_raise_nie()
def __irshift__(self, other):
_raise_nie()
def digitize(x, bins):
_raise_nie()
def __ixor__(self, other):
_raise_nie()
def __ipow__(self, other, modulo=None):
_raise_nie()
def aslocalarray(object, dtype=None, order=None):
_raise_nie()
def __imod__(self, other):
_raise_nie()
def identity(n, dtype=np.intp):
_raise_nie()
def trace(self, offset=0, axis1=0, axis2=1, dtype=None, out=None):
_raise_nie()
def __imul__(self, other):
_raise_nie()
def sum(self, axis=None, dtype=None, out=None):
if axis or out is not None:
_raise_nie()
return sum(self, dtype=dtype)
def histogram(x, bins=None, range=None, normed=False):
_raise_nie()
def __irshift__(self, other):
_raise_nie()
