def compute_block_shape_static(shape: tuple, dtype: Union[type, np.dtype], cluster_shape: tuple, num_cores: int): # TODO (hme): This should also compute parameters for DeviceGrid. if array_utils.is_float(dtype, type_test=True): dtype = np.finfo(dtype).dtype elif array_utils.is_int(dtype, type_test=True) \ or array_utils.is_uint(dtype, type_test=True): dtype = np.iinfo(dtype).dtype elif array_utils.is_complex(dtype, type_test=True): dtype = np.dtype(dtype) elif dtype in (bool, np.bool_): dtype = np.dtype(np.bool_) else: raise ValueError("dtype %s not supported" % str(dtype)) nbytes = dtype.alignment size = np.product(shape) * nbytes # If the object is less than 100 megabytes, there's not much value in constructing # a block tensor. if size < 10**8: block_shape = shape return block_shape if len(shape) < len(cluster_shape): cluster_shape = cluster_shape[:len(shape)] elif len(shape) > len(cluster_shape): cluster_shape = list(cluster_shape) for axis in range(len(shape)): if axis >= len(cluster_shape): cluster_shape.append(1) cluster_shape = tuple(cluster_shape) shape_np = np.array(shape, dtype=int) # Softmax on cluster shape gives strong preference to larger dimensions. cluster_weights = np.exp(np.array(cluster_shape)) / np.sum( np.exp(cluster_shape)) shape_fracs = np.array(shape) / np.sum(shape) # cluster_weights weight the proportion of cores available along each axis, # and shape_fracs is the proportion of data along each axis. weighted_shape_fracs = cluster_weights * shape_fracs weighted_shape_fracs = weighted_shape_fracs / np.sum( weighted_shape_fracs) # Compute dimensions of grid shape # so that the number of blocks are close to the number of cores. grid_shape_frac = num_cores**weighted_shape_fracs grid_shape = np.floor(grid_shape_frac) # Put remainder on largest axis. remaining = np.sum(grid_shape_frac - grid_shape) grid_shape[np.argmax(shape)] += remaining grid_shape = np.ceil(grid_shape).astype(int) # We use ceiling of floating block shape # so that resulting grid shape is <= to what we compute above. block_shape = tuple((shape_np + grid_shape - 1) // grid_shape) return block_shape
def permutation(self, x): app = _instance() if _array_utils.is_int(x): shape = (x, ) block_shape = app.compute_block_shape(shape=shape, dtype=_np.int64) return self.rs().permutation(shape[0], block_shape[0]) else: assert isinstance(x, BlockArray) shape = x.shape block_shape = x.shape arr_perm = self.rs().permutation(shape[0], block_shape[0]).get() return x[arr_perm]
def _get_shapes(self, size=None, dtype=None): if dtype is None: dtype = _np.float64 if size is None: size = () if not isinstance(size, tuple): assert _array_utils.is_int(size) shape = (size, ) else: shape = size block_shape = _instance().get_block_shape(shape, dtype) return shape, block_shape
def reshape(self, shape=None, **kwargs): block_shape = kwargs.get("block_shape", None) if array_utils.is_int(shape): shape = (shape,) if (block_shape is None or self.block_shape == block_shape) \ and (shape is None or self.shape == shape): return Reshape()(self, self.shape, self.block_shape) if shape is None: shape = self.shape else: shape = Reshape.compute_shape(self.shape, shape) if block_shape is None: block_shape = self._get_and_register_block_shape(shape) return Reshape()(self, shape, block_shape)
def __init__( self, penalty="none", alpha=1.0, l1_ratio=0.5, tol=0.0001, max_iter=100, solver="newton", lr=0.01, random_state=None, fit_intercept=True, normalize=False, ): if fit_intercept is False: raise NotImplementedError( "fit_incercept=False currently not supported.") if normalize is True: raise NotImplementedError( "normalize=True currently not supported.") self._app = _instance() if random_state is None: self.rs: NumsRandomState = self._app.random elif array_utils.is_int(random_state): self.rs: NumsRandomState = NumsRandomState(cm=self._app.cm, seed=random_state) elif isinstance(random_state, NumsRandomState): self.rs: NumsRandomState = random_state else: raise Exception("Unexpected type for random_state %s" % str(type(random_state))) self._penalty = None if penalty == "none" else penalty if self._penalty not in (None, "l1", "l2", "elasticnet"): raise NotImplementedError("%s penalty not supported" % self._penalty) # All sources use lambda as regularization term, and alpha l1/l2 ratio. self._lambda = alpha self._l1penalty = None self._l1penalty_vec = None self._l2penalty = None self._l2penalty_vec = None self._l2penalty_diag = None self.alpha = l1_ratio self._tol = tol self._max_iter = max_iter self._opt = solver self._lr = lr self._beta = None self._beta0 = None
def reshape(self, shape=None, **kwargs): block_shape = kwargs.get("block_shape", None) if array_utils.is_int(shape): shape = (shape, ) elif shape is None: shape = self.shape shape = Reshape.compute_shape(self.shape, shape) if block_shape is None: if shape == self.shape: # This is a noop. block_shape = self.block_shape else: block_shape = self._get_and_register_block_shape(shape) return Reshape()(self, shape, block_shape)
def nbytes(self): if array_utils.is_float(self.dtype, type_test=True): dtype = np.finfo(self.dtype).dtype elif array_utils.is_int(self.dtype, type_test=True) \ or array_utils.is_uint(self.dtype, type_test=True): dtype = np.iinfo(self.dtype).dtype elif array_utils.is_complex(self.dtype, type_test=True): dtype = np.dtype(self.dtype) elif self.dtype in (bool, np.bool_): dtype = np.dtype(np.bool_) else: raise ValueError("dtype %s not supported" % str(self.dtype)) dtype_nbytes = dtype.alignment nbytes = np.product(self.shape) * dtype_nbytes return nbytes
def reshape(self, *shape, **kwargs): block_shape = kwargs.get("block_shape", None) if array_utils.is_int(shape): shape = (shape, ) elif len(shape) == 0: shape = self.shape elif isinstance(shape[0], (tuple, list)): assert len(shape) == 1 shape = shape[0] else: assert all(np.issubdtype(type(n), int) for n in shape) shape = Reshape.compute_shape(self.shape, shape) if block_shape is None: if shape == self.shape: # This is a noop. block_shape = self.block_shape else: block_shape = self.cm.get_block_shape(shape, self.dtype) return Reshape()(self, shape, block_shape)
def __init__( self, penalty="none", C=1.0, tol=0.0001, max_iter=100, solver="newton-cg", lr=0.01, random_state=None, fit_intercept=True, normalize=False, ): if fit_intercept is False: raise NotImplementedError("fit_incercept=False currently not supported.") if normalize is True: raise NotImplementedError("normalize=True currently not supported.") self._app = _instance() if random_state is None: self.rs: NumsRandomState = self._app.random elif array_utils.is_int(random_state): self.rs: NumsRandomState = NumsRandomState( cm=self._app.cm, seed=random_state ) elif isinstance(random_state, NumsRandomState): self.rs: NumsRandomState = random_state else: raise Exception( "Unexpected type for random_state %s" % str(type(random_state)) ) self._penalty = None if penalty == "none" else penalty if not (self._penalty is None or self._penalty == "l2"): raise NotImplementedError("%s penalty not supported" % self._penalty) self._lambda = 1.0 / C self._lambda_vec = None self._tol = tol self._max_iter = max_iter self._opt = solver self._lr = lr self._beta = None self._beta0 = None