def add_data(self, gdf): # Populate columns idxs if not self.col_idx: for i, x in enumerate(gdf.columns.values): self.col_idx[str(x)] = i # Generate `ind` array to map each row to an output file. # This approach is certainly more optimized for shuffling # than it is for non-shuffling, but using a single code # path is probably worth the (possible) minor overhead. nrows = gdf.shape[0] typ = np.min_scalar_type(nrows * 2) if self.shuffle: ind = cp.random.choice(cp.arange(self.num_out_files, dtype=typ), nrows) else: ind = cp.arange(nrows, dtype=typ) cp.floor_divide(ind, math.ceil(nrows / self.num_out_files), out=ind) for x, group in enumerate( gdf.scatter_by_map(ind, map_size=self.num_out_files, keep_index=False)): self.num_samples[x] += len(group) if self.num_threads > 1: self.queue.put((x, group)) else: self._write_table(x, group) # wait for all writes to finish before exiting # (so that we aren't using memory) if self.num_threads > 1: self.queue.join()
def _add_data_scatter(self, gdf): """Write scattered pieces. This method is for cudf-backed data only. """ assert not self.cpu # Generate `ind` array to map each row to an output file. # This approach is certainly more optimized for shuffling # than it is for non-shuffling, but using a single code # path is probably worth the (possible) minor overhead. nrows = gdf.shape[0] typ = np.min_scalar_type(nrows * 2) if self.shuffle: ind = cp.random.choice(cp.arange(self.num_out_files, dtype=typ), nrows) else: ind = cp.arange(nrows, dtype=typ) cp.floor_divide(ind, math.ceil(nrows / self.num_out_files), out=ind) for x, group in enumerate( gdf.scatter_by_map(ind, map_size=self.num_out_files, keep_index=False)): self.num_samples[x] += len(group) if self.num_threads > 1: self.queue.put((x, group)) else: self._write_table(x, group)
def _masked_column_median(arr, masked_value): """Compute the median of each column in the 2D array arr, ignoring any instances of masked_value""" mask = _get_mask(arr, masked_value) if arr.size == 0: return cp.full(arr.shape[1], cp.nan) arr_sorted = arr.copy() if not cp.isnan(masked_value): # If nan is not the missing value, any column with nans should # have a median of nan nan_cols = cp.any(cp.isnan(arr), axis=0) arr_sorted[mask] = cp.nan else: nan_cols = cp.full(arr.shape[1], False) # nans are always sorted to end of array arr_sorted = cp.sort(arr_sorted, axis=0) count_missing_values = mask.sum(axis=0) # Ignore missing values in determining "halfway" index of sorted # array n_elems = arr.shape[0] - count_missing_values # If no elements remain after removing missing value, median for # that colum is nan nan_cols = cp.logical_or(nan_cols, n_elems <= 0) col_index = cp.arange(arr_sorted.shape[1]) median = (arr_sorted[cp.floor_divide(n_elems - 1, 2), col_index] + arr_sorted[cp.floor_divide(n_elems, 2), col_index]) / 2 median[nan_cols] = cp.nan return median
def add_data(self, gdf): # Populate columns idxs if not self.col_idx: for i, x in enumerate(gdf.columns.values): self.col_idx[str(x)] = i # list columns in cudf don't currently support chunked writing in parquet. # hack around this by just writing a single file with this partition # this restriction can be removed once cudf supports chunked writing # in parquet if any(is_list_dtype(gdf[col].dtype) for col in gdf.columns): self._write_table(0, gdf, True) return # Generate `ind` array to map each row to an output file. # This approach is certainly more optimized for shuffling # than it is for non-shuffling, but using a single code # path is probably worth the (possible) minor overhead. nrows = gdf.shape[0] typ = np.min_scalar_type(nrows * 2) if self.shuffle: ind = cp.random.choice(cp.arange(self.num_out_files, dtype=typ), nrows) else: ind = cp.arange(nrows, dtype=typ) cp.floor_divide(ind, math.ceil(nrows / self.num_out_files), out=ind) for x, group in enumerate( gdf.scatter_by_map(ind, map_size=self.num_out_files, keep_index=False)): self.num_samples[x] += len(group) if self.num_threads > 1: self.queue.put((x, group)) else: self._write_table(x, group) # wait for all writes to finish before exiting # (so that we aren't using memory) if self.num_threads > 1: self.queue.join()
def floor_divide(x1: Array, x2: Array, /) -> Array: """ Array API compatible wrapper for :py:func:`np.floor_divide <numpy.floor_divide>`. See its docstring for more information. """ if x1.dtype not in _numeric_dtypes or x2.dtype not in _numeric_dtypes: raise TypeError("Only numeric dtypes are allowed in floor_divide") # Call result type here just to raise on disallowed type combinations _result_type(x1.dtype, x2.dtype) x1, x2 = Array._normalize_two_args(x1, x2) return Array._new(np.floor_divide(x1._array, x2._array))
def __rfloordiv__(self, other): return cupy.floor_divide(other, self)
def __floordiv__(self, other): return cupy.floor_divide(self, other)
if __name__ == "__main__": random.seed(2222) tempwt = [] tempval = [] for i in range(3000000): tempwt.append(random.randint(1, 100)) tempval.append(i + 1) wt = cp.array(tempwt) val = cp.array(tempval) capacity = 5000000 start = time.time() cost = cp.floor_divide(val, wt) iVal = cp.column_stack((wt, val, cost)) # sorting items by value sorted_iVal = iVal[cp.argsort(-iVal[:, -1])] iVal_cpu = cp.asnumpy(sorted_iVal) #print(iVal) #print(sorted_iVal) #print("Finding Max") maxValue = getMaxValue(iVal_cpu, capacity) t = time.time() - start
def _scale(a, n, m, copy=True): """Scale an array of unsigned/positive integers from `n` to `m` bits. Numbers can be represented exactly only if `m` is a multiple of `n`. Parameters ---------- a : ndarray Input image array. n : int Number of bits currently used to encode the values in `a`. m : int Desired number of bits to encode the values in `out`. copy : bool, optional If True, allocates and returns new array. Otherwise, modifies `a` in place. Returns ------- out : array Output image array. Has the same kind as `a`. """ kind = a.dtype.kind if n > m and a.max() < 2**m: mnew = math.ceil(m / 2) * 2 if mnew > m: dtype = "int{}".format(mnew) else: dtype = "uint{}".format(mnew) n = math.ceil(n / 2) * 2 warn("Downcasting {} to {} without scaling because max " "value {} fits in {}".format(a.dtype, dtype, a.max(), dtype), stacklevel=3) return a.astype(_dtype_bits(kind, m)) elif n == m: return a.copy() if copy else a elif n > m: # downscale with precision loss if copy: b = cp.empty(a.shape, _dtype_bits(kind, m)) cp.floor_divide(a, 2**(n - m), out=b, dtype=a.dtype, casting='unsafe') return b else: a //= 2**(n - m) return a elif m % n == 0: # exact upscale to a multiple of `n` bits if copy: b = cp.empty(a.shape, _dtype_bits(kind, m)) cp.multiply(a, (2**m - 1) // (2**n - 1), out=b, dtype=b.dtype) return b else: a = a.astype(_dtype_bits(kind, m, a.dtype.itemsize), copy=False) a *= (2**m - 1) // (2**n - 1) return a else: # upscale to a multiple of `n` bits, # then downscale with precision loss o = (m // n + 1) * n if copy: b = cp.empty(a.shape, _dtype_bits(kind, o)) cp.multiply(a, (2**o - 1) // (2**n - 1), out=b, dtype=b.dtype) b //= 2**(o - m) return b else: a = a.astype(_dtype_bits(kind, o, a.dtype.itemsize), copy=False) a *= (2**o - 1) // (2**n - 1) a //= 2**(o - m) return a