def _convert_value(self, value): """Convert a string into a numpy object (scalar or array). The value is most of the time a string, but it can be python object in case if TIFF decoder for example. """ if isinstance(value, list): # convert to a numpy array return numpy.array(value) if isinstance(value, dict): # convert to a numpy associative array key_dtype = numpy.min_scalar_type(list(value.keys())) value_dtype = numpy.min_scalar_type(list(value.values())) associative_type = [('key', key_dtype), ('value', value_dtype)] assert key_dtype.kind != "O" and value_dtype.kind != "O" return numpy.array(list(value.items()), dtype=associative_type) if isinstance(value, numbers.Number): dtype = numpy.min_scalar_type(value) assert dtype.kind != "O" return dtype.type(value) if isinstance(value, six.binary_type): try: value = value.decode('utf-8') except UnicodeDecodeError: return numpy.void(value) if " " in value: result = self._convert_list(value) else: result = self._convert_scalar_value(value) return result
def shuffle_group(df, col, stage, k, npartitions): """ Splits dataframe into groups The group is determined by their final partition, and which stage we are in in the shuffle """ if col == '_partitions': ind = df[col] else: ind = hash_pandas_object(df[col], index=False) c = ind._values typ = np.min_scalar_type(npartitions * 2) npartitions, k, stage = [np.array(x, dtype=np.min_scalar_type(x))[()] for x in [npartitions, k, stage]] c = np.mod(c, npartitions).astype(typ, copy=False) c = np.floor_divide(c, k ** stage, out=c) c = np.mod(c, k, out=c) indexer, locations = groupsort_indexer(c.astype(np.int64), k) df2 = df.take(indexer) locations = locations.cumsum() parts = [df2.iloc[a:b] for a, b in zip(locations[:-1], locations[1:])] return dict(zip(range(k), parts))
def __init__(self,Np): if (type(Np) is not int): raise ValueError("expecting integer for Np") self._Np = Np self._Ns = Np+1 self._dtype = _np.min_scalar_type(-self.Ns) self._basis = _np.arange(self.Ns,dtype=_np.min_scalar_type(self.Ns)) self._operators = ("availible operators for ho_basis:"+ "\n\tI: identity "+ "\n\t+: raising operator"+ "\n\t-: lowering operator"+ "\n\tn: number operator")
def histograma(imagen): ajuste = 0 minimo_valor = np.min(imagen) if minimo_valor < 0: ajuste = minimo_valor rango = np.max(imagen).astype(np.int64) - minimo_valor ajuste_dtype = np.promote_types(np.min_scalar_type(rango),np.min_scalar_type(minimo_valor)) if imagen.dtype != ajuste_dtype: imagen = imagen.astype(ajuste_dtype) imagen = imagen - ajuste hist = np.bincount(imagen.ravel()) valores_centrales = np.arange(len(hist)) + ajuste idx = np.nonzero(hist)[0][0] return hist[idx:], valores_centrales[idx:]
def _offset_array(arr, low_boundary, high_boundary): """Offset the array to get the lowest value at 0 if negative.""" if low_boundary < 0: offset = low_boundary dyn_range = high_boundary - low_boundary # get smallest dtype that can hold both minimum and offset maximum offset_dtype = np.promote_types(np.min_scalar_type(dyn_range), np.min_scalar_type(low_boundary)) if arr.dtype != offset_dtype: # prevent overflow errors when offsetting arr = arr.astype(offset_dtype) arr = arr - offset else: offset = 0 return arr, offset
def normalize_binop_value(self, other): if other is None: return other other_dtype = np.min_scalar_type(other) if other_dtype.kind in "biuf": other_dtype = np.promote_types(self.dtype, other_dtype) if other_dtype == np.dtype("float16"): other = np.dtype("float32").type(other) other_dtype = other.dtype if other_dtype.kind in "u": other_dtype = min_signed_type(other) if np.isscalar(other): other = np.dtype(other_dtype).type(other) return other else: ary = utils.scalar_broadcast_to(other, size=len(self), dtype=other_dtype) return column.build_column( data=Buffer.from_array_lik(ary), dtype=ary.dtype, mask=self.mask, ) else: raise TypeError("cannot broadcast {}".format(type(other)))
def _get_matching_coords(coords, params, shape): """ Get the matching coords across a number of broadcast operands. Parameters ---------- coords : list[numpy.ndarray] The input coordinates. params : list[Union[bool, none]] The broadcast parameters. Returns ------- numpy.ndarray The broacasted coordinates """ matching_coords = [] dims = np.zeros(len(coords), dtype=np.uint8) for p_all in zip(*params): for i, p in enumerate(p_all): if p: matching_coords.append(coords[i][dims[i]]) break else: matching_coords.append(coords[dims[0]]) for i, p in enumerate(p_all): if p is not None: dims[i] += 1 dtype = np.min_scalar_type(max(shape) - 1) return np.asarray(matching_coords, dtype=dtype)
def __init__(self, func, nbins, args=None, kwargs=None): self.func = func self.args = args or () self.kwargs = kwargs or {} self.nbins = nbins self.index_dtype = np.min_scalar_type(self.nbins) self.labels = ['{!r} bin {:d}'.format(func, ibin) for ibin in range(nbins)]
def unpaint_mask(painted_mask, inverse_colormap=None): # Covert color mask to index mask # mask: HWC BGR [0; 255] # colormap: (R, G, B) -> index assert len(painted_mask.shape) == 3 if inverse_colormap is None: inverse_colormap = _default_unpaint_colormap if callable(inverse_colormap): map_fn = lambda a: inverse_colormap((a >> 16) & 255, (a >> 8) & 255, a & 255) else: map_fn = lambda a: inverse_colormap[((a >> 16) & 255, (a >> 8) & 255, a & 255)] painted_mask = painted_mask.astype(int) painted_mask = painted_mask[:, :, 0] + \ (painted_mask[:, :, 1] << 8) + \ (painted_mask[:, :, 2] << 16) uvals, unpainted_mask = np.unique(painted_mask, return_inverse=True) palette = np.array([map_fn(v) for v in uvals], dtype=np.min_scalar_type(len(uvals))) unpainted_mask = palette[unpainted_mask].reshape(painted_mask.shape[:2]) return unpainted_mask
def full_cumsum(data, axis=None, dtype=None): """ A version of `numpy.cumsum` that includes the sum of the empty slice (zero). This makes it satisfy the invariant:: cumsum(a)[i] == sum(a[:i]) which is a useful property to simplify the formula for the moving average. The result will be one entry longer than *data* along *axis*. """ # All we need to do is construct a result array with the appropriate type and # dimensions, and then feed a slice of it to cumsum, setting the rest to zero. shape = list(data.shape) if axis is None: shape[0] += 1 else: shape[axis] += 1 # Mimic cumsum's behavior with the dtype argument: use the original data type or # the system's native word, whichever has the greater width. (This prevents us from # attempting a cumulative sum using an 8-bit integer, for instance.) if dtype is None: dtype = np.promote_types(data.dtype, np.min_scalar_type(-sys.maxint)) out = np.zeros(shape, dtype) s = axis_slice(axis) np.cumsum(data, axis, dtype, out[s[1:]]) return out
def start(self, sess, summary_manager=None): super(ExperienceReplay, self).start(sess, summary_manager) self._latest_observation = self._env.reset() if self._experience is not None: # Experience was restored. return obs_type = self._latest_observation.dtype obs_shape = self._latest_observation.shape if isinstance(self._env.action_space, spaces.Discrete): act_type = np.min_scalar_type(self._env.action_space.n) act_shape = tuple() else: act_type = self._env.action_space.sample().dtype act_shape = self._env.action_space.shape self._experience = Experience(obs_shape, obs_type, act_shape, act_type, self._experience_size) for _ in range(self._experience_start_size): obs = self._latest_observation action = self.action_space.sample() self._latest_observation, reward, done, _ = self._env.step(action) self._experience.put(obs, action, reward, done) if done: self._latest_observation = self._env.reset()
def mask(self, new_mask): try: assert (np.min_scalar_type(new_mask) <= np.dtype(np.bool8)) except: raise TypeError('`mask` must be of boolean type') new_mask = np.asarray(new_mask).astype(np.bool8) self._mask = new_mask
def da_sub(daa, dab): """ subtract 2 DataArrays as cleverly as possible: * keep the metadata of the first DA in the result * ensures the result has the right type so that no underflows happen returns (DataArray): the result of daa - dab """ rt = numpy.result_type(daa, dab) # dtype of result of daa-dab dt = None # default is to let numpy decide if rt.kind == "f": # float should always be fine pass elif rt.kind in "iub": # underflow can happen (especially if unsigned) # find the worse case value (could be improved, but would be longer) worse_val = int(daa.min()) - int(dab.max()) dt = numpy.result_type(rt, numpy.min_scalar_type(worse_val)) else: # subtracting such a data is suspicious, but try anyway logging.warning("Subtraction on data of type %s unsupported", rt.name) res = numpy.subtract(daa, dab, dtype=dt) # metadata is copied from daa logging.error("type = %s , %s", res.dtype.name, daa.dtype.name) return res
def count_over_time_interval( time, values, time_interval, ignore_nodata, nodata=None): def aggregate_ignore_nodata( values): return (numpy.ones(values.shape[1:], dtype=numpy.uint8) * values.shape[0]).tolist() def aggregate_dont_ignore_nodata( values): return numpy.sum(values != nodata, 0) aggregate = { mds.constants.IGNORE_NODATA: aggregate_ignore_nodata, mds.constants.DONT_IGNORE_NODATA: aggregate_dont_ignore_nodata } result_time, result_values = aggregate_over_time_interval(time, values, TIME_POINT_TO_ID_BY_TIME_INTERVAL[time_interval], aggregate[ignore_nodata]) return result_time, [numpy.array(result_values[i], numpy.min_scalar_type( numpy.max(result_values[i]))) for i in xrange(len(result_values))]
def go(self): pi = self.progress.indicator pi.operation = 'Initializing' with pi: self.duration = self.kinetics_file['durations'][self.iter_start-1:self.iter_stop-1] ##Only select transition events from specified istate to fstate mask = (self.duration['istate'] == self.istate) & (self.duration['fstate'] == self.fstate) self.duration_dsspec = DurationDataset(self.kinetics_file['durations']['duration'], mask, self.iter_start) self.wt_dsspec = DurationDataset(self.kinetics_file['durations']['weight'], mask, self.iter_start) self.output_file = h5py.File(self.output_filename, 'w') h5io.stamp_creator_data(self.output_file) # Construct bin boundaries self.construct_bins(self.parse_binspec(self.binspec)) for idim, (binbounds, midpoints) in enumerate(izip(self.binbounds, self.midpoints)): self.output_file['binbounds_{}'.format(idim)] = binbounds self.output_file['midpoints_{}'.format(idim)] = midpoints # construct histogram self.construct_histogram() # Record iteration range iter_range = numpy.arange(self.iter_start, self.iter_stop, 1, dtype=(numpy.min_scalar_type(self.iter_stop))) self.output_file['n_iter'] = iter_range self.output_file['histograms'].attrs['iter_start'] = self.iter_start self.output_file['histograms'].attrs['iter_stop'] = self.iter_stop self.output_file.close()
def _make_matrix(self,op_list,dtype): """ takes list of operator strings and couplings to create matrix.""" off_diag = None diag = None for opstr,indx,J in op_list: ME,row,col = self.Op(opstr,indx,J,dtype) if(len(ME)>0): imax = max(row.max(),col.max()) index_type = _np.result_type(_np.int32,_np.min_scalar_type(imax)) row = row.astype(index_type) col = col.astype(index_type) if _is_diagonal(row,col): if diag is None: diag = _np.zeros(self.Ns,dtype=dtype) _update_diag(diag,row,ME) else: if off_diag is None: off_diag = _sp.csr_matrix((ME,(row,col)),shape=(self.Ns,self.Ns),dtype=dtype) else: off_diag = off_diag + _sp.csr_matrix((ME,(row,col)),shape=(self.Ns,self.Ns),dtype=dtype) if diag is not None and off_diag is not None: indptr = _np.arange(self.Ns+1) return off_diag + _sp.csr_matrix((diag,indptr[:self.Ns],indptr),shape=(self.Ns,self.Ns),dtype=dtype) elif off_diag is not None: return off_diag elif diag is not None: return _sp.dia_matrix((_np.atleast_2d(diag),[0]),shape=(self.Ns,self.Ns),dtype=dtype) else: return _sp.dia_matrix((self.Ns,self.Ns),dtype=dtype)
def __init__(self, num_values, dtype=np.int32, name=None): """Initializes a new `DiscreteArray` spec. Args: num_values: Integer specifying the number of possible values to represent. dtype: The dtype of the array. Must be an integral type large enough to hold `num_values` without overflow. name: Optional string specifying the name of the array. Raises: ValueError: If `num_values` is not positive, if `dtype` is not integral, or if `dtype` is not large enough to hold `num_values` without overflow. """ if num_values <= 0 or not np.issubdtype(type(num_values), np.integer): raise ValueError(_NUM_VALUES_NOT_POSITIVE.format(num_values)) if not np.issubdtype(dtype, np.integer): raise ValueError(_DTYPE_NOT_INTEGRAL.format(dtype)) num_values = int(num_values) maximum = num_values - 1 dtype = np.dtype(dtype) if np.min_scalar_type(maximum) > dtype: raise ValueError(_DTYPE_OVERFLOW.format(dtype, num_values)) super(DiscreteArray, self).__init__(shape=(), dtype=dtype, minimum=0, maximum=maximum, name=name) self._num_values = num_values
def da_sub(daa, dab): """ subtract 2 DataArrays as cleverly as possible: * keep the metadata of the first DA in the result * ensures the result has the right type so that no underflows happen returns (DataArray): the result of daa - dab """ rt = numpy.result_type(daa, dab) # dtype of result of daa-dab dt = None # default is to let numpy decide if rt.kind == "f": # float should always be fine pass elif rt.kind in "iub": # underflow can happen (especially if unsigned) # find the worse case value (could be improved, but would be longer) worse_val = int(daa.min()) - int(dab.max()) dt = numpy.result_type(rt, numpy.min_scalar_type(worse_val)) else: # subtracting such a data is suspicious, but try anyway logging.warning("Subtraction on data of type %s unsupported", rt.name) res = numpy.subtract(daa, dab, dtype=dt) # metadata is copied from daa logging.debug("type = %s, %s", res.dtype.name, daa.dtype.name) return res
def __getitem__(self, key): """ Look up a cell in the map, but clip to the edges For instance, `map[-1, -1] == map[0, 0]`, unlike in a normal np array where it would be `map[map.shape[0]-1, map.shape[1]-1]` Note that this only applies for pairwise integer indexing. Indexing with boolean masks or slice objects uses the normal indexing rules. """ if not isinstance(key, tuple): # probably a mask? return self.grid[key] if len(key) != 2: # row, column, or just wrong return self.grid[key] if any(np.min_scalar_type(i) == np.bool for i in key): # partial mask return self.grid[key] if any(isinstance(i, slice) for i in key): # normal slicing return self.grid[key] keys = np.ravel_multi_index(key, dims=self.grid.shape, mode='clip') # workaround for https://github.com/numpy/numpy/pull/7586 if keys.ndim == 0: return self.grid.take(keys[np.newaxis])[0] else: return self.grid.take(keys)
def normalize_binop_value( self, other: ScalarLike ) -> Union[ColumnBase, ScalarLike]: if other is None: return other if isinstance(other, cudf.Scalar): if self.dtype == other.dtype: return other # expensive device-host transfer just to # adjust the dtype other = other.value elif isinstance(other, np.ndarray) and other.ndim == 0: other = other.item() other_dtype = np.min_scalar_type(other) if other_dtype.kind in {"b", "i", "u", "f"}: if isinstance(other, cudf.Scalar): return other other_dtype = np.promote_types(self.dtype, other_dtype) if other_dtype == np.dtype("float16"): other_dtype = np.dtype("float32") other = other_dtype.type(other) if self.dtype.kind == "b": other_dtype = min_signed_type(other) if np.isscalar(other): other = np.dtype(other_dtype).type(other) return other else: ary = utils.scalar_broadcast_to( other, size=len(self), dtype=other_dtype ) return column.build_column( data=Buffer(ary), dtype=ary.dtype, mask=self.mask, ) else: raise TypeError(f"cannot broadcast {type(other)}")
def __init__(self, func, nbins, args=None, kwargs=None): self.func = func self.args = args or () self.kwargs = kwargs or {} self.nbins = nbins self.index_dtype = numpy.min_scalar_type(self.nbins) self.labels = ['{!r} bin {:d}'.format(func, ibin) for ibin in xrange(nbins)]
def __init__( self, *, max_node_iloc, source_ilocs, rel_ilocs, target_ilocs, batch_size, shuffle, negative_samples, sample_strategy, seed, ): self.max_node_iloc = max_node_iloc num_edges = len(source_ilocs) self.indices = np.arange(num_edges, dtype=np.min_scalar_type(num_edges)) self.source_ilocs = np.asarray(source_ilocs) self.rel_ilocs = np.asarray(rel_ilocs) self.target_ilocs = np.asarray(target_ilocs) self.negative_samples = negative_samples self.sample_strategy = sample_strategy self.batch_size = batch_size self.seed = seed self.shuffle = shuffle _, self._global_rs = random_state(seed) self._batch_sampler = SeededPerBatch( np.random.RandomState, self._global_rs.randint(2**32, dtype=np.uint32))
def _check_and_cast_columns_with_other( source_col: ColumnBase, other: Union[ScalarLike, ColumnBase], inplace: bool, ) -> Tuple[ColumnBase, Union[ScalarLike, ColumnBase]]: """ Returns type-casted column `source_col` & scalar `other_scalar` based on `inplace` parameter. """ if cudf.utils.dtypes.is_categorical_dtype(source_col.dtype): return source_col, other if cudf.utils.dtypes.is_scalar(other): device_obj = _normalize_scalars(source_col, other) else: device_obj = other if other is None: return source_col, device_obj elif cudf.utils.dtypes.is_mixed_with_object_dtype(device_obj, source_col): raise TypeError( "cudf does not support mixed types, please type-cast " "the column of dataframe/series and other " "to same dtypes." ) if inplace: if not cudf.utils.dtypes._can_cast(device_obj.dtype, source_col.dtype): warnings.warn( f"Type-casting from {device_obj.dtype} " f"to {source_col.dtype}, there could be potential data loss" ) return source_col, device_obj.astype(source_col.dtype) else: if ( cudf.utils.dtypes.is_scalar(other) and cudf.utils.dtypes._is_non_decimal_numeric_dtype( source_col.dtype ) and cudf.utils.dtypes._can_cast(other, source_col.dtype) ): common_dtype = source_col.dtype return ( source_col.astype(common_dtype), cudf.Scalar(other, dtype=common_dtype), ) else: common_dtype = cudf.utils.dtypes.find_common_type( [ source_col.dtype, np.min_scalar_type(other) if cudf.utils.dtypes.is_scalar(other) else other.dtype, ] ) if cudf.utils.dtypes.is_scalar(device_obj): device_obj = cudf.Scalar(other, dtype=common_dtype) else: device_obj = device_obj.astype(common_dtype) return source_col.astype(common_dtype), device_obj
def _from_coo(x, compressed_axes=None, idx_dtype=None): if x.ndim == 0: if compressed_axes is not None: raise ValueError("no axes to compress for 0d array") return ((x.data, x.coords, []), x.shape, None, x.fill_value) if x.ndim == 1: if compressed_axes is not None: raise ValueError("no axes to compress for 1d array") return ((x.data, x.coords[0], ()), x.shape, None, x.fill_value) compressed_axes = normalize_axis(compressed_axes, x.ndim) if compressed_axes is None: # defaults to best compression ratio compressed_axes = (np.argmin(x.shape), ) check_compressed_axes(x.shape, compressed_axes) axis_order = list(compressed_axes) # array location where the uncompressed dimensions start axisptr = len(compressed_axes) axis_order.extend(np.setdiff1d(np.arange(len(x.shape)), compressed_axes)) reordered_shape = tuple(x.shape[i] for i in axis_order) row_size = np.prod(reordered_shape[:axisptr]) col_size = np.prod(reordered_shape[axisptr:]) compressed_shape = (row_size, col_size) shape = x.shape if idx_dtype and not can_store(idx_dtype, max(max(compressed_shape), x.nnz)): raise ValueError( "cannot store array with the compressed shape {} and nnz {} with dtype {}." .format( compressed_shape, x.nnz, idx_dtype, )) if not idx_dtype: idx_dtype = x.coords.dtype if not can_store(idx_dtype, max(max(compressed_shape), x.nnz)): idx_dtype = np.min_scalar_type(max(max(compressed_shape), x.nnz)) # transpose axes, linearize, reshape, and compress linear = linear_loc(x.coords[axis_order], reordered_shape) order = np.argsort(linear) linear = linear[order] coords = np.empty((2, x.nnz), dtype=idx_dtype) strides = 1 for i, d in enumerate(compressed_shape[::-1]): coords[-(i + 1), :] = (linear // strides) % d strides *= d indptr = np.empty(row_size + 1, dtype=idx_dtype) indptr[0] = 0 np.cumsum(np.bincount(coords[0], minlength=row_size), out=indptr[1:]) indices = coords[1] data = x.data[order] return ((data, indices, indptr), shape, compressed_axes, x.fill_value)
def neighbor_pairs(self, unique=False): """Returns all neighbor pairs with their corresponding distances in the lattice. Parameters ---------- unique : bool, optional If True, only unique pairs with i < j are returned. The default is False. Returns ------- pairs : (N, 2) np.ndarray An array containing all neighbor pairs of the lattice. If `unique=True`, the first index is always smaller than the second index in each element. distindices : (N, ) np.ndarray The corresponding distance indices of the neighbor pairs. Examples -------- >>> latt = Lattice.chain() >>> latt.add_atom(neighbors=1) >>> latt.build(5) >>> idx, distidx = latt.neighbor_pairs() >>> idx array([[0, 1], [1, 2], [1, 0], [2, 3], [2, 1], [3, 2]], dtype=uint8) >>> distidx array([0, 0, 0, 0, 0, 0], dtype=uint8) >>> idx, distidx = latt.neighbor_pairs(unique=True) >>> idx array([[0, 1], [1, 2], [2, 3]], dtype=uint8) """ # Build index pairs and corresponding distance array dtype = np.min_scalar_type(self.num_sites) sites = np.arange(self.num_sites, dtype=dtype) sites_t = np.tile(sites, (self.data.neighbors.shape[1], 1)).T pairs = np.reshape([sites_t, self.data.neighbors], newshape=(2, -1)).T distindices = self.data.distances.flatten() # Filter pairs with invalid indices mask = distindices != self.data.invalid_distidx pairs = pairs[mask] distindices = distindices[mask] if unique: # Filter for unique pairs (i < j) mask = pairs[:, 0] < pairs[:, 1] pairs = pairs[mask] distindices = distindices[mask] return pairs, distindices
def __init__(self,Np): if (type(Np) is not int): raise ValueError("expecting integer for Np") self._Np = Np self._Ns = Np+1 self._N = 1 self._dtype = _np.min_scalar_type(-self.Ns) self._basis = _np.arange(self.Ns,dtype=_np.min_scalar_type(self.Ns)) self._operators = ("availible operators for ho_basis:"+ "\n\tI: identity "+ "\n\t+: raising operator"+ "\n\t-: lowering operator"+ "\n\tn: number operator") self._blocks = {} self._unique_me = True
def concatenate(arrays, axis=0): """ Concatenate the input arrays along the given dimension. Parameters ---------- arrays : Iterable[SparseArray] The input arrays to concatenate. axis : int, optional The axis along which to concatenate the input arrays. The default is zero. Returns ------- COO The output concatenated array. Raises ------ ValueError If all elements of :code:`arrays` don't have the same fill-value. See Also -------- numpy.concatenate : NumPy equivalent function """ from .core import COO check_consistent_fill_value(arrays) arrays = [x if isinstance(x, COO) else COO(x) for x in arrays] axis = normalize_axis(axis, arrays[0].ndim) assert all(x.shape[ax] == arrays[0].shape[ax] for x in arrays for ax in set(range(arrays[0].ndim)) - {axis}) nnz = 0 dim = sum(x.shape[axis] for x in arrays) shape = list(arrays[0].shape) shape[axis] = dim data = np.concatenate([x.data for x in arrays]) coords = np.concatenate([x.coords for x in arrays], axis=1) if not can_store(coords.dtype, max(shape)): coords = coords.astype(np.min_scalar_type(max(shape))) dim = 0 for x in arrays: if dim: coords[axis, nnz:x.nnz + nnz] += dim dim += x.shape[axis] nnz += x.nnz return COO( coords, data, shape=shape, has_duplicates=False, sorted=(axis == 0), fill_value=arrays[0].fill_value, )
def render(self): # Convert data to QImage for display. profile = debug.Profiler() if self.image is None or self.image.size == 0: return if isinstance(self.lut, collections.Callable): lut = self.lut(self.image) else: lut = self.lut if self.autoDownsample: # reduce dimensions of image based on screen resolution o = self.mapToDevice(QtCore.QPointF(0, 0)) x = self.mapToDevice(QtCore.QPointF(1, 0)) y = self.mapToDevice(QtCore.QPointF(0, 1)) w = Point(x - o).length() h = Point(y - o).length() xds = int(1 / max(1, w)) yds = int(1 / max(1, h)) image = fn.downsample(self.image, xds, axis=0) image = fn.downsample(image, yds, axis=1) else: image = self.image # if the image data is a small int, then we can combine levels + lut # into a single lut for better performance levels = self.levels if levels is not None and levels.ndim == 1 and image.dtype in ( np.ubyte, np.uint16): if self._effectiveLut is None: eflsize = 2**(image.itemsize * 8) ind = np.arange(eflsize) minlev, maxlev = levels if lut is None: efflut = fn.rescaleData(ind, scale=255. / (maxlev - minlev), offset=minlev, dtype=np.ubyte) else: lutdtype = np.min_scalar_type(lut.shape[0] - 1) efflut = fn.rescaleData(ind, scale=(lut.shape[0] - 1) / (maxlev - minlev), offset=minlev, dtype=lutdtype, clip=(0, lut.shape[0] - 1)) efflut = lut[efflut] self._effectiveLut = efflut lut = self._effectiveLut levels = None argb, alpha = fn.makeARGB(image.transpose((1, 0, 2)[:image.ndim]), lut=lut, levels=levels) self.qimage = fn.makeQImage(argb, alpha, transpose=False)
def main(): args = get_parser().parse_args() label_images = (safeintload(file_) for file_ in args.input_files) out = reduce(combine2labels, label_images) # reduce to smallest type possible out = out.astype(np.min_scalar_type(out.max()), copy=False) outnifti = nibabel.Nifti1Image(out, None) copy_forms(nibabel.load(args.input_files[0]), outnifti) outnifti.to_filename(args.output_file)
def normalize_compare_value(self, other): other_dtype = np.min_scalar_type(other) if other_dtype.kind in 'biuf': other_dtype = np.promote_types(self.dtype, other_dtype) ary = utils.scalar_broadcast_to(other, shape=len(self), dtype=other_dtype) return self.replace(data=Buffer(ary), dtype=ary.dtype) else: raise TypeError('cannot broadcast {}'.format(type(other)))
def normalize_binop_value(self, other): other_dtype = np.min_scalar_type(other) if other_dtype.kind in 'biuf': other_dtype = np.promote_types(self.dtype, other_dtype) ary = utils.scalar_broadcast_to(other, shape=len(self), dtype=other_dtype) return self.replace(data=Buffer(ary), dtype=ary.dtype) else: raise TypeError('cannot broadcast {}'.format(type(other)))
def _convert_scalar_value(self, value): """Convert a string into a numpy int or float. If it is not possible it returns a numpy string. """ try: value = int(value) dtype = numpy.min_scalar_type(value) assert dtype.kind != "O" return dtype.type(value) except ValueError: try: value = float(value) dtype = numpy.min_scalar_type(value) assert dtype.kind != "O" return dtype.type(value) except ValueError: return numpy.string_(value)
def _Constant(t, symbols, inferred_symbols): # String value if isinstance(t.value, (str, bytes)): return dtypes.pointer(dtypes.int8) # Numeric value return dtypes.result_type_of( dtypes.typeclass(type(t.value)), dtypes.typeclass(np.min_scalar_type(t.value).name))
def iter_range(self, iter_start=None, iter_stop=None, iter_step=None, dtype=None): '''Return a sequence for the given iteration numbers and stride, filling in missing values from those stored on ``self``. The smallest data type capable of holding ``iter_stop`` is returned unless otherwise specified using the ``dtype`` argument.''' iter_start = self.iter_start if iter_start is None else iter_start iter_stop = self.iter_stop if iter_stop is None else iter_stop iter_step = self.iter_step if iter_step is None else iter_step return np.arange(iter_start, iter_stop, iter_step, dtype=(dtype or np.min_scalar_type(iter_stop)))
def check_ME(basis_1d, basis_gen, opstr, indx, dtype, err_msg): ME1, row1, col1 = basis_1d.Op(opstr, indx, 1.0, dtype) ME2, row2, col2 = basis_gen.Op(opstr, indx, 1.0, dtype) if len(ME1) != len(ME2): print(opstr, list(indx)) print(basis_1d) print("spin_basis_1d:") print(ME1) print(row1) print(col1) print() print("spin_basis_general") print(ME2) print(row2) print(col2) raise Exception("number of matrix elements do not match.") if len(ME1) > 0 and len(ME2) > 0: row1 = row1.astype(np.min_scalar_type(row1.max())) row2 = row2.astype(np.min_scalar_type(row2.max())) col1 = row2.astype(np.min_scalar_type(col1.max())) col2 = row2.astype(np.min_scalar_type(col2.max())) try: np.testing.assert_allclose(row1, row2, atol=1e-6, err_msg=err_msg) np.testing.assert_allclose(col1, col2, atol=1e-6, err_msg=err_msg) np.testing.assert_allclose(ME1, ME2, atol=1e-6, err_msg=err_msg) except AssertionError: print(err_msg) print(basis_1d) print("difference:") print(ME1 - ME2) print(row1 - row2) print(col1 - col2) print("spin_basis_1d:") print(ME1) print(row1) print(col1) print("spin_basis_general") print(ME2) print(row2) print(col2) raise Exception
def get_proj(self,dtype,pcon=False): """Calculates transformation/projector from symmetry-reduced basis to full (symmetry-free) basis. Notes ----- * particularly useful when a given operation canot be carried out in the symmetry-reduced basis in a straightforward manner. * see also `Op_shift_sector()`. Parameters ----------- dtype : 'type' Data type (e.g. numpy.float64) to construct the projector with. pcon : bool, optional Whether or not to return the projector to the particle number (magnetisation) conserving basis (useful in bosonic/single particle systems). Default is `pcon=False`. Returns -------- scipy.sparse.csc_matrix Transformation/projector between the symmetry-reduced and the full basis. Examples -------- >>> P = get_proj(np.float64,pcon=False) >>> print(P.shape) """ if not self._made_basis: raise AttributeError('this function requires the basis to be constructed first; use basis.make().') basis_pcon = None Ns_full = (self._sps**self._N) if pcon and self._get_proj_pcon: if self._basis_pcon is None: self._basis_pcon = self.__class__(**self._pcon_args) basis_pcon = self._basis_pcon._basis Ns_full = basis_pcon.shape[0] elif pcon and self._get_proj_pcon: raise TypeError("pcon=True only works for basis of a single particle number sector.") sign = _np.ones_like(self._basis,dtype=_np.int8) c = self._n.astype(dtype,copy=True) c *= self._pers.prod() _np.sqrt(c,out=c) _np.power(c,-1,out=c) index_type = _np.result_type(_np.min_scalar_type(-Ns_full),_np.int32) indptr = _np.arange(self._Ns+1,dtype=index_type) indices = _np.arange(self._Ns,dtype=index_type) return self._core.get_proj(self._basis,dtype,sign,c,indices,indptr,basis_pcon=basis_pcon)
def _elemwise_n_ary(func, *args, **kwargs): """ Apply a function to any number of arguments with broadcasting. Parameters ---------- func : Callable The function to apply to arguments. Must support broadcasting. args : list Input :obj:`COO` or :obj:`numpy.ndarray`s. kwargs : dict Additional arguments to pass to the function. Returns ------- COO The output array. Raises ------ ValueError If the input shapes aren't compatible or the result will be dense. """ from .core import COO args = list(args) args_zeros = tuple(_zero_of_dtype(np.dtype(arg)) for arg in args) func_value = func(*args_zeros, **kwargs) func_zero = _zero_of_dtype(func_value.dtype) if func_value != func_zero: raise ValueError("Performing this operation would produce " "a dense result: %s" % str(func)) data_list = [] coords_list = [] cache = {} for mask in product([True, False], repeat=len(args)): if not any(mask): continue ci, di = _unmatch_coo(func, args, mask, cache, **kwargs) coords_list.extend(ci) data_list.extend(di) result_shape = _get_nary_broadcast_shape(*[arg.shape for arg in args]) # Concatenate matches and mismatches data = np.concatenate(data_list) if len(data_list) else np.empty( (0, ), dtype=func_value.dtype) coords = np.concatenate(coords_list, axis=1) if len(coords_list) else \ np.empty((0, len(result_shape)), dtype=np.min_scalar_type(max(result_shape) - 1)) return COO(coords, data, shape=result_shape, has_duplicates=False)
def __init__(self, ids): self._index = pd.Index(ids) self._dtype = np.min_scalar_type(len(self._index)) if not self._index.is_unique: # had some duplicated IDs, which is an error duplicated = self._index[self._index.duplicated()].unique() raise ValueError( f"expected IDs to appear once, found some that appeared more: {comma_sep(duplicated)}" )
def __init__(self, *args): multiples = [np.unique(ids) for ids in args] unique = np.concat(multiples) if len(multiples) > 1: unique = np.unique(unique) unique.sort() self._index = pd.Index(unique) self._dtype = np.min_scalar_type(len(self._index))
def main(): args = get_parser().parse_args() label_map = parse_mapstr(args.map) x = nibabel.load(args.input) xdata = np.asarray(x.dataobj) ydata = np.zeros(xdata.shape, np.min_scalar_type(max(label_map.values()))) for key, val in label_map.items(): ydata[xdata == key] = val y = nibabel.Nifti1Image(ydata, None) copy_forms(x, y) y.to_filename(args.output)
def safely_reduce_dtype(ser): # pandas.Series or numpy.array # reduces the datatype to the lowest possible to reduce storage. orig_dtype = "".join([x for x in ser.dtype.name if x.isalpha()]) # float/int mx = 1 new_itemsize = np.min_scalar_type(ser).itemsize if mx < new_itemsize: mx = new_itemsize new_dtype = orig_dtype + str(mx * 8) return new_dtype
def _init_undirected_adj_lists(self): # record the edge ilocs of incoming, outgoing and both-direction edges undirected = {} for i, (src, tgt) in enumerate(zip(self.sources, self.targets)): undirected.setdefault(tgt, []).append(i) if src != tgt: undirected.setdefault(src, []).append(i) dtype = np.min_scalar_type(len(self.sources)) self._edges_dict = _numpyise(undirected, dtype=dtype)
def iter_range(self, iter_start = None, iter_stop = None, iter_step = None, dtype=None): '''Return a sequence for the given iteration numbers and stride, filling in missing values from those stored on ``self``. The smallest data type capable of holding ``iter_stop`` is returned unless otherwise specified using the ``dtype`` argument.''' iter_start = self.iter_start if iter_start is None else iter_start iter_stop = self.iter_stop if iter_stop is None else iter_stop iter_step = self.iter_step if iter_step is None else iter_step return numpy.arange(iter_start, iter_stop, iter_step, dtype=(dtype or numpy.min_scalar_type(iter_stop)))
def create(predict_fn, word_representations, batch_size, window_size, vocabulary_size, result_callback): assert result_callback is not None instance_dtype = np.min_scalar_type(vocabulary_size - 1) logging.info('Instance elements will be stored using %s.', instance_dtype) batcher = WordBatcher( predict_fn, batch_size, window_size, instance_dtype, result_callback) return batcher
def shuffle_group(df, col, stage, k, npartitions): if col == '_partitions': ind = df[col] else: ind = hash_pandas_object(df[col], index=False) c = ind._values typ = np.min_scalar_type(npartitions * 2) c = c.astype(typ) npartitions, k, stage = [np.array(x, dtype=np.min_scalar_type(x))[()] for x in [npartitions, k, stage]] c = np.mod(c, npartitions, out=c) c = np.floor_divide(c, k ** stage, out=c) c = np.mod(c, k, out=c) indexer, locations = pd.algos.groupsort_indexer(c.astype(np.int64), k) df2 = df.take(indexer) locations = locations.cumsum() parts = [df2.iloc[a:b] for a, b in zip(locations[:-1], locations[1:])] return dict(zip(range(k), parts))
def render(self): # Convert data to QImage for display. profile = debug.Profiler() if self.image is None or self.image.size == 0: return if isinstance(self.lut, collections.Callable): lut = self.lut(self.image) else: lut = self.lut if self.autoDownsample: # reduce dimensions of image based on screen resolution o = self.mapToDevice(QtCore.QPointF(0,0)) x = self.mapToDevice(QtCore.QPointF(1,0)) y = self.mapToDevice(QtCore.QPointF(0,1)) w = Point(x-o).length() h = Point(y-o).length() xds = int(1/max(1, w)) yds = int(1/max(1, h)) image = fn.downsample(self.image, xds, axis=0) image = fn.downsample(image, yds, axis=1) else: image = self.image # if the image data is a small int, then we can combine levels + lut # into a single lut for better performance levels = self.levels if levels is not None and levels.ndim == 1 and image.dtype in (np.ubyte, np.uint16): if self._effectiveLut is None: eflsize = 2**(image.itemsize*8) ind = np.arange(eflsize) minlev, maxlev = levels if lut is None: efflut = fn.rescaleData(ind, scale=255./(maxlev-minlev), offset=minlev, dtype=np.ubyte) else: lutdtype = np.min_scalar_type(lut.shape[0]-1) efflut = fn.rescaleData(ind, scale=(lut.shape[0]-1)/(maxlev-minlev), offset=minlev, dtype=lutdtype, clip=(0, lut.shape[0]-1)) efflut = lut[efflut] self._effectiveLut = efflut lut = self._effectiveLut levels = None argb, alpha = fn.makeARGB(image.transpose((1, 0, 2)[:image.ndim]), lut=lut, levels=levels) self.qimage = fn.makeQImage(argb, alpha, transpose=False)
def __init__(self,b1,b2): if not isinstance(b1,basis): raise ValueError("b1 must be instance of basis class") if not isinstance(b2,basis): raise ValueError("b2 must be instance of basis class") if isinstance(b1,tensor_basis): raise TypeError("Can only create tensor basis with non-tensor type basis") if isinstance(b2,tensor_basis): raise TypeError("Can only create tensor basis with non-tensor type basis") self._b1=b1 self._b2=b2 self._Ns = b1.Ns*b2.Ns self._dtype = _np.min_scalar_type(-self._Ns) self._operators = self._b1._operators +"\n"+ self._b2._operators
def _get_dtype(operand): """ Get the numpy dtype corresponding to the numeric data in the object provided. Args: * operand: An instance of :class:`iris.cube.Cube` or :class:`iris.coords.Coord`, or a number or :class:`numpy.ndarray`. Returns: An instance of :class:`numpy.dtype` """ return np.min_scalar_type(operand) if np.isscalar(operand) \ else operand.dtype
def __init__(self, n_bins, output_group, calc_fpts = True): self.calc_fpts = calc_fpts self.n_bins = n_bins self.iibins = numpy.arange(n_bins) self.iibdisc = numpy.empty((n_bins,), numpy.bool_) self.bin_index_dtype = numpy.min_scalar_type(n_bins) self.tdat_dtype = numpy.dtype( [('traj', self.index_dtype), ('n_iter', self.index_dtype), ('timepoint', self.index_dtype), ('initial_bin', self.bin_index_dtype), ('final_bin', self.bin_index_dtype), ('initial_weight', self.weight_dtype), ('final_weight', self.weight_dtype), ('initial_bin_pop', self.weight_dtype), ('duration', self.index_dtype), ('fpt', self.index_dtype), ]) # HDF5 group in which to store results self.output_group = output_group self.tdat_buffer = numpy.empty((self.tdat_buffersize,), dtype=self.tdat_dtype) self.tdat_buffer_offset = 0 self.output_tdat_offset = 0 self.output_tdat_ds = None # Accumulators/counters self.n_trans = None # shape (n_bins,n_bins) # Time points and per-timepoint data self.last_exit = None # (n_bins,) self.last_entry = None # (n_bins,) self.last_completion = None # (n_bins,n_bins) self.weight_last_exit = None # (n_bins) self.bin_pops_last_exit = None # (n_bins,) # Analysis continuation information self.timepoint = None # current time index for separate calls on same trajectory self.last_bin = None # last region occupied, for separate calls on same trajectory self.last_bin_pop = None # total weight in self.last_region at end of last processing step self.clear()
def shuffle_group(df, col, stage, k, npartitions): """ Splits dataframe into groups The group is determined by their final partition, and which stage we are in in the shuffle Parameters ---------- df: DataFrame col: str Column name on which to split the dataframe stage: int We shuffle dataframes with many partitions we in a few stages to avoid a quadratic number of tasks. This number corresponds to which stage we're in, starting from zero up to some small integer k: int Desired number of splits from this dataframe npartition: int Total number of output partitions for the full dataframe Returns ------- out: Dict[int, DataFrame] A dictionary mapping integers in {0..k} to dataframes such that the hash values of ``df[col]`` are well partitioned. """ if col == '_partitions': ind = df[col] else: ind = hash_pandas_object(df[col], index=False) c = ind._values typ = np.min_scalar_type(npartitions * 2) c = np.mod(c, npartitions).astype(typ, copy=False) np.floor_divide(c, k ** stage, out=c) np.mod(c, k, out=c) indexer, locations = groupsort_indexer(c.astype(np.int64), k) df2 = df.take(indexer) locations = locations.cumsum() parts = [df2.iloc[a:b] for a, b in zip(locations[:-1], locations[1:])] return dict(zip(range(k), parts))
def rasterize_pctcover_geom(geom, shape, affine, scale=None, all_touched=False): """ """ if scale is None: scale = 10 min_dtype = min_scalar_type(scale**2) pixel_size = affine[0]/scale topleftlon = affine[2] topleftlat = affine[5] new_affine = Affine(pixel_size, 0, topleftlon, 0, -pixel_size, topleftlat) new_shape = (shape[0]*scale, shape[1]*scale) rv_array = rasterize_geom(geom, new_shape, new_affine, all_touched=all_touched) rv_array = rebin_sum(rv_array, shape, min_dtype) return rv_array.astype('float32') / (scale**2)
def create(predict_fn, word_representations, batch_size, window_size, vocabulary_size, result_callback): assert result_callback is not None instance_dtype = np.min_scalar_type(vocabulary_size - 1) logging.info('Instance elements will be stored using %s.', instance_dtype) if result_callback.should_average_input(): batcher = EmbeddingMapper( predict_fn, word_representations, result_callback) else: batcher = WordBatcher( predict_fn, batch_size, window_size, instance_dtype, result_callback) return batcher
def _run_interface(self, runtime): from numpy import min_scalar_type tracks, header = nbt.read(self.inputs.in_file) streams = ((ii[0]) for ii in tracks) if isdefined(self.inputs.reference): refnii = nb.load(self.inputs.reference) affine = refnii.get_affine() data_dims = refnii.get_shape()[:3] kwargs = dict(affine=affine) else: iflogger.warn(('voxel_dims and data_dims are deprecated' 'as of dipy 0.7.1. Please use reference ' 'input instead')) if not isdefined(self.inputs.data_dims): data_dims = header['dim'] else: data_dims = self.inputs.data_dims if not isdefined(self.inputs.voxel_dims): voxel_size = header['voxel_size'] else: voxel_size = self.inputs.voxel_dims affine = header['vox_to_ras'] kwargs = dict(voxel_size=voxel_size) data = density_map(streams, data_dims, **kwargs) data = data.astype(min_scalar_type(data.max())) img = nb.Nifti1Image(data, affine) out_file = op.abspath(self.inputs.out_filename) nb.save(img, out_file) iflogger.info( ('Track density map saved as {i}, size={d}, ' 'dimensions={v}').format( i=out_file, d=img.get_shape(), v=img.get_header().get_zooms())) return runtime
def _run_interface(self, runtime): from numpy import min_scalar_type from dipy.tracking.utils import density_map tracks, header = nbt.read(self.inputs.in_file) streams = ((ii[0]) for ii in tracks) if isdefined(self.inputs.reference): refnii = nb.load(self.inputs.reference) affine = refnii.affine data_dims = refnii.shape[:3] kwargs = dict(affine=affine) else: IFLOGGER.warn( 'voxel_dims and data_dims are deprecated as of dipy 0.7.1. Please use reference ' 'input instead') if not isdefined(self.inputs.data_dims): data_dims = header['dim'] else: data_dims = self.inputs.data_dims if not isdefined(self.inputs.voxel_dims): voxel_size = header['voxel_size'] else: voxel_size = self.inputs.voxel_dims affine = header['vox_to_ras'] kwargs = dict(voxel_size=voxel_size) data = density_map(streams, data_dims, **kwargs) data = data.astype(min_scalar_type(data.max())) img = nb.Nifti1Image(data, affine) out_file = op.abspath(self.inputs.out_filename) nb.save(img, out_file) IFLOGGER.info( 'Track density map saved as %s, size=%s, dimensions=%s', out_file, img.shape, img.header.get_zooms()) return runtime
def assign_to_bins(self): '''Assign WEST segment data to bins. Requires the DataReader mixin to be in the inheritance tree''' self.require_binning_group() n_iters = self.last_iter - self.first_iter + 1 max_n_segs = self.max_iter_segs_in_range(self.first_iter, self.last_iter) pcoord_len = self.get_pcoord_len(self.first_iter) assignments = numpy.zeros((n_iters, max_n_segs,pcoord_len), numpy.min_scalar_type(self.n_bins)) populations = numpy.zeros((n_iters, pcoord_len, self.n_bins), numpy.float64) westpa.rc.pstatus('Assigning to bins...') for (iiter, n_iter) in enumerate(xrange(self.first_iter, self.last_iter+1)): westpa.rc.pstatus('\r Iteration {:d}'.format(n_iter), end='') seg_index = self.get_seg_index(n_iter) pcoords = self.get_iter_group(n_iter)['pcoord'][...] weights = seg_index['weight'] for seg_id in xrange(len(seg_index)): assignments[iiter,seg_id,:] = self.mapper.assign(pcoords[seg_id,:,:]) for it in xrange(pcoord_len): populations[iiter, it, :] = numpy.bincount(assignments[iiter,:len(seg_index),it], weights, minlength=self.n_bins) westpa.rc.pflush() del pcoords, weights, seg_index assignments_ds = self.binning_h5group.create_dataset('bin_assignments', data=assignments, compression='gzip') populations_ds = self.binning_h5group.create_dataset('bin_populations', data=populations, compression='gzip') for h5object in (self.binning_h5group, assignments_ds, populations_ds): self.record_data_iter_range(h5object) self.record_data_iter_step(h5object, 1) self.record_data_binhash(h5object) westpa.rc.pstatus()
def model(filename, format='ascii'): """Read an IMOD model file into a numpy array. Initially only ASCII input files are supported. Parameters ---------- filename : string The filename to read. format : string in {'ascii'}, optional Whether the file is binary or ASCII format. Returns ------- mod : array of int A numpy array of the same size as the minimal bounding box containing the model. offset : 3-tuple of int The top-left corner of the bounding box of the model. """ object_dict = model2coords(filename, format=format) offset = np.zeros((3,), dtype=int) limits = np.zeros((3,), dtype=int) max_object_id = max(object_dict.keys()) for value in object_dict: coords = np.round(object_dict[value]['coords']).astype(int) new_offset = np.min(coords, axis=1) offset = np.minimum(offset, new_offset, out=offset) new_limits = np.max(coords, axis=1) limits = np.maximum(limits, new_limits, out=limits) mod = np.zeros(limits + 1, dtype=np.min_scalar_type(max_object_id)) for value in object_dict: coords = np.round(object_dict[value]['coords']).astype(int) coords -= offset[np.newaxis,] mod[tuple(coords)] = value return mod, offset
def label2rgb(label, image=None, colors=None, alpha=0.3, bg_label=-1, bg_color=None, image_alpha=1): """Return an RGB image where color-coded labels are painted over the image. Parameters ---------- label : array Integer array of labels with the same shape as `image`. image : array Image used as underlay for labels. If the input is an RGB image, it's converted to grayscale before coloring. colors : list List of colors. If the number of labels exceeds the number of colors, then the colors are cycled. alpha : float [0, 1] Opacity of colorized labels. Ignored if image is `None`. bg_label : int Label that's treated as the background. bg_color : str or array Background color. Must be a name in `color_dict` or RGB float values between [0, 1]. image_alpha : float [0, 1] Opacity of the image. """ if colors is None: colors = DEFAULT_COLORS colors = [_rgb_vector(c) for c in colors] if image is None: image = np.zeros(label.shape + (3,), dtype=np.float64) # Opacity doesn't make sense if no image exists. alpha = 1 else: if not image.shape[:2] == label.shape: raise ValueError("`image` and `label` must be the same shape") if image.min() < 0: warnings.warn("Negative intensities in `image` are not supported") image = img_as_float(rgb2gray(image)) image = gray2rgb(image) * image_alpha + (1 - image_alpha) # Ensure that all labels are non-negative so we can index into # `label_to_color` correctly. offset = min(label.min(), bg_label) if offset != 0: label = label - offset # Make sure you don't modify the input array. bg_label -= offset new_type = np.min_scalar_type(label.max()) if new_type == np.bool: new_type = np.uint8 label = label.astype(new_type) unique_labels, color_cycle = _match_label_with_color(label, colors, bg_label, bg_color) if len(unique_labels) == 0: return image dense_labels = range(max(unique_labels) + 1) label_to_color = np.array([c for i, c in zip(dense_labels, color_cycle)]) result = label_to_color[label] * alpha + image * (1 - alpha) # Remove background label if its color was not specified. remove_background = bg_label in unique_labels and bg_color is None if remove_background: result[label == bg_label] = image[label == bg_label] return result
def _run_interface(self, runtime): nii1 = nb.load(self.inputs.volume1) nii2 = nb.load(self.inputs.volume2) scale = 1.0 if self.inputs.vol_units == 'mm': voxvol = nii1.header.get_zooms() for i in range(nii1.get_data().ndim - 1): scale = scale * voxvol[i] data1 = nii1.get_data() data1[np.logical_or(data1 < 0, np.isnan(data1))] = 0 max1 = int(data1.max()) data1 = data1.astype(np.min_scalar_type(max1)) data2 = nii2.get_data().astype(np.min_scalar_type(max1)) data2[np.logical_or(data1 < 0, np.isnan(data1))] = 0 max2 = data2.max() maxlabel = max(max1, max2) if isdefined(self.inputs.mask_volume): maskdata = nb.load(self.inputs.mask_volume).get_data() maskdata = ~np.logical_or(maskdata == 0, np.isnan(maskdata)) data1[~maskdata] = 0 data2[~maskdata] = 0 res = [] volumes1 = [] volumes2 = [] labels = np.unique(data1[data1 > 0].reshape(-1)).tolist() if self.inputs.bg_overlap: labels.insert(0, 0) for l in labels: res.append(self._bool_vec_dissimilarity(data1 == l, data2 == l, method='jaccard')) volumes1.append(scale * len(data1[data1 == l])) volumes2.append(scale * len(data2[data2 == l])) results = dict(jaccard=[], dice=[]) results['jaccard'] = np.array(res) results['dice'] = 2.0 * results['jaccard'] / (results['jaccard'] + 1.0) weights = np.ones((len(volumes1),), dtype=np.float32) if self.inputs.weighting != 'none': weights = weights / np.array(volumes1) if self.inputs.weighting == 'squared_vol': weights = weights**2 weights = weights / np.sum(weights) both_data = np.zeros(data1.shape) both_data[(data1 - data2) != 0] = 1 nb.save(nb.Nifti1Image(both_data, nii1.affine, nii1.header), self.inputs.out_file) self._labels = labels self._ove_rois = results self._vol_rois = (np.array(volumes1) - np.array(volumes2)) / np.array(volumes1) self._dice = round(np.sum(weights * results['dice']), 5) self._jaccard = round(np.sum(weights * results['jaccard']), 5) self._volume = np.sum(weights * self._vol_rois) return runtime
def min_numerical_convertible_type(string, check_accuracy=True): """ Parse the string and return the smallest numerical type to use for a safe conversion. :param str string: Representation of a float/integer with text :param bool check_accuracy: If true, a warning is pushed on the logger in case there is a loss of accuracy. :raise ValueError: When the string is not a numerical value :retrun: A numpy numerical type """ if string == "": raise ValueError("Not a numerical value") match = _parse_numeric_value.match(string) if match is None: raise ValueError("Not a numerical value") number, decimal, exponent = match.groups() if decimal is None and exponent is None: # It's an integer # TODO: We could find the int type without converting the number value = int(string) return numpy.min_scalar_type(value).type # Try floating-point try: value = _biggest_float(string) except ValueError: raise ValueError("Not a numerical value") if number is None: number = "" if decimal is None: decimal = "" if exponent is None: exponent = "0" nb_precision_digits = int(exponent) - len(decimal) - 1 precision = _biggest_float(10) ** nb_precision_digits * 2.5 previous_type = _biggest_float for numpy_type in _float_types: if numpy_type == _biggest_float: # value was already casted using the bigger type continue reduced_value = numpy_type(value) if not numpy.isfinite(reduced_value): break # numpy isclose(atol=is not accurate enough) diff = value - reduced_value # numpy 1.8.2 looks to do the substraction using float64... # we lose precision here diff = numpy.abs(diff) if diff > precision: break previous_type = numpy_type # It's the smaller float type which fit with enougth precision numpy_type = previous_type if check_accuracy and numpy_type == _biggest_float: # Check the precision using the original string expected = number + decimal # This format the number without python convertion try: result = numpy.array2string(value, precision=len(number) + len(decimal), floatmode="fixed") except TypeError: # numpy 1.8.2 do not have floatmode argument _logger.warning("Not able to check accuracy of the conversion of '%s' using %s", string, _biggest_float) return numpy_type result = result.replace(".", "").replace("-", "") if not result.startswith(expected): _logger.warning("Not able to convert '%s' using %s without losing precision", string, _biggest_float) return numpy_type
def min_scalar_type(val): """ Wrapper for :py:func:`numpy.min_scalar_dtype` which takes into account types supported by GPUs. """ return _promote_dtype(numpy.min_scalar_type(val))