def _convert_value(self, value):
"""Convert a string into a numpy object (scalar or array).
The value is most of the time a string, but it can be python object
in case if TIFF decoder for example.
"""
if isinstance(value, list):
# convert to a numpy array
return numpy.array(value)
if isinstance(value, dict):
# convert to a numpy associative array
key_dtype = numpy.min_scalar_type(list(value.keys()))
value_dtype = numpy.min_scalar_type(list(value.values()))
associative_type = [('key', key_dtype), ('value', value_dtype)]
assert key_dtype.kind != "O" and value_dtype.kind != "O"
return numpy.array(list(value.items()), dtype=associative_type)
if isinstance(value, numbers.Number):
dtype = numpy.min_scalar_type(value)
assert dtype.kind != "O"
return dtype.type(value)
if isinstance(value, six.binary_type):
try:
value = value.decode('utf-8')
except UnicodeDecodeError:
return numpy.void(value)
if " " in value:
result = self._convert_list(value)
else:
result = self._convert_scalar_value(value)
return result
def shuffle_group(df, col, stage, k, npartitions):
""" Splits dataframe into groups
The group is determined by their final partition, and which stage we are in
in the shuffle
"""
if col == '_partitions':
ind = df[col]
else:
ind = hash_pandas_object(df[col], index=False)
c = ind._values
typ = np.min_scalar_type(npartitions * 2)
npartitions, k, stage = [np.array(x, dtype=np.min_scalar_type(x))[()]
for x in [npartitions, k, stage]]
c = np.mod(c, npartitions).astype(typ, copy=False)
c = np.floor_divide(c, k ** stage, out=c)
c = np.mod(c, k, out=c)
indexer, locations = groupsort_indexer(c.astype(np.int64), k)
df2 = df.take(indexer)
locations = locations.cumsum()
parts = [df2.iloc[a:b] for a, b in zip(locations[:-1], locations[1:])]
return dict(zip(range(k), parts))
def __init__(self,Np):
if (type(Np) is not int):
raise ValueError("expecting integer for Np")
self._Np = Np
self._Ns = Np+1
self._dtype = _np.min_scalar_type(-self.Ns)
self._basis = _np.arange(self.Ns,dtype=_np.min_scalar_type(self.Ns))
self._operators = ("availible operators for ho_basis:"+
"\n\tI: identity "+
"\n\t+: raising operator"+
"\n\t-: lowering operator"+
"\n\tn: number operator")
def histograma(imagen): ajuste = 0 minimo_valor = np.min(imagen) if minimo_valor < 0: ajuste = minimo_valor rango = np.max(imagen).astype(np.int64) - minimo_valor ajuste_dtype = np.promote_types(np.min_scalar_type(rango),np.min_scalar_type(minimo_valor)) if imagen.dtype != ajuste_dtype: imagen = imagen.astype(ajuste_dtype) imagen = imagen - ajuste hist = np.bincount(imagen.ravel()) valores_centrales = np.arange(len(hist)) + ajuste idx = np.nonzero(hist)[0][0] return hist[idx:], valores_centrales[idx:]
def _offset_array(arr, low_boundary, high_boundary):
"""Offset the array to get the lowest value at 0 if negative."""
if low_boundary < 0:
offset = low_boundary
dyn_range = high_boundary - low_boundary
# get smallest dtype that can hold both minimum and offset maximum
offset_dtype = np.promote_types(np.min_scalar_type(dyn_range),
np.min_scalar_type(low_boundary))
if arr.dtype != offset_dtype:
# prevent overflow errors when offsetting
arr = arr.astype(offset_dtype)
arr = arr - offset
else:
offset = 0
return arr, offset
def normalize_binop_value(self, other):
if other is None:
return other
other_dtype = np.min_scalar_type(other)
if other_dtype.kind in "biuf":
other_dtype = np.promote_types(self.dtype, other_dtype)
if other_dtype == np.dtype("float16"):
other = np.dtype("float32").type(other)
other_dtype = other.dtype
if other_dtype.kind in "u":
other_dtype = min_signed_type(other)
if np.isscalar(other):
other = np.dtype(other_dtype).type(other)
return other
else:
ary = utils.scalar_broadcast_to(other,
size=len(self),
dtype=other_dtype)
return column.build_column(
data=Buffer.from_array_lik(ary),
dtype=ary.dtype,
mask=self.mask,
)
else:
raise TypeError("cannot broadcast {}".format(type(other)))
def _get_matching_coords(coords, params, shape):
"""
Get the matching coords across a number of broadcast operands.
Parameters
----------
coords : list[numpy.ndarray]
The input coordinates.
params : list[Union[bool, none]]
The broadcast parameters.
Returns
-------
numpy.ndarray
The broacasted coordinates
"""
matching_coords = []
dims = np.zeros(len(coords), dtype=np.uint8)
for p_all in zip(*params):
for i, p in enumerate(p_all):
if p:
matching_coords.append(coords[i][dims[i]])
break
else:
matching_coords.append(coords[dims[0]])
for i, p in enumerate(p_all):
if p is not None:
dims[i] += 1
dtype = np.min_scalar_type(max(shape) - 1)
return np.asarray(matching_coords, dtype=dtype)
def __init__(self, func, nbins, args=None, kwargs=None):
self.func = func
self.args = args or ()
self.kwargs = kwargs or {}
self.nbins = nbins
self.index_dtype = np.min_scalar_type(self.nbins)
self.labels = ['{!r} bin {:d}'.format(func, ibin) for ibin in range(nbins)]
def unpaint_mask(painted_mask, inverse_colormap=None):
# Covert color mask to index mask
# mask: HWC BGR [0; 255]
# colormap: (R, G, B) -> index
assert len(painted_mask.shape) == 3
if inverse_colormap is None:
inverse_colormap = _default_unpaint_colormap
if callable(inverse_colormap):
map_fn = lambda a: inverse_colormap((a >> 16) & 255,
(a >> 8) & 255, a & 255)
else:
map_fn = lambda a: inverse_colormap[((a >> 16) & 255,
(a >> 8) & 255, a & 255)]
painted_mask = painted_mask.astype(int)
painted_mask = painted_mask[:, :, 0] + \
(painted_mask[:, :, 1] << 8) + \
(painted_mask[:, :, 2] << 16)
uvals, unpainted_mask = np.unique(painted_mask, return_inverse=True)
palette = np.array([map_fn(v) for v in uvals],
dtype=np.min_scalar_type(len(uvals)))
unpainted_mask = palette[unpainted_mask].reshape(painted_mask.shape[:2])
return unpainted_mask
def full_cumsum(data, axis=None, dtype=None):
"""
A version of `numpy.cumsum` that includes the sum of the empty slice (zero). This
makes it satisfy the invariant::
cumsum(a)[i] == sum(a[:i])
which is a useful property to simplify the formula for the moving average. The result
will be one entry longer than *data* along *axis*.
"""
# All we need to do is construct a result array with the appropriate type and
# dimensions, and then feed a slice of it to cumsum, setting the rest to zero.
shape = list(data.shape)
if axis is None:
shape[0] += 1
else:
shape[axis] += 1
# Mimic cumsum's behavior with the dtype argument: use the original data type or
# the system's native word, whichever has the greater width. (This prevents us from
# attempting a cumulative sum using an 8-bit integer, for instance.)
if dtype is None:
dtype = np.promote_types(data.dtype, np.min_scalar_type(-sys.maxint))
out = np.zeros(shape, dtype)
s = axis_slice(axis)
np.cumsum(data, axis, dtype, out[s[1:]])
return out
def start(self, sess, summary_manager=None):
super(ExperienceReplay, self).start(sess, summary_manager)
self._latest_observation = self._env.reset()
if self._experience is not None:
# Experience was restored.
return
obs_type = self._latest_observation.dtype
obs_shape = self._latest_observation.shape
if isinstance(self._env.action_space, spaces.Discrete):
act_type = np.min_scalar_type(self._env.action_space.n)
act_shape = tuple()
else:
act_type = self._env.action_space.sample().dtype
act_shape = self._env.action_space.shape
self._experience = Experience(obs_shape, obs_type, act_shape, act_type,
self._experience_size)
for _ in range(self._experience_start_size):
obs = self._latest_observation
action = self.action_space.sample()
self._latest_observation, reward, done, _ = self._env.step(action)
self._experience.put(obs, action, reward, done)
if done:
self._latest_observation = self._env.reset()
def mask(self, new_mask):
try:
assert (np.min_scalar_type(new_mask) <= np.dtype(np.bool8))
except:
raise TypeError('`mask` must be of boolean type')
new_mask = np.asarray(new_mask).astype(np.bool8)
self._mask = new_mask
def da_sub(daa, dab):
"""
subtract 2 DataArrays as cleverly as possible:
* keep the metadata of the first DA in the result
* ensures the result has the right type so that no underflows happen
returns (DataArray): the result of daa - dab
"""
rt = numpy.result_type(daa, dab) # dtype of result of daa-dab
dt = None # default is to let numpy decide
if rt.kind == "f":
# float should always be fine
pass
elif rt.kind in "iub":
# underflow can happen (especially if unsigned)
# find the worse case value (could be improved, but would be longer)
worse_val = int(daa.min()) - int(dab.max())
dt = numpy.result_type(rt, numpy.min_scalar_type(worse_val))
else:
# subtracting such a data is suspicious, but try anyway
logging.warning("Subtraction on data of type %s unsupported", rt.name)
res = numpy.subtract(daa, dab, dtype=dt) # metadata is copied from daa
logging.error("type = %s , %s", res.dtype.name, daa.dtype.name)
return res
def count_over_time_interval(
time,
values,
time_interval,
ignore_nodata,
nodata=None):
def aggregate_ignore_nodata(
values):
return (numpy.ones(values.shape[1:], dtype=numpy.uint8) *
values.shape[0]).tolist()
def aggregate_dont_ignore_nodata(
values):
return numpy.sum(values != nodata, 0)
aggregate = {
mds.constants.IGNORE_NODATA: aggregate_ignore_nodata,
mds.constants.DONT_IGNORE_NODATA: aggregate_dont_ignore_nodata
}
result_time, result_values = aggregate_over_time_interval(time, values,
TIME_POINT_TO_ID_BY_TIME_INTERVAL[time_interval],
aggregate[ignore_nodata])
return result_time, [numpy.array(result_values[i], numpy.min_scalar_type(
numpy.max(result_values[i]))) for i in xrange(len(result_values))]
def go(self):
pi = self.progress.indicator
pi.operation = 'Initializing'
with pi:
self.duration = self.kinetics_file['durations'][self.iter_start-1:self.iter_stop-1]
##Only select transition events from specified istate to fstate
mask = (self.duration['istate'] == self.istate) & (self.duration['fstate'] == self.fstate)
self.duration_dsspec = DurationDataset(self.kinetics_file['durations']['duration'], mask, self.iter_start)
self.wt_dsspec = DurationDataset(self.kinetics_file['durations']['weight'], mask, self.iter_start)
self.output_file = h5py.File(self.output_filename, 'w')
h5io.stamp_creator_data(self.output_file)
# Construct bin boundaries
self.construct_bins(self.parse_binspec(self.binspec))
for idim, (binbounds, midpoints) in enumerate(izip(self.binbounds, self.midpoints)):
self.output_file['binbounds_{}'.format(idim)] = binbounds
self.output_file['midpoints_{}'.format(idim)] = midpoints
# construct histogram
self.construct_histogram()
# Record iteration range
iter_range = numpy.arange(self.iter_start, self.iter_stop, 1, dtype=(numpy.min_scalar_type(self.iter_stop)))
self.output_file['n_iter'] = iter_range
self.output_file['histograms'].attrs['iter_start'] = self.iter_start
self.output_file['histograms'].attrs['iter_stop'] = self.iter_stop
self.output_file.close()
def _make_matrix(self,op_list,dtype): """ takes list of operator strings and couplings to create matrix.""" off_diag = None diag = None for opstr,indx,J in op_list: ME,row,col = self.Op(opstr,indx,J,dtype) if(len(ME)>0): imax = max(row.max(),col.max()) index_type = _np.result_type(_np.int32,_np.min_scalar_type(imax)) row = row.astype(index_type) col = col.astype(index_type) if _is_diagonal(row,col): if diag is None: diag = _np.zeros(self.Ns,dtype=dtype) _update_diag(diag,row,ME) else: if off_diag is None: off_diag = _sp.csr_matrix((ME,(row,col)),shape=(self.Ns,self.Ns),dtype=dtype) else: off_diag = off_diag + _sp.csr_matrix((ME,(row,col)),shape=(self.Ns,self.Ns),dtype=dtype) if diag is not None and off_diag is not None: indptr = _np.arange(self.Ns+1) return off_diag + _sp.csr_matrix((diag,indptr[:self.Ns],indptr),shape=(self.Ns,self.Ns),dtype=dtype) elif off_diag is not None: return off_diag elif diag is not None: return _sp.dia_matrix((_np.atleast_2d(diag),[0]),shape=(self.Ns,self.Ns),dtype=dtype) else: return _sp.dia_matrix((self.Ns,self.Ns),dtype=dtype)
def __init__(self, num_values, dtype=np.int32, name=None):
"""Initializes a new `DiscreteArray` spec.
Args:
num_values: Integer specifying the number of possible values to represent.
dtype: The dtype of the array. Must be an integral type large enough to
hold `num_values` without overflow.
name: Optional string specifying the name of the array.
Raises:
ValueError: If `num_values` is not positive, if `dtype` is not integral,
or if `dtype` is not large enough to hold `num_values` without overflow.
"""
if num_values <= 0 or not np.issubdtype(type(num_values), np.integer):
raise ValueError(_NUM_VALUES_NOT_POSITIVE.format(num_values))
if not np.issubdtype(dtype, np.integer):
raise ValueError(_DTYPE_NOT_INTEGRAL.format(dtype))
num_values = int(num_values)
maximum = num_values - 1
dtype = np.dtype(dtype)
if np.min_scalar_type(maximum) > dtype:
raise ValueError(_DTYPE_OVERFLOW.format(dtype, num_values))
super(DiscreteArray, self).__init__(shape=(),
dtype=dtype,
minimum=0,
maximum=maximum,
name=name)
self._num_values = num_values
def da_sub(daa, dab):
"""
subtract 2 DataArrays as cleverly as possible:
* keep the metadata of the first DA in the result
* ensures the result has the right type so that no underflows happen
returns (DataArray): the result of daa - dab
"""
rt = numpy.result_type(daa, dab) # dtype of result of daa-dab
dt = None # default is to let numpy decide
if rt.kind == "f":
# float should always be fine
pass
elif rt.kind in "iub":
# underflow can happen (especially if unsigned)
# find the worse case value (could be improved, but would be longer)
worse_val = int(daa.min()) - int(dab.max())
dt = numpy.result_type(rt, numpy.min_scalar_type(worse_val))
else:
# subtracting such a data is suspicious, but try anyway
logging.warning("Subtraction on data of type %s unsupported", rt.name)
res = numpy.subtract(daa, dab, dtype=dt) # metadata is copied from daa
logging.debug("type = %s, %s", res.dtype.name, daa.dtype.name)
return res
def __getitem__(self, key): """ Look up a cell in the map, but clip to the edges For instance, `map[-1, -1] == map[0, 0]`, unlike in a normal np array where it would be `map[map.shape[0]-1, map.shape[1]-1]` Note that this only applies for pairwise integer indexing. Indexing with boolean masks or slice objects uses the normal indexing rules. """ if not isinstance(key, tuple): # probably a mask? return self.grid[key] if len(key) != 2: # row, column, or just wrong return self.grid[key] if any(np.min_scalar_type(i) == np.bool for i in key): # partial mask return self.grid[key] if any(isinstance(i, slice) for i in key): # normal slicing return self.grid[key] keys = np.ravel_multi_index(key, dims=self.grid.shape, mode='clip') # workaround for https://github.com/numpy/numpy/pull/7586 if keys.ndim == 0: return self.grid.take(keys[np.newaxis])[0] else: return self.grid.take(keys)
def normalize_binop_value(
self, other: ScalarLike
) -> Union[ColumnBase, ScalarLike]:
if other is None:
return other
if isinstance(other, cudf.Scalar):
if self.dtype == other.dtype:
return other
# expensive device-host transfer just to
# adjust the dtype
other = other.value
elif isinstance(other, np.ndarray) and other.ndim == 0:
other = other.item()
other_dtype = np.min_scalar_type(other)
if other_dtype.kind in {"b", "i", "u", "f"}:
if isinstance(other, cudf.Scalar):
return other
other_dtype = np.promote_types(self.dtype, other_dtype)
if other_dtype == np.dtype("float16"):
other_dtype = np.dtype("float32")
other = other_dtype.type(other)
if self.dtype.kind == "b":
other_dtype = min_signed_type(other)
if np.isscalar(other):
other = np.dtype(other_dtype).type(other)
return other
else:
ary = utils.scalar_broadcast_to(
other, size=len(self), dtype=other_dtype
)
return column.build_column(
data=Buffer(ary), dtype=ary.dtype, mask=self.mask,
)
else:
raise TypeError(f"cannot broadcast {type(other)}")
def __init__(self, func, nbins, args=None, kwargs=None):
self.func = func
self.args = args or ()
self.kwargs = kwargs or {}
self.nbins = nbins
self.index_dtype = numpy.min_scalar_type(self.nbins)
self.labels = ['{!r} bin {:d}'.format(func, ibin) for ibin in xrange(nbins)]
def __init__(
self,
*,
max_node_iloc,
source_ilocs,
rel_ilocs,
target_ilocs,
batch_size,
shuffle,
negative_samples,
sample_strategy,
seed,
):
self.max_node_iloc = max_node_iloc
num_edges = len(source_ilocs)
self.indices = np.arange(num_edges,
dtype=np.min_scalar_type(num_edges))
self.source_ilocs = np.asarray(source_ilocs)
self.rel_ilocs = np.asarray(rel_ilocs)
self.target_ilocs = np.asarray(target_ilocs)
self.negative_samples = negative_samples
self.sample_strategy = sample_strategy
self.batch_size = batch_size
self.seed = seed
self.shuffle = shuffle
_, self._global_rs = random_state(seed)
self._batch_sampler = SeededPerBatch(
np.random.RandomState,
self._global_rs.randint(2**32, dtype=np.uint32))
def _check_and_cast_columns_with_other(
source_col: ColumnBase,
other: Union[ScalarLike, ColumnBase],
inplace: bool,
) -> Tuple[ColumnBase, Union[ScalarLike, ColumnBase]]:
"""
Returns type-casted column `source_col` & scalar `other_scalar`
based on `inplace` parameter.
"""
if cudf.utils.dtypes.is_categorical_dtype(source_col.dtype):
return source_col, other
if cudf.utils.dtypes.is_scalar(other):
device_obj = _normalize_scalars(source_col, other)
else:
device_obj = other
if other is None:
return source_col, device_obj
elif cudf.utils.dtypes.is_mixed_with_object_dtype(device_obj, source_col):
raise TypeError(
"cudf does not support mixed types, please type-cast "
"the column of dataframe/series and other "
"to same dtypes."
)
if inplace:
if not cudf.utils.dtypes._can_cast(device_obj.dtype, source_col.dtype):
warnings.warn(
f"Type-casting from {device_obj.dtype} "
f"to {source_col.dtype}, there could be potential data loss"
)
return source_col, device_obj.astype(source_col.dtype)
else:
if (
cudf.utils.dtypes.is_scalar(other)
and cudf.utils.dtypes._is_non_decimal_numeric_dtype(
source_col.dtype
)
and cudf.utils.dtypes._can_cast(other, source_col.dtype)
):
common_dtype = source_col.dtype
return (
source_col.astype(common_dtype),
cudf.Scalar(other, dtype=common_dtype),
)
else:
common_dtype = cudf.utils.dtypes.find_common_type(
[
source_col.dtype,
np.min_scalar_type(other)
if cudf.utils.dtypes.is_scalar(other)
else other.dtype,
]
)
if cudf.utils.dtypes.is_scalar(device_obj):
device_obj = cudf.Scalar(other, dtype=common_dtype)
else:
device_obj = device_obj.astype(common_dtype)
return source_col.astype(common_dtype), device_obj
def _from_coo(x, compressed_axes=None, idx_dtype=None):
if x.ndim == 0:
if compressed_axes is not None:
raise ValueError("no axes to compress for 0d array")
return ((x.data, x.coords, []), x.shape, None, x.fill_value)
if x.ndim == 1:
if compressed_axes is not None:
raise ValueError("no axes to compress for 1d array")
return ((x.data, x.coords[0], ()), x.shape, None, x.fill_value)
compressed_axes = normalize_axis(compressed_axes, x.ndim)
if compressed_axes is None:
# defaults to best compression ratio
compressed_axes = (np.argmin(x.shape), )
check_compressed_axes(x.shape, compressed_axes)
axis_order = list(compressed_axes)
# array location where the uncompressed dimensions start
axisptr = len(compressed_axes)
axis_order.extend(np.setdiff1d(np.arange(len(x.shape)), compressed_axes))
reordered_shape = tuple(x.shape[i] for i in axis_order)
row_size = np.prod(reordered_shape[:axisptr])
col_size = np.prod(reordered_shape[axisptr:])
compressed_shape = (row_size, col_size)
shape = x.shape
if idx_dtype and not can_store(idx_dtype, max(max(compressed_shape),
x.nnz)):
raise ValueError(
"cannot store array with the compressed shape {} and nnz {} with dtype {}."
.format(
compressed_shape,
x.nnz,
idx_dtype,
))
if not idx_dtype:
idx_dtype = x.coords.dtype
if not can_store(idx_dtype, max(max(compressed_shape), x.nnz)):
idx_dtype = np.min_scalar_type(max(max(compressed_shape), x.nnz))
# transpose axes, linearize, reshape, and compress
linear = linear_loc(x.coords[axis_order], reordered_shape)
order = np.argsort(linear)
linear = linear[order]
coords = np.empty((2, x.nnz), dtype=idx_dtype)
strides = 1
for i, d in enumerate(compressed_shape[::-1]):
coords[-(i + 1), :] = (linear // strides) % d
strides *= d
indptr = np.empty(row_size + 1, dtype=idx_dtype)
indptr[0] = 0
np.cumsum(np.bincount(coords[0], minlength=row_size), out=indptr[1:])
indices = coords[1]
data = x.data[order]
return ((data, indices, indptr), shape, compressed_axes, x.fill_value)
def neighbor_pairs(self, unique=False):
"""Returns all neighbor pairs with their corresponding distances in the lattice.
Parameters
----------
unique : bool, optional
If True, only unique pairs with i < j are returned. The default is False.
Returns
-------
pairs : (N, 2) np.ndarray
An array containing all neighbor pairs of the lattice. If `unique=True`,
the first index is always smaller than the second index in each element.
distindices : (N, ) np.ndarray
The corresponding distance indices of the neighbor pairs.
Examples
--------
>>> latt = Lattice.chain()
>>> latt.add_atom(neighbors=1)
>>> latt.build(5)
>>> idx, distidx = latt.neighbor_pairs()
>>> idx
array([[0, 1],
[1, 2],
[1, 0],
[2, 3],
[2, 1],
[3, 2]], dtype=uint8)
>>> distidx
array([0, 0, 0, 0, 0, 0], dtype=uint8)
>>> idx, distidx = latt.neighbor_pairs(unique=True)
>>> idx
array([[0, 1],
[1, 2],
[2, 3]], dtype=uint8)
"""
# Build index pairs and corresponding distance array
dtype = np.min_scalar_type(self.num_sites)
sites = np.arange(self.num_sites, dtype=dtype)
sites_t = np.tile(sites, (self.data.neighbors.shape[1], 1)).T
pairs = np.reshape([sites_t, self.data.neighbors], newshape=(2, -1)).T
distindices = self.data.distances.flatten()
# Filter pairs with invalid indices
mask = distindices != self.data.invalid_distidx
pairs = pairs[mask]
distindices = distindices[mask]
if unique:
# Filter for unique pairs (i < j)
mask = pairs[:, 0] < pairs[:, 1]
pairs = pairs[mask]
distindices = distindices[mask]
return pairs, distindices
def __init__(self,Np):
if (type(Np) is not int):
raise ValueError("expecting integer for Np")
self._Np = Np
self._Ns = Np+1
self._N = 1
self._dtype = _np.min_scalar_type(-self.Ns)
self._basis = _np.arange(self.Ns,dtype=_np.min_scalar_type(self.Ns))
self._operators = ("availible operators for ho_basis:"+
"\n\tI: identity "+
"\n\t+: raising operator"+
"\n\t-: lowering operator"+
"\n\tn: number operator")
self._blocks = {}
self._unique_me = True
def concatenate(arrays, axis=0):
"""
Concatenate the input arrays along the given dimension.
Parameters
----------
arrays : Iterable[SparseArray]
The input arrays to concatenate.
axis : int, optional
The axis along which to concatenate the input arrays. The default is zero.
Returns
-------
COO
The output concatenated array.
Raises
------
ValueError
If all elements of :code:`arrays` don't have the same fill-value.
See Also
--------
numpy.concatenate : NumPy equivalent function
"""
from .core import COO
check_consistent_fill_value(arrays)
arrays = [x if isinstance(x, COO) else COO(x) for x in arrays]
axis = normalize_axis(axis, arrays[0].ndim)
assert all(x.shape[ax] == arrays[0].shape[ax] for x in arrays
for ax in set(range(arrays[0].ndim)) - {axis})
nnz = 0
dim = sum(x.shape[axis] for x in arrays)
shape = list(arrays[0].shape)
shape[axis] = dim
data = np.concatenate([x.data for x in arrays])
coords = np.concatenate([x.coords for x in arrays], axis=1)
if not can_store(coords.dtype, max(shape)):
coords = coords.astype(np.min_scalar_type(max(shape)))
dim = 0
for x in arrays:
if dim:
coords[axis, nnz:x.nnz + nnz] += dim
dim += x.shape[axis]
nnz += x.nnz
return COO(
coords,
data,
shape=shape,
has_duplicates=False,
sorted=(axis == 0),
fill_value=arrays[0].fill_value,
)
def render(self):
# Convert data to QImage for display.
profile = debug.Profiler()
if self.image is None or self.image.size == 0:
return
if isinstance(self.lut, collections.Callable):
lut = self.lut(self.image)
else:
lut = self.lut
if self.autoDownsample:
# reduce dimensions of image based on screen resolution
o = self.mapToDevice(QtCore.QPointF(0, 0))
x = self.mapToDevice(QtCore.QPointF(1, 0))
y = self.mapToDevice(QtCore.QPointF(0, 1))
w = Point(x - o).length()
h = Point(y - o).length()
xds = int(1 / max(1, w))
yds = int(1 / max(1, h))
image = fn.downsample(self.image, xds, axis=0)
image = fn.downsample(image, yds, axis=1)
else:
image = self.image
# if the image data is a small int, then we can combine levels + lut
# into a single lut for better performance
levels = self.levels
if levels is not None and levels.ndim == 1 and image.dtype in (
np.ubyte, np.uint16):
if self._effectiveLut is None:
eflsize = 2**(image.itemsize * 8)
ind = np.arange(eflsize)
minlev, maxlev = levels
if lut is None:
efflut = fn.rescaleData(ind,
scale=255. / (maxlev - minlev),
offset=minlev,
dtype=np.ubyte)
else:
lutdtype = np.min_scalar_type(lut.shape[0] - 1)
efflut = fn.rescaleData(ind,
scale=(lut.shape[0] - 1) /
(maxlev - minlev),
offset=minlev,
dtype=lutdtype,
clip=(0, lut.shape[0] - 1))
efflut = lut[efflut]
self._effectiveLut = efflut
lut = self._effectiveLut
levels = None
argb, alpha = fn.makeARGB(image.transpose((1, 0, 2)[:image.ndim]),
lut=lut,
levels=levels)
self.qimage = fn.makeQImage(argb, alpha, transpose=False)
def main():
args = get_parser().parse_args()
label_images = (safeintload(file_) for file_ in args.input_files)
out = reduce(combine2labels, label_images)
# reduce to smallest type possible
out = out.astype(np.min_scalar_type(out.max()), copy=False)
outnifti = nibabel.Nifti1Image(out, None)
copy_forms(nibabel.load(args.input_files[0]), outnifti)
outnifti.to_filename(args.output_file)
def normalize_compare_value(self, other):
other_dtype = np.min_scalar_type(other)
if other_dtype.kind in 'biuf':
other_dtype = np.promote_types(self.dtype, other_dtype)
ary = utils.scalar_broadcast_to(other, shape=len(self),
dtype=other_dtype)
return self.replace(data=Buffer(ary), dtype=ary.dtype)
else:
raise TypeError('cannot broadcast {}'.format(type(other)))
def normalize_binop_value(self, other):
other_dtype = np.min_scalar_type(other)
if other_dtype.kind in 'biuf':
other_dtype = np.promote_types(self.dtype, other_dtype)
ary = utils.scalar_broadcast_to(other, shape=len(self),
dtype=other_dtype)
return self.replace(data=Buffer(ary), dtype=ary.dtype)
else:
raise TypeError('cannot broadcast {}'.format(type(other)))
def _convert_scalar_value(self, value):
"""Convert a string into a numpy int or float.
If it is not possible it returns a numpy string.
"""
try:
value = int(value)
dtype = numpy.min_scalar_type(value)
assert dtype.kind != "O"
return dtype.type(value)
except ValueError:
try:
value = float(value)
dtype = numpy.min_scalar_type(value)
assert dtype.kind != "O"
return dtype.type(value)
except ValueError:
return numpy.string_(value)
def _Constant(t, symbols, inferred_symbols):
# String value
if isinstance(t.value, (str, bytes)):
return dtypes.pointer(dtypes.int8)
# Numeric value
return dtypes.result_type_of(
dtypes.typeclass(type(t.value)),
dtypes.typeclass(np.min_scalar_type(t.value).name))
def iter_range(self, iter_start=None, iter_stop=None, iter_step=None, dtype=None):
'''Return a sequence for the given iteration numbers and stride, filling
in missing values from those stored on ``self``. The smallest data type capable of
holding ``iter_stop`` is returned unless otherwise specified using the ``dtype``
argument.'''
iter_start = self.iter_start if iter_start is None else iter_start
iter_stop = self.iter_stop if iter_stop is None else iter_stop
iter_step = self.iter_step if iter_step is None else iter_step
return np.arange(iter_start, iter_stop, iter_step, dtype=(dtype or np.min_scalar_type(iter_stop)))
def check_ME(basis_1d, basis_gen, opstr, indx, dtype, err_msg):
ME1, row1, col1 = basis_1d.Op(opstr, indx, 1.0, dtype)
ME2, row2, col2 = basis_gen.Op(opstr, indx, 1.0, dtype)
if len(ME1) != len(ME2):
print(opstr, list(indx))
print(basis_1d)
print("spin_basis_1d:")
print(ME1)
print(row1)
print(col1)
print()
print("spin_basis_general")
print(ME2)
print(row2)
print(col2)
raise Exception("number of matrix elements do not match.")
if len(ME1) > 0 and len(ME2) > 0:
row1 = row1.astype(np.min_scalar_type(row1.max()))
row2 = row2.astype(np.min_scalar_type(row2.max()))
col1 = row2.astype(np.min_scalar_type(col1.max()))
col2 = row2.astype(np.min_scalar_type(col2.max()))
try:
np.testing.assert_allclose(row1, row2, atol=1e-6, err_msg=err_msg)
np.testing.assert_allclose(col1, col2, atol=1e-6, err_msg=err_msg)
np.testing.assert_allclose(ME1, ME2, atol=1e-6, err_msg=err_msg)
except AssertionError:
print(err_msg)
print(basis_1d)
print("difference:")
print(ME1 - ME2)
print(row1 - row2)
print(col1 - col2)
print("spin_basis_1d:")
print(ME1)
print(row1)
print(col1)
print("spin_basis_general")
print(ME2)
print(row2)
print(col2)
raise Exception
def get_proj(self,dtype,pcon=False):
"""Calculates transformation/projector from symmetry-reduced basis to full (symmetry-free) basis.
Notes
-----
* particularly useful when a given operation canot be carried out in the symmetry-reduced basis
in a straightforward manner.
* see also `Op_shift_sector()`.
Parameters
-----------
dtype : 'type'
Data type (e.g. numpy.float64) to construct the projector with.
pcon : bool, optional
Whether or not to return the projector to the particle number (magnetisation) conserving basis
(useful in bosonic/single particle systems). Default is `pcon=False`.
Returns
--------
scipy.sparse.csc_matrix
Transformation/projector between the symmetry-reduced and the full basis.
Examples
--------
>>> P = get_proj(np.float64,pcon=False)
>>> print(P.shape)
"""
if not self._made_basis:
raise AttributeError('this function requires the basis to be constructed first; use basis.make().')
basis_pcon = None
Ns_full = (self._sps**self._N)
if pcon and self._get_proj_pcon:
if self._basis_pcon is None:
self._basis_pcon = self.__class__(**self._pcon_args)
basis_pcon = self._basis_pcon._basis
Ns_full = basis_pcon.shape[0]
elif pcon and self._get_proj_pcon:
raise TypeError("pcon=True only works for basis of a single particle number sector.")
sign = _np.ones_like(self._basis,dtype=_np.int8)
c = self._n.astype(dtype,copy=True)
c *= self._pers.prod()
_np.sqrt(c,out=c)
_np.power(c,-1,out=c)
index_type = _np.result_type(_np.min_scalar_type(-Ns_full),_np.int32)
indptr = _np.arange(self._Ns+1,dtype=index_type)
indices = _np.arange(self._Ns,dtype=index_type)
return self._core.get_proj(self._basis,dtype,sign,c,indices,indptr,basis_pcon=basis_pcon)
def _elemwise_n_ary(func, *args, **kwargs):
"""
Apply a function to any number of arguments with broadcasting.
Parameters
----------
func : Callable
The function to apply to arguments. Must support broadcasting.
args : list
Input :obj:`COO` or :obj:`numpy.ndarray`s.
kwargs : dict
Additional arguments to pass to the function.
Returns
-------
COO
The output array.
Raises
------
ValueError
If the input shapes aren't compatible or the result will be dense.
"""
from .core import COO
args = list(args)
args_zeros = tuple(_zero_of_dtype(np.dtype(arg)) for arg in args)
func_value = func(*args_zeros, **kwargs)
func_zero = _zero_of_dtype(func_value.dtype)
if func_value != func_zero:
raise ValueError("Performing this operation would produce "
"a dense result: %s" % str(func))
data_list = []
coords_list = []
cache = {}
for mask in product([True, False], repeat=len(args)):
if not any(mask):
continue
ci, di = _unmatch_coo(func, args, mask, cache, **kwargs)
coords_list.extend(ci)
data_list.extend(di)
result_shape = _get_nary_broadcast_shape(*[arg.shape for arg in args])
# Concatenate matches and mismatches
data = np.concatenate(data_list) if len(data_list) else np.empty(
(0, ), dtype=func_value.dtype)
coords = np.concatenate(coords_list, axis=1) if len(coords_list) else \
np.empty((0, len(result_shape)), dtype=np.min_scalar_type(max(result_shape) - 1))
return COO(coords, data, shape=result_shape, has_duplicates=False)
def __init__(self, ids):
self._index = pd.Index(ids)
self._dtype = np.min_scalar_type(len(self._index))
if not self._index.is_unique:
# had some duplicated IDs, which is an error
duplicated = self._index[self._index.duplicated()].unique()
raise ValueError(
f"expected IDs to appear once, found some that appeared more: {comma_sep(duplicated)}"
)
def __init__(self, *args):
multiples = [np.unique(ids) for ids in args]
unique = np.concat(multiples)
if len(multiples) > 1:
unique = np.unique(unique)
unique.sort()
self._index = pd.Index(unique)
self._dtype = np.min_scalar_type(len(self._index))
def main():
args = get_parser().parse_args()
label_map = parse_mapstr(args.map)
x = nibabel.load(args.input)
xdata = np.asarray(x.dataobj)
ydata = np.zeros(xdata.shape, np.min_scalar_type(max(label_map.values())))
for key, val in label_map.items():
ydata[xdata == key] = val
y = nibabel.Nifti1Image(ydata, None)
copy_forms(x, y)
y.to_filename(args.output)
def safely_reduce_dtype(ser): # pandas.Series or numpy.array
# reduces the datatype to the lowest possible to reduce storage.
orig_dtype = "".join([x for x in ser.dtype.name
if x.isalpha()]) # float/int
mx = 1
new_itemsize = np.min_scalar_type(ser).itemsize
if mx < new_itemsize:
mx = new_itemsize
new_dtype = orig_dtype + str(mx * 8)
return new_dtype
def _init_undirected_adj_lists(self):
# record the edge ilocs of incoming, outgoing and both-direction edges
undirected = {}
for i, (src, tgt) in enumerate(zip(self.sources, self.targets)):
undirected.setdefault(tgt, []).append(i)
if src != tgt:
undirected.setdefault(src, []).append(i)
dtype = np.min_scalar_type(len(self.sources))
self._edges_dict = _numpyise(undirected, dtype=dtype)
def iter_range(self, iter_start = None, iter_stop = None, iter_step = None, dtype=None):
'''Return a sequence for the given iteration numbers and stride, filling
in missing values from those stored on ``self``. The smallest data type capable of
holding ``iter_stop`` is returned unless otherwise specified using the ``dtype``
argument.'''
iter_start = self.iter_start if iter_start is None else iter_start
iter_stop = self.iter_stop if iter_stop is None else iter_stop
iter_step = self.iter_step if iter_step is None else iter_step
return numpy.arange(iter_start, iter_stop, iter_step, dtype=(dtype or numpy.min_scalar_type(iter_stop)))
def create(predict_fn, word_representations,
batch_size, window_size, vocabulary_size,
result_callback):
assert result_callback is not None
instance_dtype = np.min_scalar_type(vocabulary_size - 1)
logging.info('Instance elements will be stored using %s.', instance_dtype)
batcher = WordBatcher(
predict_fn,
batch_size, window_size, instance_dtype,
result_callback)
return batcher
def shuffle_group(df, col, stage, k, npartitions):
if col == '_partitions':
ind = df[col]
else:
ind = hash_pandas_object(df[col], index=False)
c = ind._values
typ = np.min_scalar_type(npartitions * 2)
c = c.astype(typ)
npartitions, k, stage = [np.array(x, dtype=np.min_scalar_type(x))[()]
for x in [npartitions, k, stage]]
c = np.mod(c, npartitions, out=c)
c = np.floor_divide(c, k ** stage, out=c)
c = np.mod(c, k, out=c)
indexer, locations = pd.algos.groupsort_indexer(c.astype(np.int64), k)
df2 = df.take(indexer)
locations = locations.cumsum()
parts = [df2.iloc[a:b] for a, b in zip(locations[:-1], locations[1:])]
return dict(zip(range(k), parts))
def render(self):
# Convert data to QImage for display.
profile = debug.Profiler()
if self.image is None or self.image.size == 0:
return
if isinstance(self.lut, collections.Callable):
lut = self.lut(self.image)
else:
lut = self.lut
if self.autoDownsample:
# reduce dimensions of image based on screen resolution
o = self.mapToDevice(QtCore.QPointF(0,0))
x = self.mapToDevice(QtCore.QPointF(1,0))
y = self.mapToDevice(QtCore.QPointF(0,1))
w = Point(x-o).length()
h = Point(y-o).length()
xds = int(1/max(1, w))
yds = int(1/max(1, h))
image = fn.downsample(self.image, xds, axis=0)
image = fn.downsample(image, yds, axis=1)
else:
image = self.image
# if the image data is a small int, then we can combine levels + lut
# into a single lut for better performance
levels = self.levels
if levels is not None and levels.ndim == 1 and image.dtype in (np.ubyte, np.uint16):
if self._effectiveLut is None:
eflsize = 2**(image.itemsize*8)
ind = np.arange(eflsize)
minlev, maxlev = levels
if lut is None:
efflut = fn.rescaleData(ind, scale=255./(maxlev-minlev),
offset=minlev, dtype=np.ubyte)
else:
lutdtype = np.min_scalar_type(lut.shape[0]-1)
efflut = fn.rescaleData(ind, scale=(lut.shape[0]-1)/(maxlev-minlev),
offset=minlev, dtype=lutdtype, clip=(0, lut.shape[0]-1))
efflut = lut[efflut]
self._effectiveLut = efflut
lut = self._effectiveLut
levels = None
argb, alpha = fn.makeARGB(image.transpose((1, 0, 2)[:image.ndim]), lut=lut, levels=levels)
self.qimage = fn.makeQImage(argb, alpha, transpose=False)
def __init__(self,b1,b2):
if not isinstance(b1,basis):
raise ValueError("b1 must be instance of basis class")
if not isinstance(b2,basis):
raise ValueError("b2 must be instance of basis class")
if isinstance(b1,tensor_basis):
raise TypeError("Can only create tensor basis with non-tensor type basis")
if isinstance(b2,tensor_basis):
raise TypeError("Can only create tensor basis with non-tensor type basis")
self._b1=b1
self._b2=b2
self._Ns = b1.Ns*b2.Ns
self._dtype = _np.min_scalar_type(-self._Ns)
self._operators = self._b1._operators +"\n"+ self._b2._operators
def _get_dtype(operand):
"""
Get the numpy dtype corresponding to the numeric data in the object
provided.
Args:
* operand:
An instance of :class:`iris.cube.Cube` or :class:`iris.coords.Coord`,
or a number or :class:`numpy.ndarray`.
Returns:
An instance of :class:`numpy.dtype`
"""
return np.min_scalar_type(operand) if np.isscalar(operand) \
else operand.dtype
def __init__(self, n_bins, output_group, calc_fpts = True):
self.calc_fpts = calc_fpts
self.n_bins = n_bins
self.iibins = numpy.arange(n_bins)
self.iibdisc = numpy.empty((n_bins,), numpy.bool_)
self.bin_index_dtype = numpy.min_scalar_type(n_bins)
self.tdat_dtype = numpy.dtype( [('traj', self.index_dtype),
('n_iter', self.index_dtype),
('timepoint', self.index_dtype),
('initial_bin', self.bin_index_dtype),
('final_bin', self.bin_index_dtype),
('initial_weight', self.weight_dtype),
('final_weight', self.weight_dtype),
('initial_bin_pop', self.weight_dtype),
('duration', self.index_dtype),
('fpt', self.index_dtype),
])
# HDF5 group in which to store results
self.output_group = output_group
self.tdat_buffer = numpy.empty((self.tdat_buffersize,), dtype=self.tdat_dtype)
self.tdat_buffer_offset = 0
self.output_tdat_offset = 0
self.output_tdat_ds = None
# Accumulators/counters
self.n_trans = None # shape (n_bins,n_bins)
# Time points and per-timepoint data
self.last_exit = None # (n_bins,)
self.last_entry = None # (n_bins,)
self.last_completion = None # (n_bins,n_bins)
self.weight_last_exit = None # (n_bins)
self.bin_pops_last_exit = None # (n_bins,)
# Analysis continuation information
self.timepoint = None # current time index for separate calls on same trajectory
self.last_bin = None # last region occupied, for separate calls on same trajectory
self.last_bin_pop = None # total weight in self.last_region at end of last processing step
self.clear()
def shuffle_group(df, col, stage, k, npartitions):
""" Splits dataframe into groups
The group is determined by their final partition, and which stage we are in
in the shuffle
Parameters
----------
df: DataFrame
col: str
Column name on which to split the dataframe
stage: int
We shuffle dataframes with many partitions we in a few stages to avoid
a quadratic number of tasks. This number corresponds to which stage
we're in, starting from zero up to some small integer
k: int
Desired number of splits from this dataframe
npartition: int
Total number of output partitions for the full dataframe
Returns
-------
out: Dict[int, DataFrame]
A dictionary mapping integers in {0..k} to dataframes such that the
hash values of ``df[col]`` are well partitioned.
"""
if col == '_partitions':
ind = df[col]
else:
ind = hash_pandas_object(df[col], index=False)
c = ind._values
typ = np.min_scalar_type(npartitions * 2)
c = np.mod(c, npartitions).astype(typ, copy=False)
np.floor_divide(c, k ** stage, out=c)
np.mod(c, k, out=c)
indexer, locations = groupsort_indexer(c.astype(np.int64), k)
df2 = df.take(indexer)
locations = locations.cumsum()
parts = [df2.iloc[a:b] for a, b in zip(locations[:-1], locations[1:])]
return dict(zip(range(k), parts))
def rasterize_pctcover_geom(geom, shape, affine, scale=None, all_touched=False):
"""
"""
if scale is None:
scale = 10
min_dtype = min_scalar_type(scale**2)
pixel_size = affine[0]/scale
topleftlon = affine[2]
topleftlat = affine[5]
new_affine = Affine(pixel_size, 0, topleftlon,
0, -pixel_size, topleftlat)
new_shape = (shape[0]*scale, shape[1]*scale)
rv_array = rasterize_geom(geom, new_shape, new_affine, all_touched=all_touched)
rv_array = rebin_sum(rv_array, shape, min_dtype)
return rv_array.astype('float32') / (scale**2)
def create(predict_fn, word_representations,
batch_size, window_size, vocabulary_size,
result_callback):
assert result_callback is not None
instance_dtype = np.min_scalar_type(vocabulary_size - 1)
logging.info('Instance elements will be stored using %s.', instance_dtype)
if result_callback.should_average_input():
batcher = EmbeddingMapper(
predict_fn,
word_representations,
result_callback)
else:
batcher = WordBatcher(
predict_fn,
batch_size, window_size,
instance_dtype,
result_callback)
return batcher
def _run_interface(self, runtime):
from numpy import min_scalar_type
tracks, header = nbt.read(self.inputs.in_file)
streams = ((ii[0]) for ii in tracks)
if isdefined(self.inputs.reference):
refnii = nb.load(self.inputs.reference)
affine = refnii.get_affine()
data_dims = refnii.get_shape()[:3]
kwargs = dict(affine=affine)
else:
iflogger.warn(('voxel_dims and data_dims are deprecated'
'as of dipy 0.7.1. Please use reference '
'input instead'))
if not isdefined(self.inputs.data_dims):
data_dims = header['dim']
else:
data_dims = self.inputs.data_dims
if not isdefined(self.inputs.voxel_dims):
voxel_size = header['voxel_size']
else:
voxel_size = self.inputs.voxel_dims
affine = header['vox_to_ras']
kwargs = dict(voxel_size=voxel_size)
data = density_map(streams, data_dims, **kwargs)
data = data.astype(min_scalar_type(data.max()))
img = nb.Nifti1Image(data, affine)
out_file = op.abspath(self.inputs.out_filename)
nb.save(img, out_file)
iflogger.info(
('Track density map saved as {i}, size={d}, '
'dimensions={v}').format(
i=out_file,
d=img.get_shape(),
v=img.get_header().get_zooms()))
return runtime
def _run_interface(self, runtime):
from numpy import min_scalar_type
from dipy.tracking.utils import density_map
tracks, header = nbt.read(self.inputs.in_file)
streams = ((ii[0]) for ii in tracks)
if isdefined(self.inputs.reference):
refnii = nb.load(self.inputs.reference)
affine = refnii.affine
data_dims = refnii.shape[:3]
kwargs = dict(affine=affine)
else:
IFLOGGER.warn(
'voxel_dims and data_dims are deprecated as of dipy 0.7.1. Please use reference '
'input instead')
if not isdefined(self.inputs.data_dims):
data_dims = header['dim']
else:
data_dims = self.inputs.data_dims
if not isdefined(self.inputs.voxel_dims):
voxel_size = header['voxel_size']
else:
voxel_size = self.inputs.voxel_dims
affine = header['vox_to_ras']
kwargs = dict(voxel_size=voxel_size)
data = density_map(streams, data_dims, **kwargs)
data = data.astype(min_scalar_type(data.max()))
img = nb.Nifti1Image(data, affine)
out_file = op.abspath(self.inputs.out_filename)
nb.save(img, out_file)
IFLOGGER.info(
'Track density map saved as %s, size=%s, dimensions=%s',
out_file, img.shape, img.header.get_zooms())
return runtime
def assign_to_bins(self):
'''Assign WEST segment data to bins. Requires the DataReader mixin to be in the inheritance tree'''
self.require_binning_group()
n_iters = self.last_iter - self.first_iter + 1
max_n_segs = self.max_iter_segs_in_range(self.first_iter, self.last_iter)
pcoord_len = self.get_pcoord_len(self.first_iter)
assignments = numpy.zeros((n_iters, max_n_segs,pcoord_len), numpy.min_scalar_type(self.n_bins))
populations = numpy.zeros((n_iters, pcoord_len, self.n_bins), numpy.float64)
westpa.rc.pstatus('Assigning to bins...')
for (iiter, n_iter) in enumerate(xrange(self.first_iter, self.last_iter+1)):
westpa.rc.pstatus('\r Iteration {:d}'.format(n_iter), end='')
seg_index = self.get_seg_index(n_iter)
pcoords = self.get_iter_group(n_iter)['pcoord'][...]
weights = seg_index['weight']
for seg_id in xrange(len(seg_index)):
assignments[iiter,seg_id,:] = self.mapper.assign(pcoords[seg_id,:,:])
for it in xrange(pcoord_len):
populations[iiter, it, :] = numpy.bincount(assignments[iiter,:len(seg_index),it], weights, minlength=self.n_bins)
westpa.rc.pflush()
del pcoords, weights, seg_index
assignments_ds = self.binning_h5group.create_dataset('bin_assignments', data=assignments, compression='gzip')
populations_ds = self.binning_h5group.create_dataset('bin_populations', data=populations, compression='gzip')
for h5object in (self.binning_h5group, assignments_ds, populations_ds):
self.record_data_iter_range(h5object)
self.record_data_iter_step(h5object, 1)
self.record_data_binhash(h5object)
westpa.rc.pstatus()
def model(filename, format='ascii'):
"""Read an IMOD model file into a numpy array.
Initially only ASCII input files are supported.
Parameters
----------
filename : string
The filename to read.
format : string in {'ascii'}, optional
Whether the file is binary or ASCII format.
Returns
-------
mod : array of int
A numpy array of the same size as the minimal bounding box
containing the model.
offset : 3-tuple of int
The top-left corner of the bounding box of the model.
"""
object_dict = model2coords(filename, format=format)
offset = np.zeros((3,), dtype=int)
limits = np.zeros((3,), dtype=int)
max_object_id = max(object_dict.keys())
for value in object_dict:
coords = np.round(object_dict[value]['coords']).astype(int)
new_offset = np.min(coords, axis=1)
offset = np.minimum(offset, new_offset, out=offset)
new_limits = np.max(coords, axis=1)
limits = np.maximum(limits, new_limits, out=limits)
mod = np.zeros(limits + 1, dtype=np.min_scalar_type(max_object_id))
for value in object_dict:
coords = np.round(object_dict[value]['coords']).astype(int)
coords -= offset[np.newaxis,]
mod[tuple(coords)] = value
return mod, offset
def label2rgb(label, image=None, colors=None, alpha=0.3,
bg_label=-1, bg_color=None, image_alpha=1):
"""Return an RGB image where color-coded labels are painted over the image.
Parameters
----------
label : array
Integer array of labels with the same shape as `image`.
image : array
Image used as underlay for labels. If the input is an RGB image, it's
converted to grayscale before coloring.
colors : list
List of colors. If the number of labels exceeds the number of colors,
then the colors are cycled.
alpha : float [0, 1]
Opacity of colorized labels. Ignored if image is `None`.
bg_label : int
Label that's treated as the background.
bg_color : str or array
Background color. Must be a name in `color_dict` or RGB float values
between [0, 1].
image_alpha : float [0, 1]
Opacity of the image.
"""
if colors is None:
colors = DEFAULT_COLORS
colors = [_rgb_vector(c) for c in colors]
if image is None:
image = np.zeros(label.shape + (3,), dtype=np.float64)
# Opacity doesn't make sense if no image exists.
alpha = 1
else:
if not image.shape[:2] == label.shape:
raise ValueError("`image` and `label` must be the same shape")
if image.min() < 0:
warnings.warn("Negative intensities in `image` are not supported")
image = img_as_float(rgb2gray(image))
image = gray2rgb(image) * image_alpha + (1 - image_alpha)
# Ensure that all labels are non-negative so we can index into
# `label_to_color` correctly.
offset = min(label.min(), bg_label)
if offset != 0:
label = label - offset # Make sure you don't modify the input array.
bg_label -= offset
new_type = np.min_scalar_type(label.max())
if new_type == np.bool:
new_type = np.uint8
label = label.astype(new_type)
unique_labels, color_cycle = _match_label_with_color(label, colors,
bg_label, bg_color)
if len(unique_labels) == 0:
return image
dense_labels = range(max(unique_labels) + 1)
label_to_color = np.array([c for i, c in zip(dense_labels, color_cycle)])
result = label_to_color[label] * alpha + image * (1 - alpha)
# Remove background label if its color was not specified.
remove_background = bg_label in unique_labels and bg_color is None
if remove_background:
result[label == bg_label] = image[label == bg_label]
return result
def _run_interface(self, runtime):
nii1 = nb.load(self.inputs.volume1)
nii2 = nb.load(self.inputs.volume2)
scale = 1.0
if self.inputs.vol_units == 'mm':
voxvol = nii1.header.get_zooms()
for i in range(nii1.get_data().ndim - 1):
scale = scale * voxvol[i]
data1 = nii1.get_data()
data1[np.logical_or(data1 < 0, np.isnan(data1))] = 0
max1 = int(data1.max())
data1 = data1.astype(np.min_scalar_type(max1))
data2 = nii2.get_data().astype(np.min_scalar_type(max1))
data2[np.logical_or(data1 < 0, np.isnan(data1))] = 0
max2 = data2.max()
maxlabel = max(max1, max2)
if isdefined(self.inputs.mask_volume):
maskdata = nb.load(self.inputs.mask_volume).get_data()
maskdata = ~np.logical_or(maskdata == 0, np.isnan(maskdata))
data1[~maskdata] = 0
data2[~maskdata] = 0
res = []
volumes1 = []
volumes2 = []
labels = np.unique(data1[data1 > 0].reshape(-1)).tolist()
if self.inputs.bg_overlap:
labels.insert(0, 0)
for l in labels:
res.append(self._bool_vec_dissimilarity(data1 == l,
data2 == l, method='jaccard'))
volumes1.append(scale * len(data1[data1 == l]))
volumes2.append(scale * len(data2[data2 == l]))
results = dict(jaccard=[], dice=[])
results['jaccard'] = np.array(res)
results['dice'] = 2.0 * results['jaccard'] / (results['jaccard'] + 1.0)
weights = np.ones((len(volumes1),), dtype=np.float32)
if self.inputs.weighting != 'none':
weights = weights / np.array(volumes1)
if self.inputs.weighting == 'squared_vol':
weights = weights**2
weights = weights / np.sum(weights)
both_data = np.zeros(data1.shape)
both_data[(data1 - data2) != 0] = 1
nb.save(nb.Nifti1Image(both_data, nii1.affine, nii1.header),
self.inputs.out_file)
self._labels = labels
self._ove_rois = results
self._vol_rois = (np.array(volumes1) -
np.array(volumes2)) / np.array(volumes1)
self._dice = round(np.sum(weights * results['dice']), 5)
self._jaccard = round(np.sum(weights * results['jaccard']), 5)
self._volume = np.sum(weights * self._vol_rois)
return runtime
def min_numerical_convertible_type(string, check_accuracy=True):
"""
Parse the string and return the smallest numerical type to use for a safe
conversion.
:param str string: Representation of a float/integer with text
:param bool check_accuracy: If true, a warning is pushed on the logger
in case there is a loss of accuracy.
:raise ValueError: When the string is not a numerical value
:retrun: A numpy numerical type
"""
if string == "":
raise ValueError("Not a numerical value")
match = _parse_numeric_value.match(string)
if match is None:
raise ValueError("Not a numerical value")
number, decimal, exponent = match.groups()
if decimal is None and exponent is None:
# It's an integer
# TODO: We could find the int type without converting the number
value = int(string)
return numpy.min_scalar_type(value).type
# Try floating-point
try:
value = _biggest_float(string)
except ValueError:
raise ValueError("Not a numerical value")
if number is None:
number = ""
if decimal is None:
decimal = ""
if exponent is None:
exponent = "0"
nb_precision_digits = int(exponent) - len(decimal) - 1
precision = _biggest_float(10) ** nb_precision_digits * 2.5
previous_type = _biggest_float
for numpy_type in _float_types:
if numpy_type == _biggest_float:
# value was already casted using the bigger type
continue
reduced_value = numpy_type(value)
if not numpy.isfinite(reduced_value):
break
# numpy isclose(atol=is not accurate enough)
diff = value - reduced_value
# numpy 1.8.2 looks to do the substraction using float64...
# we lose precision here
diff = numpy.abs(diff)
if diff > precision:
break
previous_type = numpy_type
# It's the smaller float type which fit with enougth precision
numpy_type = previous_type
if check_accuracy and numpy_type == _biggest_float:
# Check the precision using the original string
expected = number + decimal
# This format the number without python convertion
try:
result = numpy.array2string(value, precision=len(number) + len(decimal), floatmode="fixed")
except TypeError:
# numpy 1.8.2 do not have floatmode argument
_logger.warning("Not able to check accuracy of the conversion of '%s' using %s", string, _biggest_float)
return numpy_type
result = result.replace(".", "").replace("-", "")
if not result.startswith(expected):
_logger.warning("Not able to convert '%s' using %s without losing precision", string, _biggest_float)
return numpy_type
def min_scalar_type(val):
"""
Wrapper for :py:func:`numpy.min_scalar_dtype`
which takes into account types supported by GPUs.
"""
return _promote_dtype(numpy.min_scalar_type(val))
