def match(match, indices, out=None):
"""Matches a sorted list of 1d indices to another larger one
Arguments:
match : array
array of indices to match to indices
indices : array or None
array of indices
Returns:
array
array with specified entries set to new value
Note:
Uses numpy if there is no match of dimension implemented!
"""
if match.ndim != 1:
raise ValueError('Match array dimension required to be 1d, found %d!' %
(match.ndim))
if indices.ndim != 1:
raise ValueError(
'Indices array dimension required to be 1d, found %d!' %
(indices.ndim))
if out is None:
out = np.empty(len(match), dtype=match.dtype)
code.match1d(match, indices, out)
return out
def take(data,
indices,
out=None,
cutoff=defaultCutoff,
processes=defaultProcesses):
"""Extracts the values at specified indices
Arguments:
data : array
array to search for nonzero indices
out : array or None
if not None results is written into this array
cutoff : int
number of elements below whih to switch to numpy.where
processes : None or int
number of processes, if None use number of cpus
Returns:
array
positions of the nonzero entries of the input array
Note:
Uses numpy data[indices] if there is no match of dimension implemented!
"""
if data.ndim != 1:
raise Warning('Using numpy where for dimension %d and type %s!' %
(data.ndim, data.dtype))
return data[indices]
if cutoff is None:
cutoff = 1
cutoff = min(1, cutoff)
if data.size < cutoff:
return data[indices]
if processes is None:
processes = defaultProcesses
if data.dtype == bool:
d = data.view('uint8')
else:
d = data
if out is None:
out = np.empty(len(indices), dtype=data.dtype)
if out.dtype == bool:
o = out.view('uint8')
else:
o = out
code.take1d(d, indices, o, processes=processes)
return out
def setArray(data,
indices,
values,
cutoff=defaultCutoff,
processes=defaultProcesses):
"""Set value at specified indices of an array
Arguments:
data : array
array to search for nonzero indices
indices : array or None
list of indices to set
values : array
values to set elements in data to
processes : None or int
number of processes, if None use number of cpus
Returns:
array
array with specified entries set to new value
Note:
Uses numpy if there is no match of dimension implemented!
"""
if data.ndim != 1:
raise Warning('Using numpy where for dimension %d and type %s!' %
(data.ndim, data.dtype))
data[indices] = values
return data
if cutoff is None:
cutoff = 1
cutoff = min(1, cutoff)
if data.size <= cutoff:
data[indices] = values
return data
if processes is None:
processes = defaultProcesses
if data.dtype == bool:
d = data.view('uint8')
else:
d = data
code.set1darray(d, indices, values, processes=processes)
return data
def blockSums(data, blocks=None, processes=defaultProcesses):
"""Sums of evenly spaced blocks in array
Arguments:
data : array
array to perform the block sums on
blocks : int or None
number of blocks to split array into
processes : None or int
number of processes, if None use number of cpus
Returns:
array
sums of the values in the different blocks
"""
if processes is None:
processes = defaultProcesses
if blocks is None:
blocks = processes * defaultBlocksPerProcess
d = data.reshape(-1, order='A')
if data.dtype == bool:
d = d.view('uint8')
return code.blockSums1d(d, blocks=blocks, processes=processes)
def findNeighbours(indices, center, shape, strides, mask):
"""Finds all indices within a specified kernel region centered at a point"""
if len(strides) != 3 or len(shape) != 3 or (strides[0] != 1
and strides[2] != 1):
raise RuntimeError('only 3d C or F contiguous arrays suported')
if isinstance(mask, int):
mask = (mask, )
if isinstance(mask, tuple):
mask = mask * 3
return code.neighbourlistRadius(indices, center, shape[0], shape[1],
shape[2], strides[0], strides[1],
strides[2], mask[0], mask[1], mask[2])
else:
if mask.dtype == bool:
mask = mask.view(dtype='uint8')
return code.neighbourlistMask(indices, center, shape[0], shape[1],
shape[2], strides[0], strides[1],
strides[2], mask)
def sum(data, processes=defaultProcesses):
"""Returns the sum of the array entries
Arguments:
data : array
input array to sum
processes : None or int
number of processes, if None use number of cpus
Returns:
float
sum of entries in input array
"""
if processes is None:
processes = defaultProcesses
if data.dtype == bool:
d = data.view('uint8')
else:
d = data
d = d.reshape(-1, order='A')
return code.sum(d, processes)
def save(filename,
data,
region=None,
blocks=None,
processes=cpu_count(),
verbose=False):
"""Save a large npy array to disk in parallel
Arguments:
filename : str
filename of array to load
data : array
array to save to disk
blocks : int or None
number of blocks to split array into for parallel processing
processes : None or int
number of processes, if None use number of cpus
verbose : bool
print info about the file to be loaded
Returns:
str
the filename of the numpy array on disk
"""
if processes is None:
processes = cpu_count()
if blocks is None:
blocks = processes * defaultBlocksPerProcess
if region is None:
#create file on disk via memmap
memmap = np.lib.format.open_memmap(filename,
mode='w+',
shape=data.shape,
dtype=data.dtype,
fortran_order=np.isfortran(data))
memmap.flush()
del (memmap)
#get specs from header specs
shape, dtype, order, offset = readNumpyHeader(filename)
if verbose:
timer = tmr.Timer()
print(
'Saving array of shape = %r, dtype = %r, order = %r, offset = %r' %
(shape, dtype, order, offset))
if (np.isfortran(data) and order != 'F') or (not np.isfortran(data)
and order != 'C'):
raise RuntimeError(
'Order of arrays do not match isfortran=%r and order=%s' %
(np.isfortran(data), order))
d = data.reshape(-1, order='A')
if dtype == bool:
d = d.view('uint8')
if region is not None:
sourceSlice = region.sourceSlice()
off = _offsetFromSlice(sourceSlice, order=order)
if order == 'F':
offset += data.strides[-1] * off
else:
offset += data.strides[1] * off
#print d.dtype, filename, offset, blocks, processes
filename = str(filename).encode('UTF-8')
code.save(data=d,
filename=filename,
offset=offset,
blocks=blocks,
processes=processes)
if verbose:
timer.print_elapsed_time(head='Saving array to %s' % filename)
return filename
def load(filename,
region=None,
shared=False,
blocks=None,
processes=cpu_count(),
verbose=False):
"""Load a large npy array into memory in parallel
Arguments:
filename : str
filename of array to load
region : Region or None
if not None this specifies the sub-region to read
shared : bool
if True read into shared memory
blocks : int or None
number of blocks to split array into for parallel processing
processes : None or int
number of processes, if None use number of cpus
verbose : bool
print info about the file to be loaded
Returns:
array
the data as numpy array
"""
if processes is None:
processes = cpu_count()
if blocks is None:
blocks = processes * defaultBlocksPerProcess
#get specs from header specs
shape, dtype, order, offset = readNumpyHeader(filename)
if verbose:
timer = tmr.Timer()
print(
'Loading array of shape = %r, dtype = %r, order = %r, offset = %r'
% (shape, dtype, order, offset))
if region is not None:
shape = region.shape()
sourceSlice = region.sourceSlice()
off = _offsetFromSlice(sourceSlice, order=order)
if shared:
data = shm.create(shape, dtype=dtype, order=order)
else:
data = np.empty(shape, dtype=dtype, order=order)
d = data.reshape(-1, order='A')
if dtype == bool:
d = d.view('uint8')
if region is not None:
if order == 'F':
offset += data.strides[-1] * off
else:
offset += data.strides[1] * off
filename = str(filename).encode('UTF-8')
code.load(data=d,
filename=filename,
offset=offset,
blocks=blocks,
processes=processes)
if verbose:
timer.print_elapsed_time(head='Loading array from %s' % filename)
return data
def neighbours(indices, offset, processes=defaultProcesses):
"""Returns all pairs of indices that are a part a specified offset"""
return code.neighbours(indices, offset=offset, processes=processes)
def where(data,
out=None,
blocks=None,
cutoff=defaultCutoff,
processes=defaultProcesses):
"""Returns the indices of the non-zero entries of the array
Arguments:
data : array
array to search for nonzero indices
out : array or None
if not None results is written into this array
blocks : int or None
number of blocks to split array into for parallel processing
cutoff : int
number of elements below whih to switch to numpy.where
processes : None or int
number of processes, if None use number of cpus
Returns:
array
positions of the nonzero entries of the input array
Note:
Uses numpy.where if there is no match of dimension implemented!
"""
if data.ndim != 1 and data.ndim != 3:
raise Warning('Using numpy where for dimension %d and type %s!' %
(data.ndim, data.dtype))
return np.vstack(np.where(data)).T
if cutoff is None:
cutoff = 1
cutoff = min(1, cutoff)
if data.size <= cutoff:
return np.vstack(np.where(data)).T
if processes is None:
processes = defaultProcesses
if blocks is None:
blocks = processes * defaultBlocksPerProcess
if data.dtype == bool:
d = data.view('uint8')
else:
d = data
if out is None:
if d.ndim == 1:
sums = code.blockSums1d(d, blocks=blocks, processes=processes)
else:
sums = code.blockSums3d(d, blocks=blocks, processes=processes)
out = np.squeeze(np.zeros((np.sum(sums), data.ndim), dtype=np.int))
else:
sums = None
if d.ndim == 1:
code.where1d(d, out=out, sums=sums, blocks=blocks, processes=processes)
else: # d.ndim == 3:
code.where3d(d, out=out, sums=sums, blocks=blocks, processes=processes)
return out