def assert_array_list_equal(xlist, ylist, err_msg='', verbose=True):
if len(xlist) != len(ylist):
raise AssertionError('List size is different')
for x, y in zip(xlist, ylist):
numpy.testing.assert_array_equal(
cupy.asnumpy(x), cupy.asnumpy(y), err_msg=err_msg,
verbose=verbose)
def assert_array_list_equal(xlist, ylist, err_msg='', verbose=True):
"""Compares lists of arrays pairwise with ``assert_array_equal``.
Args:
x(array_like): Array of the actual objects.
y(array_like): Array of the desired, expected objects.
err_msg(str): The error message to be printed in case of failure.
verbose(bool): If True, the conflicting values
are appended to the error message.
Each element of ``x`` and ``y`` must be either :class:`numpy.ndarray`
or :class:`cupy.ndarray`. ``x`` and ``y`` must have same length.
Otherwise, this function raises ``AssertionError``.
It compares elements of ``x`` and ``y`` pairwise
with :func:`assert_array_equal` and raises error if at least one
pair is not equal.
.. seealso:: :func:`numpy.testing.assert_array_equal`
"""
if len(xlist) != len(ylist):
raise AssertionError('List size is different')
for x, y in zip(xlist, ylist):
numpy.testing.assert_array_equal(
cupy.asnumpy(x), cupy.asnumpy(y), err_msg=err_msg,
verbose=verbose)
def test_func(self, *args, **kw):
kw[name] = cupy
cupy_result, cupy_error, cupy_tb = _call_func(self, impl, args, kw)
kw[name] = numpy
numpy_result, numpy_error, numpy_tb = \
_call_func(self, impl, args, kw)
if cupy_error or numpy_error:
_check_cupy_numpy_error(self, cupy_error, cupy_tb,
numpy_error, numpy_tb,
accept_error=accept_error)
return
# Behavior of assigning a negative value to an unsigned integer
# variable is undefined.
# nVidia GPUs and Intel CPUs behave differently.
# To avoid this difference, we need to ignore dimensions whose
# values are negative.
if _contains_signed_and_unsigned(kw):
inds = _make_positive_indices(self, impl, args, kw)
cupy_result = cupy.asnumpy(cupy_result)[inds]
numpy_result = cupy.asnumpy(numpy_result)[inds]
check_func(cupy_result, numpy_result)
if type_check:
self.assertEqual(cupy_result.dtype, numpy_result.dtype)
def assert_array_almost_equal_nulp(x, y, nulp=1):
"""Compare two arrays relatively to their spacing.
Args:
x(numpy.ndarray or cupy.ndarray): The actual object to check.
y(numpy.ndarray or cupy.ndarray): The desired, expected object.
nulp(int): The maximum number of unit in the last place for tolerance.
.. seealso:: :func:`numpy.testing.assert_array_almost_equal_nulp`
"""
numpy.testing.assert_array_almost_equal_nulp(
cupy.asnumpy(x), cupy.asnumpy(y), nulp=nulp)
def assert_array_max_ulp(a, b, maxulp=1, dtype=None):
"""Check that all items of arrays differ in at most N Units in the Last Place.
Args:
a(numpy.ndarray or cupy.ndarray): The actual object to check.
b(numpy.ndarray or cupy.ndarray): The desired, expected object.
maxulp(int): The maximum number of units in the last place
that elements of ``a`` and ``b`` can differ.
dtype(numpy.dtype): Data-type to convert ``a`` and ``b`` to if given.
.. seealso:: :func:`numpy.testing.assert_array_max_ulp`
"""
numpy.testing.assert_array_max_ulp(
cupy.asnumpy(a), cupy.asnumpy(b), maxulp=maxulp, dtype=dtype)
def assert_array_less(x, y, err_msg='', verbose=True):
"""Raises an AssertionError if array_like objects are not ordered by less than.
Args:
x(numpy.ndarray or cupy.ndarray): The smaller object to check.
y(numpy.ndarray or cupy.ndarray): The larger object to compare.
err_msg(str): The error message to be printed in case of failure.
verbose(bool): If True, the conflicting values
are appended to the error message.
.. seealso:: :func:`numpy.testing.assert_array_less`
"""
numpy.testing.assert_array_less(
cupy.asnumpy(x), cupy.asnumpy(y), err_msg=err_msg,
verbose=verbose)
def assert_array_equal(x, y, err_msg='', verbose=True):
"""Raises an AssertionError if two array_like objects are not equal.
Args:
x(numpy.ndarray or cupy.ndarray): The actual object to check.
y(numpy.ndarray or cupy.ndarray): The desired, expected object.
err_msg(str): The error message to be printed in case of failure.
verbose(bool): If True, the conflicting values
are appended to the error message.
.. seealso:: :func:`numpy.testing.assert_array_equal`
"""
numpy.testing.assert_array_equal(
cupy.asnumpy(x), cupy.asnumpy(y), err_msg=err_msg,
verbose=verbose)
def assert_array_almost_equal(x, y, decimal=6, err_msg='', verbose=True):
"""Raises an AssertionError if objects are not equal up to desired precision.
Args:
x(numpy.ndarray or cupy.ndarray): The actual object to check.
y(numpy.ndarray or cupy.ndarray): The desired, expected object.
decimal(int): Desired precision.
err_msg(str): The error message to be printed in case of failure.
verbose(bool): If ``True``, the conflicting
values are appended to the error message.
.. seealso:: :func:`numpy.testing.assert_array_almost_equal`
"""
numpy.testing.assert_array_almost_equal(
cupy.asnumpy(x), cupy.asnumpy(y), decimal=decimal,
err_msg=err_msg, verbose=verbose)
def validate(test_vis, test_dep, test_labels, N_test, num_label, model, args):
# validate
pbar = ProgressBar(N_test)
lstm_correct_cnt = 0
sum_frame = 0
pred_list = []
total_loss = np.array(0.0, dtype=np.float32)
conf_array = np.zeros((num_label, num_label), dtype=np.int32)
for i in range(0, N_test):
# shape(T, width ,height)
x_vis_batch = xp.asarray(test_vis[i], dtype=np.float32)
x_dep_batch = xp.asarray(test_dep[i], dtype=np.float32)
y_batch = xp.asarray(test_labels[i], dtype=np.int32)
video_length = min(x_vis_batch.shape[0], x_dep_batch.shape[0])
sum_frame += video_length
c = Variable(xp.zeros((1, args.num_hunits)).astype(np.float32), volatile=True)
h = Variable(xp.zeros((1, args.num_hunits)).astype(np.float32), volatile=True)
preds = []
for t in xrange(video_length):
x_vis_frame = x_vis_batch[t,:,:]
x_dep_frame = x_dep_batch[t,:,:]
y_frame = y_batch[:,t]
# LSTMの順伝播
loss_i, pred_lstm, c, h = model.forward_one_step(
x_vis_frame, x_dep_frame, y_frame, c, h, volatile=True)
if args.gpu >= 0:
import cupy
total_loss += cupy.asnumpy(loss_i.data)
else:
total_loss += loss_i.data
pred = xp.argmax(pred_lstm.data)
if(pred.tolist() == y_frame[0]):
lstm_correct_cnt += 1
conf_array[y_frame[0], pred.tolist()] += 1
preds.append(pred.tolist())
pred_list.append(preds)
pbar.update(i + 1
if (i + 1) < N_test else N_test)
"""
正答率を計算
"""
lstm_accuracy = lstm_correct_cnt / float(sum_frame)
return total_loss / sum_frame, lstm_accuracy, pred_list, conf_array
def assert_allclose(actual, desired, rtol=1e-7, atol=0, err_msg='',
verbose=True):
"""Raises an AssertionError if objects are not equal up to desired tolerance.
Args:
actual(numpy.ndarray or cupy.ndarray): The actual object to check.
desired(numpy.ndarray or cupy.ndarray): The desired, expected object.
rtol(float): Relative torelance.
atol(float): Absolute torelance.
err_msg(str): The error message to be printed in case of failure.
verbose(bool): If True, the conflicting
values are appended to the error message.
.. seealso:: :func:`numpy.testing.assert_allclose`
"""
numpy.testing.assert_allclose(
cupy.asnumpy(actual), cupy.asnumpy(desired),
rtol=rtol, atol=atol, err_msg=err_msg, verbose=verbose)
def train(train_vis, train_dep, train_labels, N, num_label, model,opt,args):
pbar = ProgressBar(N)
cnn_correct_cnt = 0
sum_frame = 0
total_loss = np.array(0, dtype=np.float32)
opt.setup(model)
conf_array = np.zeros((num_label, num_label), dtype=np.int32)
for i in range(0, N):
x_vis_batch = xp.asarray(train_vis[i], dtype=np.float32)
x_dep_batch = xp.asarray(train_dep[i], dtype=np.float32)
y_batch = xp.asarray(train_labels[i], dtype=np.int32).reshape(-1)
sum_frame += y_batch.shape[0]
opt.zero_grads()
loss, pred = model.forward(
x_vis_batch, x_dep_batch, y_batch)
pred = xp.argmax(pred.data, axis=1)
if args.gpu >= 0:
cnn_correct_cnt += xp.asnumpy(xp.sum(pred == y_batch))
else:
cnn_correct_cnt += np.sum(pred == y_batch)
if y_batch.size == 1:
conf_array[y_batch, pred] += 1
else:
for j in xrange(y_batch.size):
conf_array[y_batch[j], pred[j]] += 1
loss.backward()
opt.update()
if args.opt in ['AdaGrad', 'MomentumSGD']:
opt.weight_decay(decay=args.weight_decay)
pbar.update(i + 1 if (i + 1) < N else N)
if args.gpu >= 0:
import cupy
total_loss += cupy.asnumpy(loss.data)
else:
total_loss += loss.data
"""
正答率計算
"""
cnn_accuracy = cnn_correct_cnt / float(sum_frame)
return total_loss / sum_frame, cnn_accuracy, conf_array
def save(file, arr):
"""Saves an array to a binary file in ``.npy`` format.
Args:
file (file or str): File or filename to save.
arr (array_like): Array to save. It should be able to feed to
:func:`cupy.asnumpy`.
.. seealso:: :func:`numpy.save`
"""
numpy.save(file, cupy.asnumpy(arr))
def validate(test_vis, test_dep, test_labels, N_test, num_label, model, args):
# validate
pbar = ProgressBar(N_test)
cnn_correct_cnt = 0
sum_frame = 0
pred_list = []
total_loss = np.array(0.0, dtype=np.float32)
conf_array = np.zeros((num_label, num_label), dtype=np.int32)
for i in range(0, N_test):
# shape(T, width ,height)
x_vis_batch = xp.asarray(test_vis[i], dtype=np.float32)
x_dep_batch = xp.asarray(test_dep[i], dtype=np.float32)
y_batch = xp.asarray(test_labels[i], dtype=np.int32).reshape(-1)
sum_frame += y_batch.shape[0]
loss, pred = model.forward(
x_vis_batch, x_dep_batch, y_batch)
pred = xp.argmax(pred.data, axis=1)
if args.gpu >= 0:
cnn_correct_cnt += xp.asnumpy(xp.sum(pred == y_batch))
else:
cnn_correct_cnt += np.sum(pred == y_batch)
if y_batch.size == 1:
conf_array[y_batch, pred] += 1
else:
for j in xrange(y_batch.size):
conf_array[y_batch[j], pred[j]] += 1
if args.gpu >= 0:
import cupy
total_loss += cupy.asnumpy(loss.data)
else:
total_loss += loss.data
pred_list.append(pred.tolist())
pbar.update(i + 1
if (i + 1) < N_test else N_test)
"""
正答率を計算
"""
cnn_accuracy = cnn_correct_cnt / float(sum_frame)
return total_loss / sum_frame, cnn_accuracy, pred_list, conf_array
def savez_compressed(file, *args, **kwds):
"""Saves one or more arrays into a file in compressed ``.npz`` format.
It is equivalent to :func:`cupy.savez` function except the output file is
compressed.
.. seealso::
:func:`cupy.savez` for more detail,
:func:`numpy.savez_compressed`
"""
args = map(cupy.asnumpy, args)
for key in kwds:
kwds[key] = cupy.asnumpy(kwds[key])
numpy.savez_compressed(file, *args, **kwds)
def array_str(arr, max_line_width=None, precision=None, suppress_small=None):
"""Returns the string representation of the content of an array.
Args:
arr (array_like): Input array. It should be able to feed to
:func:`cupy.asnumpy`.
max_line_width (int): The maximum number of line lengths.
precision (int): Floating point precision. It uses the current printing
precision of NumPy.
suppress_small (bool): If True, very small number are printed as
zeros.
.. seealso:: :func:`numpy.array_str`
"""
return numpy.array_str(cupy.asnumpy(arr), max_line_width, precision,
suppress_small)
def savez(file, *args, **kwds):
"""Saves one or more arrays into a file in uncompressed ``.npz`` format.
Arguments without keys are treated as arguments with automatic keys named
``arr_0``, ``arr_1``, etc. corresponding to the positions in the argument
list. The keys of arguments are used as keys in the ``.npz`` file, which
are used for accessing NpzFile object when the file is read by
:func:`cupy.load` function.
Args:
file (file or str): File or filename to save.
*args: Arrays with implicit keys.
**kwds: Arrays with explicit keys.
.. seealso:: :func:`numpy.savez`
"""
args = map(cupy.asnumpy, args)
for key in kwds:
kwds[key] = cupy.asnumpy(kwds[key])
numpy.savez(file, *args, **kwds)
def _suppress(self, raw_cls_bbox, raw_prob):
bbox = list()
label = list()
score = list()
# skip cls_id = 0 because it is the background class
for l in range(1, self.n_class):
cls_bbox_l = raw_cls_bbox.reshape((-1, self.n_class, 4))[:, l, :]
prob_l = raw_prob[:, l]
mask = prob_l > self.score_thresh
cls_bbox_l = cls_bbox_l[mask]
prob_l = prob_l[mask]
keep = non_maximum_suppression(
cp.array(cls_bbox_l), self.nms_thresh, prob_l)
keep = cp.asnumpy(keep)
bbox.append(cls_bbox_l[keep])
# The labels are in [0, self.n_class - 2].
label.append((l - 1) * np.ones((len(keep),)))
score.append(prob_l[keep])
bbox = np.concatenate(bbox, axis=0).astype(np.float32)
label = np.concatenate(label, axis=0).astype(np.int32)
score = np.concatenate(score, axis=0).astype(np.float32)
return bbox, label, score
entries = []
for docidx, doc in enumerate(nlp.pipe(abstracts)):
############ control the representation here ##############
last_hidden_state = doc._.pytt_last_hidden_state
word_pieces_ = doc._.pytt_word_pieces_
D = 768
min_length = 5
count = 0
embedding = np.zeros( (D,) )
for idx, token in enumerate(word_pieces_):
if not(token == '[CLS]'):
if len(token) >= min_length:
embedding += cp.asnumpy(last_hidden_state[idx,:])
count += 1
embedding *= (1.0 / count)
###########################################################
pmid = data[docidx][0]
entries.append((pmid, embedding.tolist(),))
end_time = time.time()
print("elapsed: {}".format(end_time - start_time))
########################
print('writing embeddings to csv file...')
start_time = time.time()
def preprocess(ctx):
# function rez = preprocessDataSub(ops)
# this function takes an ops struct, which contains all the Kilosort2 settings and file paths
# and creates a new binary file of preprocessed data, logging new variables into rez.
# The following steps are applied:
# 1) conversion to float32
# 2) common median subtraction
# 3) bandpass filtering
# 4) channel whitening
# 5) scaling to int16 values
params = ctx.params
probe = ctx.probe
raw_data = ctx.raw_data
ir = ctx.intermediate
fs = params.fs
fshigh = params.fshigh
fslow = params.fslow
Nbatch = ir.Nbatch
NT = params.NT
NTbuff = params.NTbuff
Wrot = cp.asarray(ir.Wrot)
logger.info("Loading raw data and applying filters.")
with open(ir.proc_path, 'wb') as fw: # open for writing processed data
for ibatch in tqdm(range(Nbatch), desc="Preprocessing"):
# we'll create a binary file of batches of NT samples, which overlap consecutively
# on params.ntbuff samples
# in addition to that, we'll read another params.ntbuff samples from before and after,
# to have as buffers for filtering
# number of samples to start reading at.
i = max(0, (NT - params.ntbuff) * ibatch - 2 * params.ntbuff)
if ibatch == 0:
# The very first batch has no pre-buffer, and has to be treated separately
ioffset = 0
else:
ioffset = params.ntbuff
buff = raw_data[:, i:i + NTbuff]
if buff.size == 0:
logger.error("Loaded buffer has an empty size!")
break # this shouldn't really happen, unless we counted data batches wrong
nsampcurr = buff.shape[
1] # how many time samples the current batch has
if nsampcurr < NTbuff:
buff = np.concatenate(
(buff,
np.tile(buff[:, nsampcurr - 1][:, np.newaxis],
(1, NTbuff))),
axis=1)
# apply filters and median subtraction
buff = cp.asarray(buff, dtype=np.float32)
datr = gpufilter(buff,
chanMap=probe.chanMap,
fs=fs,
fshigh=fshigh,
fslow=fslow)
datr = datr[ioffset:ioffset +
NT, :] # remove timepoints used as buffers
datr = cp.dot(
datr, Wrot) # whiten the data and scale by 200 for int16 range
# convert to int16, and gather on the CPU side
datcpu = cp.asnumpy(datr).astype(np.int16)
# write this batch to binary file
fw.write(datcpu.tobytes('F'))
def svd_wrapper(matrix, mode, ncomp, verbose, full_output=False,
random_state=None, to_numpy=True):
""" Wrapper for different SVD libraries (CPU and GPU).
Parameters
----------
matrix : numpy ndarray, 2d
2d input matrix.
mode : {'lapack', 'arpack', 'eigen', 'randsvd', 'cupy', 'eigencupy',
'randcupy', 'pytorch', 'eigenpytorch', 'randpytorch'}, str optional
Switch for the SVD method/library to be used.
``lapack``: uses the LAPACK linear algebra library through Numpy
and it is the most conventional way of computing the SVD
(deterministic result computed on CPU).
``arpack``: uses the ARPACK Fortran libraries accessible through
Scipy (computation on CPU).
``eigen``: computes the singular vectors through the
eigendecomposition of the covariance M.M' (computation on CPU).
``randsvd``: uses the randomized_svd algorithm implemented in
Sklearn (computation on CPU).
``cupy``: uses the Cupy library for GPU computation of the SVD as in
the LAPACK version. `
`eigencupy``: offers the same method as with the ``eigen`` option
but on GPU (through Cupy).
``randcupy``: is an adaptation of the randomized_svd algorithm,
where all the computations are done on a GPU (through Cupy). `
`pytorch``: uses the Pytorch library for GPU computation of the SVD.
``eigenpytorch``: offers the same method as with the ``eigen``
option but on GPU (through Pytorch).
``randpytorch``: is an adaptation of the randomized_svd algorithm,
where all the linear algebra computations are done on a GPU
(through Pytorch).
ncomp : int
Number of singular vectors to be obtained. In the cases when the full
SVD is computed (LAPACK, ARPACK, EIGEN, CUPY), the matrix of singular
vectors is truncated.
verbose: bool
If True intermediate information is printed out.
full_output : bool optional
If True the 3 terms of the SVD factorization are returned. If ``mode``
is eigen then only S and V are returned.
random_state : int, RandomState instance or None, optional
If int, random_state is the seed used by the random number generator.
If RandomState instance, random_state is the random number generator.
If None, the random number generator is the RandomState instance used
by np.random. Used for ``randsvd`` mode.
to_numpy : bool, optional
If True (by default) the arrays computed in GPU are transferred from
VRAM and converted to numpy ndarrays.
Returns
-------
V : numpy ndarray
The right singular vectors of the input matrix. If ``full_output`` is
True it returns the left and right singular vectors and the singular
values of the input matrix. If ``mode`` is set to eigen then only S and
V are returned.
References
----------
* For ``lapack`` SVD mode see:
https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.linalg.svd.html
http://www.netlib.org/lapack/
* For ``eigen`` mode see:
https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.linalg.eigh.html
* For ``arpack`` SVD mode see:
https://docs.scipy.org/doc/scipy-0.19.1/reference/generated/scipy.sparse.linalg.svds.html
http://www.caam.rice.edu/software/ARPACK/
* For ``randsvd`` SVD mode see:
https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/utils/extmath.py
Finding structure with randomness: Stochastic algorithms for constructing
approximate matrix decompositions
Halko, et al., 2009 http://arxiv.org/abs/arXiv:0909.4061
* For ``cupy`` SVD mode see:
https://docs-cupy.chainer.org/en/stable/reference/generated/cupy.linalg.svd.html
* For ``eigencupy`` mode see:
https://docs-cupy.chainer.org/en/master/reference/generated/cupy.linalg.eigh.html
* For ``pytorch`` SVD mode see:
http://pytorch.org/docs/master/torch.html#torch.svd
* For ``eigenpytorch`` mode see:
http://pytorch.org/docs/master/torch.html#torch.eig
"""
if matrix.ndim != 2:
raise TypeError('Input matrix is not a 2d array')
if ncomp > min(matrix.shape[0], matrix.shape[1]):
msg = '{} PCs cannot be obtained from a matrix with size [{},{}].'
msg += ' Increase the size of the patches or request less PCs'
raise RuntimeError(msg.format(ncomp, matrix.shape[0], matrix.shape[1]))
if mode == 'eigen':
# building C as np.dot(matrix.T,matrix) is slower and takes more memory
C = np.dot(matrix, matrix.T) # covariance matrix
e, EV = linalg.eigh(C) # EVals and EVs
pc = np.dot(EV.T, matrix) # PCs using a compact trick when cov is MM'
V = pc[::-1] # reverse since we need the last EVs
S = np.sqrt(np.abs(e)) # SVals = sqrt(EVals)
S = S[::-1] # reverse since EVals go in increasing order
for i in range(V.shape[1]):
V[:, i] /= S # scaling EVs by the square root of EVals
V = V[:ncomp]
if verbose:
print('Done PCA with numpy linalg eigh functions')
elif mode == 'lapack':
# n_frames is usually smaller than n_pixels. In this setting taking
# the SVD of M' and keeping the left (transposed) SVs is faster than
# taking the SVD of M (right SVs)
U, S, V = linalg.svd(matrix.T, full_matrices=False)
V = V[:ncomp] # we cut projection matrix according to the # of PCs
U = U[:, :ncomp]
S = S[:ncomp]
if verbose:
print('Done SVD/PCA with numpy SVD (LAPACK)')
elif mode == 'arpack':
U, S, V = svds(matrix, k=ncomp)
if verbose:
print('Done SVD/PCA with scipy sparse SVD (ARPACK)')
elif mode == 'randsvd':
U, S, V = randomized_svd(matrix, n_components=ncomp, n_iter=2,
transpose='auto', random_state=random_state)
if verbose:
print('Done SVD/PCA with randomized SVD')
elif mode == 'cupy':
if no_cupy:
raise RuntimeError('Cupy is not installed')
a_gpu = cupy.array(matrix)
a_gpu = cupy.asarray(a_gpu) # move the data to the current device
u_gpu, s_gpu, vh_gpu = cupy.linalg.svd(a_gpu, full_matrices=True,
compute_uv=True)
V = vh_gpu[:ncomp]
if to_numpy:
V = cupy.asnumpy(V)
if full_output:
S = s_gpu[:ncomp]
if to_numpy:
S = cupy.asnumpy(S)
U = u_gpu[:, :ncomp]
if to_numpy:
U = cupy.asnumpy(U)
if verbose:
print('Done SVD/PCA with cupy (GPU)')
elif mode == 'randcupy':
if no_cupy:
raise RuntimeError('Cupy is not installed')
U, S, V = randomized_svd_gpu(matrix, ncomp, n_iter=2, lib='cupy')
if to_numpy:
V = cupy.asnumpy(V)
S = cupy.asnumpy(S)
U = cupy.asnumpy(U)
if verbose:
print('Done randomized SVD/PCA with cupy (GPU)')
elif mode == 'eigencupy':
if no_cupy:
raise RuntimeError('Cupy is not installed')
a_gpu = cupy.array(matrix)
a_gpu = cupy.asarray(a_gpu) # move the data to the current device
C = cupy.dot(a_gpu, a_gpu.T) # covariance matrix
e, EV = cupy.linalg.eigh(C) # eigenvalues and eigenvectors
pc = cupy.dot(EV.T, a_gpu) # using a compact trick when cov is MM'
V = pc[::-1] # reverse to get last eigenvectors
S = cupy.sqrt(e)[::-1] # reverse since EVals go in increasing order
for i in range(V.shape[1]):
V[:, i] /= S # scaling by the square root of eigvals
V = V[:ncomp]
if to_numpy:
V = cupy.asnumpy(V)
if verbose:
print('Done PCA with cupy eigh function (GPU)')
elif mode == 'pytorch':
if no_torch:
raise RuntimeError('Pytorch is not installed')
a_gpu = torch.Tensor.cuda(torch.from_numpy(matrix.astype('float32').T))
u_gpu, s_gpu, vh_gpu = torch.svd(a_gpu)
V = vh_gpu[:ncomp]
S = s_gpu[:ncomp]
U = torch.transpose(u_gpu, 0, 1)[:ncomp]
if to_numpy:
V = np.array(V)
S = np.array(S)
U = np.array(U)
if verbose:
print('Done SVD/PCA with pytorch (GPU)')
elif mode == 'eigenpytorch':
if no_torch:
raise RuntimeError('Pytorch is not installed')
a_gpu = torch.Tensor.cuda(torch.from_numpy(matrix.astype('float32')))
C = torch.mm(a_gpu, torch.transpose(a_gpu, 0, 1))
e, EV = torch.eig(C, eigenvectors=True)
V = torch.mm(torch.transpose(EV, 0, 1), a_gpu)
S = torch.sqrt(e[:, 0])
for i in range(V.shape[1]):
V[:, i] /= S
V = V[:ncomp]
if to_numpy:
V = np.array(V)
if verbose:
print('Done PCA with pytorch eig function')
elif mode == 'randpytorch':
if no_torch:
raise RuntimeError('Pytorch is not installed')
U, S, V = randomized_svd_gpu(matrix, ncomp, n_iter=2, lib='pytorch')
if to_numpy:
V = np.array(V)
S = np.array(S)
U = np.array(U)
if verbose:
print('Done randomized SVD/PCA with randomized pytorch (GPU)')
else:
raise ValueError('The SVD `mode` is not recognized')
if full_output:
if mode == 'lapack':
return V.T, S, U.T
elif mode == 'pytorch':
if to_numpy:
return V.T, S, U.T
else:
return torch.transpose(V, 0, 1), S, torch.transpose(U, 0, 1)
elif mode in ('eigen', 'eigencupy', 'eigenpytorch'):
return S, V
else:
return U, S, V
else:
if mode == 'lapack':
return U.T
elif mode == 'pytorch':
return U
else:
return V
def unique(ar, axis=None, *args, **kwargs):
""" For cupy v0.6.0 compatibility """
return cp.array(np.unique(cp.asnumpy(ar), axis=axis, *args, **kwargs))
def assert_array_max_ulp(a, b, maxulp=1, dtype=None):
numpy.testing.assert_array_max_ulp(
cupy.asnumpy(a), cupy.asnumpy(b), maxulp=maxulp, dtype=dtype)
def assert_array_less(x, y, err_msg='', verbose=True):
numpy.testing.assert_array_less(
cupy.asnumpy(x), cupy.asnumpy(y), err_msg=err_msg,
verbose=verbose)
def _make_positive_indices(self, impl, args, kw):
ks = [k for k, v in kw.items() if v in _unsigned_dtypes]
for k in ks:
kw[k] = numpy.intp
mask = cupy.asnumpy(impl(self, *args, **kw)) >= 0
return numpy.nonzero(mask)
def nanmedian(a, *args, **kwargs):
""" For cupy v0.6.0 compatibility """
return cp.array(np.nanmedian(cp.asnumpy(a), *args, **kwargs))
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
'--pretrained',
type=str,
help=
'path to model that has trained classifier but has not been trained through GAIN routine',
default='classifier_padding_1_model_594832')
parser.add_argument(
'--trained',
type=str,
help='path to model trained through GAIN',
default='result/MYGAIN_5_to_1_padding_1_all_update_model_20000')
parser.add_argument('--device', type=int, default=0, help='gpu id')
parser.add_argument('--shuffle',
type=bool,
default=False,
help='whether to shuffle dataset')
parser.add_argument(
'--whole',
type=bool,
default=False,
help='whether to test for the whole validation dataset')
parser.add_argument('--no',
type=int,
default=5,
help='if not whole, then no of images to visualize')
parser.add_argument(
'--name',
type=str,
default='viz1',
help='name of the subfolder or experiment under which to save')
args = parser.parse_args()
pretrained_file = args.pretrained
trained_file = args.trained
device = args.device
shuffle = args.shuffle
whole = args.whole
name = args.name
N = args.no
dataset = MyTrainingDataset(split='val')
iterator = SerialIterator(dataset, 1, shuffle=shuffle, repeat=False)
converter = chainer.dataset.concat_examples
os.makedirs('viz/' + name, exist_ok=True)
no_of_classes = 21
device = 0
pretrained = FCN8s_hand()
trained = FCN8s_hand()
load_npz(pretrained_file, pretrained)
load_npz(trained_file, trained)
if device >= 0:
pretrained.to_gpu()
trained.to_gpu()
i = 0
true_positive = [0 for j in range(21)]
true_negative = [0 for j in range(21)]
false_positive = [0 for j in range(21)]
false_negative = [0 for j in range(21)]
while not iterator.is_new_epoch:
if not whole and i >= N:
break
image, labels, metadata = converter(iterator.next())
np_input_img = image
np_input_img = np.uint8(np_input_img[0])
np_input_img = np.transpose(np_input_img, (1, 2, 0))
image = Variable(image)
if device >= 0:
image.to_gpu()
xp = get_array_module(image.data)
to_substract = np.array((-1, 0))
noise_classes = np.unique(labels[0]).astype(np.int32)
target = xp.asarray([[0] * (no_of_classes)])
gt_labels = np.setdiff1d(noise_classes, to_substract) - 1
target[0][gt_labels] = 1
gcam1, cl_scores1, class_id1 = pretrained.stream_cl(image)
gcam2, cl_scores2, class_id2 = trained.stream_cl(image)
# gcams1, cl_scores1, class_ids1 = pretrained.stream_cl_multi(image)
# gcams2, cl_scores2, class_ids2 = trained.stream_cl_multi(image)
target = cp.asnumpy(target)
cl_scores2 = cp.asnumpy(cl_scores2.data)
# print(target)
# print(cl_scores2)
# print()
# score_sigmoid = F.sigmoid(cl_scores2)
for j in range(0, len(target[0])):
# print(target[0][j] == 1)
if target[0][j] == 1:
if cl_scores2[0][j] >= 0:
true_positive[j] += 1
else:
false_negative[j] += 1
else:
if cl_scores2[0][j] <= 0:
true_negative[j] += 1
else:
false_positive[j] += 1
# bboxes = gcams_to_bboxes(gcams2, class_ids2, input_image=np_input_img)
# cv2.imshow('input', np_input_img)
# cv2.waitKey(0)
if device > -0:
class_id = cp.asnumpy(class_id)
# fig1 = plt.figure(figsize=(20, 10))
# ax1 = plt.subplot2grid((3, 9), (0, 0), colspan=3, rowspan=3)
# ax1.axis('off')
# ax1.imshow(cp.asnumpy(F.transpose(F.squeeze(image, 0), (1, 2, 0)).data) / 255.)
#
# ax2 = plt.subplot2grid((3, 9), (0, 3), colspan=3, rowspan=3)
# ax2.axis('off')
# ax2.imshow(cp.asnumpy(F.transpose(F.squeeze(image, 0), (1, 2, 0)).data) / 255.)
# ax2.imshow(cp.asnumpy(F.squeeze(gcam1[0], 0).data), cmap='jet', alpha=.5)
# ax2.set_title("Before GAIN for class - " + str(dataset.class_names[cp.asnumpy(class_id1)+1]),
# color='teal')
#
# ax3 = plt.subplot2grid((3, 9), (0, 6), colspan=3, rowspan=3)
# ax3.axis('off')
# ax3.imshow(cp.asnumpy(F.transpose(F.squeeze(image, 0), (1, 2, 0)).data) / 255.)
# ax3.imshow(cp.asnumpy(F.squeeze(gcam2[0], 0).data), cmap='jet', alpha=.5)
# ax3.set_title("After GAIN for class - " + str(dataset.class_names[cp.asnumpy(class_id2)+1]),
# color='teal')
# fig1.savefig('viz/' + name + '/' + str(i) + '.png')
# plt.close()
print(i)
i += 1
print("true postive {}".format(true_positive))
print("true negative {}".format(true_negative))
print("false positive {}".format(false_positive))
print("false negative {}".format(false_negative))
def make_image(trainer):
np.random.seed(seed)
xp = enc.xp
batch = updater.get_iterator('test').next()
batchsize = len(batch)
labels, inputs = xp.split(xp.asarray(batch), [1], axis=1)
# 2次元目はinputとlabelを分けて保持していたが、分割操作により２次元目は不要になる。
# Encoderのデータ入力形式（ndim=4）に合わせる必要あり
# (samples,input_or_label, channel(スペクトログラムと位相）, 周波数, 時刻) から input_or_labelを除去する
dataShape = list(labels.shape)
dataShape.pop(1)
labels = labels.reshape(dataShape) #(1,1,2,1024,-1) -> (1,2,1024,-1)
inputs = inputs.reshape(dataShape)
# labels = xp.array([[[[]]]])
# inputs = xp.array([])
# for i in range(batchsize):
# #np.append(inputs,xp.asarray(batch[i][0]),axis=0) #GPU計算の場合はarray操作はcupyで行う。cupyにはappendという関数はない
# xp.concatenate((inputs,xp.asarray(batch[i][0])),axis=0)
# xp.concatenate((labels,xp.asarray(batch[i][1])),axis=0)
x_in = Variable(inputs)
z = enc(x_in)
x_out = dec(z)
outputs = x_out.array
if xp != np:
#GPU計算してるなら計算後データをGPUメモリからCPU側メモリへ移動
# cupyの対応していない関数などのせいでpyplotなどでエラーでるから
inputs = cp.asnumpy(inputs)
outputs = cp.asnumpy(outputs)
labels = cp.asnumpy(labels)
for i, input, output, label in zip(range(batchsize), inputs, outputs,
labels):
# 1figureに複数imageをプロット
# 参考：https://stackoverflow.com/questions/46615554/how-to-display-multiple-images-in-one-figure-correctly
rows = 3
cols = 2
fig = plt.figure(figsize=(10, 15))
def subplot(imageData, title):
subplot.counter += 1 #1~7
s = list(imageData.shape)
imageData = imageData.reshape(
(s[-2], s[-1])) # (ch,h,w) -> (h,w)
ax = fig.add_subplot(rows, cols, subplot.counter)
plt.imshow(imageData, cmap='gray')
ax.set_title(title)
subplot.counter = 0
print('plotting preview images iter:{}'.format(
trainer.updater.iteration))
subplot(input[0], 'input spectrogram')
subplot(input[1], 'input phase')
subplot(output[0], 'output spectrogram')
subplot(output[1], 'output phase')
subplot(label[0], 'labels spectrogram')
subplot(label[1], 'labels phase')
preview_dir = '{}/preview'.format(dst)
fileName = 'iter{:0>8}_{}'.format(trainer.updater.iteration, i)
imagePath = os.path.join(preview_dir, fileName + '.jpg')
if not os.path.exists(preview_dir):
os.makedirs(preview_dir)
plt.savefig(imagePath)
D_input_abs = input[0]
D_input_phase = input[1]
D_input_abs = util.rescaleArray(D_input_abs)
y_input_hat = util.convert_to_wave(D_input_abs, D_input_phase)
inputWavPath = os.path.join(preview_dir, fileName + '_input.wav')
librosa.output.write_wav(inputWavPath,
y_input_hat,
sr=util.sample_ratio)
D_output_abs = output[0]
D_output_phase = output[1]
D_output_abs = util.rescaleArray(D_output_abs)
y_output_hat = util.convert_to_wave(D_output_abs, D_output_phase)
outputWavPath = os.path.join(preview_dir, fileName + '_output.wav')
librosa.output.write_wav(outputWavPath,
y_output_hat,
sr=util.sample_ratio)
if util.enable_output_labelWav:
D_label_abs = label[0]
D_label_phase = label[1]
D_label_abs = util.rescaleArray(D_label_abs)
y_label_hat = util.convert_to_wave(D_label_abs, D_label_phase)
labelWavPath = os.path.join(preview_dir,
fileName + '_label.wav')
librosa.output.write_wav(labelWavPath,
y_label_hat,
sr=util.sample_ratio)
def to_numpy(tensor):
"""Returns a copy of the tensor as a NumPy array"""
if isinstance(tensor, cp.ndarray):
return cp.asnumpy(tensor)
return tensor
def _calc_array(self, cpu_c_flat: np.ndarray) -> np.ndarray:
gpu_c_flat = cp.asarray(cpu_c_flat)
gpu_iteration_flat = self.server.compute_flat_array(gpu_c_flat)
cpu_iteration_flat = cp.asnumpy(gpu_iteration_flat)
# cpu_iteration_flat = compute_array.ComputeGpu.compute(cpu_c_flat)
return cpu_iteration_flat
end = timeit.default_timer()
print(f'Time Elapsed = {end-begin}')
savePath = '../GroundStateSave/gs2.txt'
simulationTest.saveSolution(solution, savePath)
loadPath = '../GroundStateSave/gs2.txt'
solution = simulationTest.loadSolution(loadPath)
# Arrays needed for plotting
y_np = simulationTest.hy * np.arange(NY) + ya
z_np = simulationTest.hz * np.arange(NZ) + za
zz, yy = np.meshgrid(z_np, y_np)
fig = plt.figure(2) # Clear figure 2 window and bring forward
ax = fig.gca(projection='3d')
surf = ax.plot_surface(zz,
yy,
np.sum(np.abs(cp.asnumpy(solution))**2, axis=(0)) *
simulationTest.hx,
rstride=1,
cstride=1,
cmap=cm.jet,
linewidth=0,
antialiased=False)
ax.set_xlabel('Z-Axis')
ax.set_ylabel('Y-Axis')
ax.set_zlabel('Amplitude)')
def evaluate(self):
self.load_from_file()
m1 = self.fiducial_binaries["mass_1"]
m2 = self.fiducial_binaries["mass_2"]
# Note that spins are redshift independent
spin_1x, spin_1y, spin_1z = [
self.fiducial_binaries[k]
for k in ["spin_1x", "spin_1y", "spin_1z"]
]
spin_2x, spin_2y, spin_2z = [
self.fiducial_binaries[k]
for k in ["spin_2x", "spin_2y", "spin_2z"]
]
if _GPU_ENABLED:
import cupy as xp
else:
import numpy as xp
m1 = xp.asarray(m1)
m2 = xp.asarray(m2)
spin_1x = xp.asarray(spin_1x)
spin_1y = xp.asarray(spin_1y)
spin_1z = xp.asarray(spin_1z)
spin_2x = xp.asarray(spin_2x)
spin_2y = xp.asarray(spin_2y)
spin_2z = xp.asarray(spin_2z)
pdf_spin_fiducial = xp.asarray(self.pdf_spin_fiducial)
pdf_mass_fiducial = xp.asarray(self.pdf_mass_fiducial)
pdf_spin_pop = self.spin_src_pop_model.prob({
"spin_1x": spin_1x,
"spin_1y": spin_1y,
"spin_1z": spin_1z,
"spin_2x": spin_2x,
"spin_2y": spin_2y,
"spin_2z": spin_2z,
})
weights_spin = pdf_spin_pop / pdf_spin_fiducial
for img in range(self.N_img):
self.predictions[img] = xp.asarray(self.predictions[img])
self.pdf_dLs_fiducial[img] = xp.asarray(self.pdf_dLs_fiducial[img])
self.apparent_dLs[img] = xp.asarray(self.apparent_dLs[img])
def epsilon(z_src):
det_mass_pop_dist = DetectorFrameComponentMassesFromSourceFrame(
self.mass_src_pop_model, z_src)
pdf_mass_pop = det_mass_pop_dist.prob({"mass_1": m1, "mass_2": m2})
weights_mass = pdf_mass_pop / pdf_mass_fiducial
weights_source = weights_mass * weights_spin
integrand = weights_source
for img in range(self.N_img):
pdf_dL_fiducial = self.pdf_dLs_fiducial[img]
dL_pop_dist = LuminosityDistancePriorFromAbsoluteMagnificationRedshift(
self.abs_magn_dist[img], z_src)
pdf_dL_pop = dL_pop_dist.prob(self.apparent_dLs[img])
weights_dL = pdf_dL_pop / pdf_dL_fiducial
integrand *= self.predictions[img] * weights_dL
return float(xp.sum(integrand) / float(self.N_inj))
logger = logging.getLogger(__prog__)
logger.info("Integrating over source redshift")
z_dist = LensedSourceRedshiftProbDist(
merger_rate_density=self.merger_rate_density_src_pop_model,
optical_depth=self.optical_depth)
zs = z_dist.sample(size=self.N_z)
if _GPU_ENABLED:
import cupy as cp
zs = cp.asnumpy(zs)
epsilons = []
for z in tqdm.tqdm(zs):
epsilons.append(epsilon(z))
beta = np.sum(epsilons).astype(float) / self.N_z
self.f.close()
return beta
def to_np(*args):
""" convert GPU arras to numpy and return them"""
if len(args) > 1:
return (cp.asnumpy(x) for x in args)
else:
return cp.asnumpy(args[0])
def svd_wrapper(matrix, mode, ncomp, debug, verbose, usv=False,
random_state=None, to_numpy=True):
""" Wrapper for different SVD libraries (CPU and GPU).
Parameters
----------
matrix : array_like, 2d
2d input matrix.
mode : {'lapack', 'arpack', 'eigen', 'randsvd', 'cupy', 'eigencupy',
'randcupy', 'pytorch', 'eigenpytorch', 'randpytorch'}, str optional
Switch for the SVD method/library to be used. ``lapack`` uses the LAPACK
linear algebra library through Numpy and it is the most conventional way
of computing the SVD (deterministic result computed on CPU). ``arpack``
uses the ARPACK Fortran libraries accessible through Scipy (computation
on CPU). ``eigen`` computes the singular vectors through the
eigendecomposition of the covariance M.M' (computation on CPU).
``randsvd`` uses the randomized_svd algorithm implemented in Sklearn
(computation on CPU). ``cupy`` uses the Cupy library for GPU computation
of the SVD as in the LAPACK version. ``eigencupy`` offers the same
method as with the ``eigen`` option but on GPU (through Cupy).
``randcupy`` is an adaptation f the randomized_svd algorithm, where all
the computations are done on a GPU (through Cupy). ``pytorch`` uses the
Pytorch library for GPU computation of the SVD. ``eigenpytorch`` offers
the same method as with the ``eigen`` option but on GPU (through
Pytorch). ``randpytorch`` is an adaptation of the randomized_svd
algorithm, where all the linear algebra computations are done on a GPU
(through Pytorch).
ncomp : int
Number of singular vectors to be obtained. In the cases when the full
SVD is computed (LAPACK, ARPACK, EIGEN, CUPY), the matrix of singular
vectors is truncated.
debug : bool
If True the explained variance ratio is computed and displayed.
verbose: bool
If True intermediate information is printed out.
usv : bool optional
If True the 3 terms of the SVD factorization are returned.
random_state : int, RandomState instance or None, optional
If int, random_state is the seed used by the random number generator.
If RandomState instance, random_state is the random number generator.
If None, the random number generator is the RandomState instance used
by np.random. Used for ``randsvd`` mode.
to_numpy : bool, optional
If True (by default) the arrays computed in GPU are transferred from
VRAM and converted to numpy ndarrays.
Returns
-------
V : array_like
The right singular vectors of the input matrix. If ``usv`` is True it
returns the left and right singular vectors and the singular values of
the input matrix.
References
----------
* For ``lapack`` SVD mode see:
https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.linalg.svd.html
http://www.netlib.org/lapack/
* For ``eigen`` mode see:
https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.linalg.eigh.html
* For ``arpack`` SVD mode see:
https://docs.scipy.org/doc/scipy-0.19.1/reference/generated/scipy.sparse.linalg.svds.html
http://www.caam.rice.edu/software/ARPACK/
* For ``randsvd`` SVD mode see:
https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/utils/extmath.py
Finding structure with randomness: Stochastic algorithms for constructing
approximate matrix decompositions
Halko, et al., 2009 http://arxiv.org/abs/arXiv:0909.4061
* For ``cupy`` SVD mode see:
https://docs-cupy.chainer.org/en/stable/reference/generated/cupy.linalg.svd.html
* For ``eigencupy`` mode see:
https://docs-cupy.chainer.org/en/master/reference/generated/cupy.linalg.eigh.html
* For ``pytorch`` SVD mode see:
http://pytorch.org/docs/master/torch.html#torch.svd
* For ``eigenpytorch`` mode see:
http://pytorch.org/docs/master/torch.html#torch.eig
"""
def reconstruction(ncomp, U, S, V, var=1):
if mode == 'lapack':
rec_matrix = np.dot(U[:, :ncomp],
np.dot(np.diag(S[:ncomp]), V[:ncomp]))
rec_matrix = rec_matrix.T
print(' Matrix reconstruction with {} PCs:'.format(ncomp))
print(' Mean Absolute Error =', MAE(matrix, rec_matrix))
print(' Mean Squared Error =', MSE(matrix, rec_matrix))
# see https://github.com/scikit-learn/scikit-learn/blob/c3980bcbabd9d2527548820581725df2904e4a0d/sklearn/decomposition/pca.py
exp_var = (S ** 2) / (S.shape[0] - 1)
full_var = np.sum(exp_var)
explained_variance_ratio = exp_var / full_var # % of variance explained by each PC
ratio_cumsum = np.cumsum(explained_variance_ratio)
elif mode == 'eigen':
exp_var = (S ** 2) / (S.shape[0] - 1)
full_var = np.sum(exp_var)
explained_variance_ratio = exp_var / full_var # % of variance explained by each PC
ratio_cumsum = np.cumsum(explained_variance_ratio)
else:
rec_matrix = np.dot(U, np.dot(np.diag(S), V))
print(' Matrix reconstruction MAE =', MAE(matrix, rec_matrix))
exp_var = (S ** 2) / (S.shape[0] - 1)
full_var = np.var(matrix, axis=0).sum()
explained_variance_ratio = exp_var / full_var # % of variance explained by each PC
if var == 1:
pass
else:
explained_variance_ratio = explained_variance_ratio[::-1]
ratio_cumsum = np.cumsum(explained_variance_ratio)
msg = ' This info makes sense when the matrix is mean centered '
msg += '(temp-mean scaling)'
print(msg)
lw = 2; alpha = 0.4
fig = plt.figure(figsize=vip_figsize)
fig.subplots_adjust(wspace=0.4)
ax1 = plt.subplot2grid((1, 3), (0, 0), colspan=2)
ax1.step(range(explained_variance_ratio.shape[0]),
explained_variance_ratio, alpha=alpha, where='mid',
label='Individual EVR', lw=lw)
ax1.plot(ratio_cumsum, '.-', alpha=alpha,
label='Cumulative EVR', lw=lw)
ax1.legend(loc='best', frameon=False, fontsize='medium')
ax1.set_ylabel('Explained variance ratio (EVR)')
ax1.set_xlabel('Principal components')
ax1.grid(linestyle='solid', alpha=0.2)
ax1.set_xlim(-10, explained_variance_ratio.shape[0] + 10)
ax1.set_ylim(0, 1)
trunc = 20
ax2 = plt.subplot2grid((1, 3), (0, 2), colspan=1)
# plt.setp(ax2.get_yticklabels(), visible=False)
ax2.step(range(trunc), explained_variance_ratio[:trunc], alpha=alpha,
where='mid', lw=lw)
ax2.plot(ratio_cumsum[:trunc], '.-', alpha=alpha, lw=lw)
ax2.set_xlabel('Principal components')
ax2.grid(linestyle='solid', alpha=0.2)
ax2.set_xlim(-2, trunc + 2)
ax2.set_ylim(0, 1)
msg = ' Cumulative explained variance ratio for {} PCs = {:.5f}'
# plt.savefig('figure.pdf', dpi=300, bbox_inches='tight')
print(msg.format(ncomp, ratio_cumsum[ncomp - 1]))
# --------------------------------------------------------------------------
if matrix.ndim != 2:
raise TypeError('Input matrix is not a 2d array')
if usv:
if mode not in ('lapack', 'arpack', 'randsvd', 'cupy', 'randcupy',
'pytorch', 'randpytorch'):
msg = "Returning USV is supported with modes lapack, arpack, "
msg += "randsvd, cupy, randcupy, pytorch or randpytorch"
raise ValueError(msg)
if ncomp > min(matrix.shape[0], matrix.shape[1]):
msg = '{} PCs cannot be obtained from a matrix with size [{},{}].'
msg += ' Increase the size of the patches or request less PCs'
raise RuntimeError(msg.format(ncomp, matrix.shape[0], matrix.shape[1]))
if mode == 'eigen':
# building C as np.dot(matrix.T,matrix) is slower and takes more memory
C = np.dot(matrix, matrix.T) # covariance matrix
e, EV = linalg.eigh(C) # EVals and EVs
pc = np.dot(EV.T, matrix) # PCs using a compact trick when cov is MM'
V = pc[::-1] # reverse since we need the last EVs
S = np.sqrt(np.abs(e)) # SVals = sqrt(EVals)
S = S[::-1] # reverse since EVals go in increasing order
if debug:
reconstruction(ncomp, None, S, None)
for i in range(V.shape[1]):
V[:, i] /= S # scaling EVs by the square root of EVals
V = V[:ncomp]
if verbose:
print('Done PCA with numpy linalg eigh functions')
elif mode == 'lapack':
# n_frames is usually smaller than n_pixels. In this setting taking the SVD of M'
# and keeping the left (transposed) SVs is faster than taking the SVD of M (right SVs)
U, S, V = linalg.svd(matrix.T, full_matrices=False)
if debug:
reconstruction(ncomp, U, S, V)
V = V[:ncomp] # we cut projection matrix according to the # of PCs
U = U[:, :ncomp]
S = S[:ncomp]
if verbose:
print('Done SVD/PCA with numpy SVD (LAPACK)')
elif mode == 'arpack':
U, S, V = svds(matrix, k=ncomp)
if debug:
reconstruction(ncomp, U, S, V, -1)
if verbose:
print('Done SVD/PCA with scipy sparse SVD (ARPACK)')
elif mode == 'randsvd':
U, S, V = randomized_svd(matrix, n_components=ncomp, n_iter=2,
transpose='auto', random_state=random_state)
if debug:
reconstruction(ncomp, U, S, V)
if verbose:
print('Done SVD/PCA with randomized SVD')
elif mode == 'cupy':
if no_cupy:
raise RuntimeError('Cupy is not installed')
a_gpu = cupy.array(matrix)
a_gpu = cupy.asarray(a_gpu) # move the data to the current device
u_gpu, s_gpu, vh_gpu = cupy.linalg.svd(a_gpu, full_matrices=True,
compute_uv=True)
V = vh_gpu[:ncomp]
if to_numpy:
V = cupy.asnumpy(V)
if usv:
S = s_gpu[:ncomp]
if to_numpy:
S = cupy.asnumpy(S)
U = u_gpu[:, :ncomp]
if to_numpy:
U = cupy.asnumpy(U)
if verbose:
print('Done SVD/PCA with cupy (GPU)')
elif mode == 'randcupy':
if no_cupy:
raise RuntimeError('Cupy is not installed')
U, S, V = randomized_svd_gpu(matrix, ncomp, n_iter=2, lib='cupy')
if to_numpy:
V = cupy.asnumpy(V)
S = cupy.asnumpy(S)
U = cupy.asnumpy(U)
if debug:
reconstruction(ncomp, U, S, V)
if verbose:
print('Done randomized SVD/PCA with cupy (GPU)')
elif mode == 'eigencupy':
if no_cupy:
raise RuntimeError('Cupy is not installed')
a_gpu = cupy.array(matrix)
a_gpu = cupy.asarray(a_gpu) # move the data to the current device
C = cupy.dot(a_gpu, a_gpu.T) # covariance matrix
e, EV = cupy.linalg.eigh(C) # eigenvalues and eigenvectors
pc = cupy.dot(EV.T, a_gpu) # PCs using a compact trick when cov is MM'
V = pc[::-1] # reverse since last eigenvectors are the ones we want
S = cupy.sqrt(e)[::-1] # reverse since eigenvalues are in increasing order
if debug:
reconstruction(ncomp, None, S, None)
for i in range(V.shape[1]):
V[:, i] /= S # scaling by the square root of eigenvalues
V = V[:ncomp]
if to_numpy:
V = cupy.asnumpy(V)
if verbose:
print('Done PCA with cupy eigh function (GPU)')
elif mode == 'pytorch':
if no_torch:
raise RuntimeError('Pytorch is not installed')
a_gpu = torch.Tensor.cuda(torch.from_numpy(matrix.astype('float32').T))
u_gpu, s_gpu, vh_gpu = torch.svd(a_gpu)
V = vh_gpu[:ncomp]
S = s_gpu[:ncomp]
U = torch.transpose(u_gpu, 0, 1)[:ncomp]
if to_numpy:
V = np.array(V)
S = np.array(S)
U = np.array(U)
if verbose:
print('Done SVD/PCA with pytorch (GPU)')
elif mode == 'eigenpytorch':
if no_torch:
raise RuntimeError('Pytorch is not installed')
a_gpu = torch.Tensor.cuda(torch.from_numpy(matrix.astype('float32')))
C = torch.mm(a_gpu, torch.transpose(a_gpu, 0, 1))
e, EV = torch.eig(C, eigenvectors=True)
V = torch.mm(torch.transpose(EV, 0, 1), a_gpu)
S = torch.sqrt(e[:, 0])
if debug:
reconstruction(ncomp, None, S, None)
for i in range(V.shape[1]):
V[:, i] /= S
V = V[:ncomp]
if to_numpy:
V = np.array(V)
if verbose:
print('Done PCA with pytorch eig function')
elif mode == 'randpytorch':
if no_torch:
raise RuntimeError('Pytorch is not installed')
U, S, V = randomized_svd_gpu(matrix, ncomp, n_iter=2, lib='pytorch')
if to_numpy:
V = np.array(V)
S = np.array(S)
U = np.array(U)
if debug:
reconstruction(ncomp, U, S, V)
if verbose:
print('Done randomized SVD/PCA with randomized pytorch (GPU)')
else:
raise ValueError('The SVD mode is not available')
if usv:
if mode == 'lapack':
return V.T, S, U.T
elif mode == 'pytorch':
if to_numpy:
return V.T, S, U.T
else:
return torch.transpose(V, 0, 1), S, torch.transpose(U, 0, 1)
else:
return U, S, V
else:
if mode == 'lapack':
return U.T
elif mode == 'pytorch':
return U
else:
return V
def calculate_correlation(data_loader,
w2g,
verbose=True,
lower=False,
metric='bures_cosine',
embedds='input',
numIters=50):
#### data_loader is a function that returns 2 lists of words and the scores
#### metric is a function that takes w1, w2 and calculate the score
word1, word2, targets = data_loader()
if lower:
word1 = [word.lower() for word in word1]
word2 = [word.lower() for word in word2]
mask = [(w1 in w2g.words_to_idxs and w2 in w2g.words_to_idxs)
for (w1, w2) in zip(word1, word2)]
word1, word2, targets = np.array(word1)[mask], np.array(
word2)[mask], np.array(targets)[mask]
distinct_words = set(np.concatenate([word1, word2]))
ndistinct = len(distinct_words)
nwords_dict = len([w in w2g.words_to_idxs for w in distinct_words])
if lower:
nwords_dict = len(
[w.lower() in w2g.words_to_idxs for w in distinct_words])
if verbose: print('# of pairs {} # words total {} # words in dictionary {}({}%)' \
.format(len(word1), ndistinct, nwords_dict, 100 * nwords_dict / (1. * ndistinct)))
word1_idxs = [w2g.words_to_idxs[w] for w in word1]
word2_idxs = [w2g.words_to_idxs[w] for w in word2]
scores = np.zeros((len(word1_idxs)))
if embedds == 'input':
if metric == 'bures_distance':
scores = -wb.batch_W2(w2g.means[word1_idxs],
w2g.means[word2_idxs],
w2g.vars[word1_idxs],
w2g.vars[word2_idxs],
numIters=numIters)[0]
elif metric == 'bures_cosine':
scores = wb.bures_cosine(w2g.means[word1_idxs],
w2g.means[word2_idxs],
w2g.vars[word1_idxs],
w2g.vars[word2_idxs],
numIters=numIters)
elif metric == 'bures_product':
scores = wb.batch_W2(w2g.means[word1_idxs],
w2g.means[word2_idxs],
w2g.vars[word1_idxs],
w2g.vars[word2_idxs],
prod=True,
numIters=numIters)[0]
else:
if metric == 'bures_distance':
scores = -wb.batch_W2(w2g.c_means[word1_idxs],
w2g.c_means[word2_idxs],
w2g.c_vars[word1_idxs],
w2g.c_vars[word2_idxs],
numIters=numIters)[0]
elif metric == 'bures_cosine':
scores = wb.bures_cosine(w2g.c_means[word1_idxs],
w2g.c_means[word2_idxs],
w2g.c_vars[word1_idxs],
w2g.c_vars[word2_idxs],
numIters=numIters)
elif metric == 'bures_product':
scores = \
wb.batch_W2(w2g.c_means[word1_idxs], w2g.c_means[word2_idxs], w2g.c_vars[word1_idxs], w2g.c_vars[word2_idxs],
prod=True, numIters=numIters)[0]
scores = cp.asnumpy(scores)
spr = scipy.stats.spearmanr(scores, targets)
if verbose:
print('Spearman correlation is {} with pvalue {}'.format(
spr.correlation, spr.pvalue))
pear = scipy.stats.pearsonr(scores, targets)
if verbose: print('Pearson correlation', pear)
spr_correlation = spr.correlation
pear_correlation = pear[0]
if np.any(np.isnan(scores)):
spr_correlation = np.NAN
pear_correlation = np.NAN
return scores, spr_correlation, pear_correlation
def to_numpy(a):
if isinstance(a, chainer.Variable):
a = a.data
if isinstance(a, cp.ndarray):
a = cp.asnumpy(a)
return np.ascontiguousarray(a)
def test_timedelta_to_typecast(data, cast_dtype):
psr = pd.Series(cp.asnumpy(data) if isinstance(data, cp.ndarray) else data)
gsr = cudf.Series(data)
assert_eq(psr.astype(cast_dtype), gsr.astype(cast_dtype))
def to_cpu(self, arr):
return cupy.asnumpy(arr)
from time import perf_counter
import numpy as np
import cupy as cp
# import cupyx.scipy.linalg
trials = 5
times = []
N = 1000
X = cp.random.randn(N, N, dtype=np.float64)
for _ in range(trials):
t0 = perf_counter()
u, s, v = cp.linalg.decomposition.svd(X)
# Y = cp.matmul(X, X)
# lu, piv = cupyx.scipy.linalg.lu_factor(X, check_finite=False)
cp.cuda.Device(0).synchronize()
times.append(perf_counter() - t0)
print("Execution time: ", min(times))
print(cp.asnumpy(s).sum())
print("CuPy version: ", cp.__version__)
def assert_allclose(actual, desired, rtol=1e-7, atol=0, err_msg='',
verbose=True):
numpy.testing.assert_allclose(
cupy.asnumpy(actual), cupy.asnumpy(desired),
rtol=rtol, atol=atol, err_msg=err_msg, verbose=verbose)
CTS = len(test) // CHUNK
if len(test) % CHUNK != 0:
CTS += 1
for j in range(CTS):
a = j * CHUNK
b = (j + 1) * CHUNK
b = min(b, len(test))
print('chunk', a, 'to', b)
# 矩阵相乘计算相似度（features * features.T）
# COSINE SIMILARITY DISTANCE，余弦相似度的知识，cos = (a*b) / (|a|*|b|)
cts = cupy.matmul(text_embeddings, text_embeddings[a:b].T).T
for k in range(b - a):
IDX = cupy.where(cts[k, ] > CFG.txt_thres)[0] # 根据阈值确定匹配商品
o = test.iloc[cupy.asnumpy(IDX)].posting_id.values
preds2.append(o)
del model, text_embeddings
_ = gc.collect()
test["preds2"] = preds2
# # 3.PHASH
# In[24]:
# 数据csv自带的phash做匹配，phash相同则认为商品匹配
tmp = test.groupby('image_phash').posting_id.agg('unique').to_dict()
test['preds3'] = test.image_phash.map(tmp)
def _make_positive_indices(self, impl, args, kw):
ks = [k for k, v in kw.items() if v in _unsigned_dtypes]
for k in ks:
kw[k] = numpy.intp
mask = cupy.asnumpy(impl(self, *args, **kw)) >= 0
return numpy.nonzero(mask)
def nanpercentile(a, *args, **kwargs):
""" For cupy v0.6.0 compatibility """
return cp.array(np.nanpercentile(cp.asnumpy(a), *args, **kwargs))
def train(train_vis, train_dep, train_labels, N, num_label, model, opt, args):
# 訓練
pbar = ProgressBar(N)
# 学習するサンプルのバッチをランダムに取る
# 当面はbatchsize=1
lstm_correct_cnt = 0
sum_frame = 0
total_loss = np.array(0, dtype=np.float32)
# 最適化器の準備
opt.setup(model)
# 混同行列
conf_array = np.zeros((num_label, num_label), dtype=np.int32)
"""
ここから訓練本体
動画ごとのフレーム数が異なるため、バッチ数は1で固定
"""
for i in range(0, N):
x_vis_batch = xp.asarray(train_vis[i], dtype=np.float32)
x_dep_batch = xp.asarray(train_dep[i], dtype=np.float32)
y_batch = xp.asarray(train_labels[i], dtype=np.int32)
# 学習の初期化
opt.zero_grads()
accum_loss = 0
video_length = min(x_vis_batch.shape[0], x_dep_batch.shape[0])
sum_frame += video_length
c = Variable(xp.zeros((1, args.num_hunits)).astype(np.float32))
h = Variable(xp.zeros((1, args.num_hunits)).astype(np.float32))
for t in xrange(video_length):
x_vis_frame = x_vis_batch[t,:,:]
x_dep_frame = x_dep_batch[t,:,:]
y_frame = y_batch[:,t]
# 順伝播
loss_i, pred, c, h = model.forward_one_step(
x_vis_frame, x_dep_frame, y_frame, c, h)
accum_loss += loss_i
if args.gpu >= 0:
import cupy
total_loss += cupy.asnumpy(loss_i.data)
else:
total_loss += loss_i.data
# 正解フレーム数をカウント
pred = xp.argmax(pred.data)
if(pred.tolist() == y_frame[0]):
lstm_correct_cnt += 1
conf_array[y_frame[0], pred.tolist()] += 1
# truncated BPTTによる逆伝播
if (t + 1) % args.bprop_len == 0 or t == video_length -1: # Run truncated BPTT
opt.zero_grads()
accum_loss.backward()
accum_loss.unchain_backward() # truncate
accum_loss = Variable(xp.zeros((), dtype=np.float32))
opt.clip_grads(args.grad_clip)
# パラメタの更新
opt.update()
if args.opt in ['AdaGrad', 'MomentumSGD']:
opt.weight_decay(decay=args.weight_decay)
pbar.update(i + 1 if (i + 1) < N else N)
"""
正答率を計算
"""
lstm_accuracy = lstm_correct_cnt / float(sum_frame)
return total_loss / sum_frame, lstm_accuracy, conf_array
def dicToDF(self, dic):
for key in dic.keys():
if 'cupy' in str(type(dic[key])):
dic[key] = xp.asnumpy(dic[key])
return pd.DataFrame(dic)
def xparray(data):
if args.gpu >= 0:
return cupy.asnumpy(data)
else:
return data
def beam_search(model, X, params, return_alphas=False, eos_sym=0, null_sym=2, model_ensemble=False, n_models=0):
"""
Beam search method for Cond models.
(https://en.wikibooks.org/wiki/Artificial_Intelligence/Search/Heuristic_search/Beam_search)
The algorithm in a nutshell does the following:
1. k = beam_size
2. open_nodes = [[]] * k
3. while k > 0:
3.1. Given the inputs, get (log) probabilities for the outputs.
3.2. Expand each open node with all possible output.
3.3. Prune and keep the k best nodes.
3.4. If a sample has reached the <eos> symbol:
3.4.1. Mark it as final sample.
3.4.2. k -= 1
3.5. Build new inputs (state_below) and go to 1.
4. return final_samples, final_scores
:param model: Model to use
:param X: Model inputs
:param params: Search parameters
:param return_alphas: Whether we should return attention weights or not.
:param eos_sym: <eos> symbol
:param null_sym: <null> symbol
:param model_ensemble: Whether we are using several models in an ensemble
:param n_models; Number of models in the ensemble.
:return: UNSORTED list of [k_best_samples, k_best_scores] (k: beam size)
"""
k = params['beam_size']
samples = []
sample_scores = []
pad_on_batch = params['pad_on_batch']
dead_k = 0 # samples that reached eos
live_k = 1 # samples that did not yet reach eos
hyp_samples = [[]] * live_k
hyp_scores = cp.zeros(live_k, dtype='float32')
ret_alphas = return_alphas or params['pos_unk']
if ret_alphas:
sample_alphas = []
hyp_alphas = [[]] * live_k
if pad_on_batch:
maxlen = int(len(X[params['dataset_inputs'][0]][0]) * params['output_max_length_depending_on_x_factor']) if \
params['output_max_length_depending_on_x'] else params['maxlen']
minlen = int(
len(X[params['dataset_inputs'][0]][0]) / params['output_min_length_depending_on_x_factor'] + 1e-7) if \
params['output_min_length_depending_on_x'] else 0
else:
minlen = int(np.argmax(X[params['dataset_inputs'][0]][0] == eos_sym) /
params['output_min_length_depending_on_x_factor'] + 1e-7) if \
params['output_min_length_depending_on_x'] else 0
maxlen = int(np.argmax(X[params['dataset_inputs'][0]][0] == eos_sym) * params[
'output_max_length_depending_on_x_factor']) if \
params['output_max_length_depending_on_x'] else params['maxlen']
maxlen = min(params['state_below_maxlen'] - 1, maxlen)
# we must include an additional dimension if the input for each timestep are all the generated "words_so_far"
if params['words_so_far']:
if k > maxlen:
raise NotImplementedError("BEAM_SIZE can't be higher than MAX_OUTPUT_TEXT_LEN on the current implementation.")
state_below = np.asarray([[null_sym]] * live_k) if pad_on_batch else np.asarray([np.zeros((maxlen, maxlen))] * live_k)
else:
state_below = np.asarray([null_sym] * live_k) if pad_on_batch else np.asarray([np.zeros(params['state_below_maxlen']) + null_sym] * live_k)
prev_out = [None] * n_models if model_ensemble else None
for ii in range(maxlen):
# for every possible live sample calc prob for every possible label
if params['optimized_search']: # use optimized search model if available
if model_ensemble:
[probs, prev_out, alphas] = model.predict_cond_optimized(X, state_below, params, ii, prev_out)
else:
[probs, prev_out] = model.predict_cond_optimized(X, state_below, params, ii, prev_out)
if ret_alphas:
alphas = prev_out[-1][0] # Shape: (k, n_steps)
prev_out = prev_out[:-1]
else:
probs = model.predict_cond(X, state_below, params, ii)
log_probs = cp.log(probs)
if minlen > 0 and ii < minlen:
log_probs[:, eos_sym] = -cp.inf
# total score for every sample is sum of -log of word prb
cand_scores = hyp_scores[:, None] - log_probs
cand_flat = cand_scores.flatten()
# Find the best options by calling argsort of flatten array
ranks_flat = cp.argsort(cand_flat)[:(k - dead_k)]
# Decypher flatten indices
voc_size = log_probs.shape[1]
trans_indices = ranks_flat // voc_size # index of row
word_indices = ranks_flat % voc_size # index of col
costs = cand_flat[ranks_flat]
best_cost = costs[0]
if cupy:
trans_indices = cp.asnumpy(trans_indices)
word_indices = cp.asnumpy(word_indices)
if ret_alphas:
alphas = cp.asnumpy(alphas)
# Form a beam for the next iteration
new_hyp_samples = []
new_trans_indices = []
new_hyp_scores = cp.zeros(k - dead_k, dtype='float32')
if ret_alphas:
new_hyp_alphas = []
for idx, [ti, wi] in list(enumerate(zip(trans_indices, word_indices))):
if params['search_pruning']:
if costs[idx] < k * best_cost:
new_hyp_samples.append(hyp_samples[ti] + [wi])
new_trans_indices.append(ti)
new_hyp_scores[idx] = copy.copy(costs[idx])
if ret_alphas:
new_hyp_alphas.append(hyp_alphas[ti] + [alphas[ti]])
else:
dead_k += 1
else:
new_hyp_samples.append(hyp_samples[ti] + [wi])
new_trans_indices.append(ti)
new_hyp_scores[idx] = copy.copy(costs[idx])
if ret_alphas:
new_hyp_alphas.append(hyp_alphas[ti] + [alphas[ti]])
# check the finished samples
new_live_k = 0
hyp_samples = []
hyp_scores = []
hyp_alphas = []
indices_alive = []
for idx in range(len(new_hyp_samples)):
if new_hyp_samples[idx][-1] == eos_sym: # finished sample
samples.append(new_hyp_samples[idx])
sample_scores.append(new_hyp_scores[idx])
if ret_alphas:
sample_alphas.append(new_hyp_alphas[idx])
dead_k += 1
else:
indices_alive.append(new_trans_indices[idx])
new_live_k += 1
hyp_samples.append(new_hyp_samples[idx])
hyp_scores.append(new_hyp_scores[idx])
if ret_alphas:
hyp_alphas.append(new_hyp_alphas[idx])
hyp_scores = cp.array(np.asarray(hyp_scores, dtype='float32'), dtype='float32')
live_k = new_live_k
if new_live_k < 1:
break
if dead_k >= k:
break
state_below = np.asarray(hyp_samples, dtype='int64')
state_below = np.hstack((np.zeros((state_below.shape[0], 1), dtype='int64') + null_sym, state_below)) \
if pad_on_batch else \
np.hstack((np.zeros((state_below.shape[0], 1), dtype='int64') + null_sym,
state_below,
np.zeros((state_below.shape[0],
max(params['state_below_maxlen'] - state_below.shape[1] - 1, 0)), dtype='int64')))
# we must include an additional dimension if the input for each timestep are all the generated words so far
if params['words_so_far']:
state_below = np.expand_dims(state_below, axis=0)
if params['optimized_search'] and ii > 0:
# filter next search inputs w.r.t. remaining samples
if model_ensemble:
for n_model in range(n_models):
# filter next search inputs w.r.t. remaining samples
for idx_vars in range(len(prev_out[n_model])):
prev_out[n_model][idx_vars] = prev_out[n_model][idx_vars][indices_alive]
else:
for idx_vars in range(len(prev_out)):
prev_out[idx_vars] = prev_out[idx_vars][indices_alive]
# dump every remaining one
if live_k > 0:
for idx in range(live_k):
samples.append(hyp_samples[idx])
sample_scores.append(hyp_scores[idx])
if ret_alphas:
sample_alphas.append(hyp_alphas[idx])
if ret_alphas:
return samples, sample_scores, np.asarray(sample_alphas)
else:
return samples, sample_scores, None
def corner_peaks(
image,
min_distance=1,
threshold_abs=None,
threshold_rel=None,
exclude_border=True,
indices=True,
num_peaks=np.inf,
footprint=None,
labels=None,
*,
num_peaks_per_label=np.inf,
p_norm=np.inf,
):
"""Find peaks in corner measure response image.
This differs from `skimage.feature.peak_local_max` in that it suppresses
multiple connected peaks with the same accumulator value.
Parameters
----------
image : ndarray
Input image.
min_distance : int, optional
The minimal allowed distance separating peaks.
* : *
See :py:meth:`skimage.feature.peak_local_max`.
p_norm : float
Which Minkowski p-norm to use. Should be in the range [1, inf].
A finite large p may cause a ValueError if overflow can occur.
``inf`` corresponds to the Chebyshev distance and 2 to the
Euclidean distance.
Returns
-------
output : ndarray or ndarray of bools
* If `indices = True` : (row, column, ...) coordinates of peaks.
* If `indices = False` : Boolean array shaped like `image`, with peaks
represented by True values.
See also
--------
skimage.feature.peak_local_max
Notes
-----
.. versionchanged:: 0.18
The default value of `threshold_rel` has changed to None, which
corresponds to letting `skimage.feature.peak_local_max` decide on the
default. This is equivalent to `threshold_rel=0`.
The `num_peaks` limit is applied before suppression of connected peaks.
To limit the number of peaks after suppression, set `num_peaks=np.inf` and
post-process the output of this function.
Examples
--------
>>> from cupyimg.skimage.feature import peak_local_max
>>> response = cp.zeros((5, 5))
>>> response[2:4, 2:4] = 1
>>> response
array([[0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0.],
[0., 0., 1., 1., 0.],
[0., 0., 1., 1., 0.],
[0., 0., 0., 0., 0.]])
>>> peak_local_max(response)
array([[2, 2],
[2, 3],
[3, 2],
[3, 3]])
>>> corner_peaks(response)
array([[2, 2]])
"""
if cp.isinf(num_peaks):
num_peaks = None
# Get the coordinates of the detected peaks
coords = peak_local_max(
image,
min_distance=min_distance,
threshold_abs=threshold_abs,
threshold_rel=threshold_rel,
exclude_border=exclude_border,
num_peaks=np.inf,
footprint=footprint,
labels=labels,
num_peaks_per_label=num_peaks_per_label,
)
if len(coords):
# TODO: modify to do KDTree on the GPU (cuSpatial?)
coords = cp.asnumpy(coords)
# Use KDtree to find the peaks that are too close to each other
tree = spatial.cKDTree(coords)
rejected_peaks_indices = set()
for idx, point in enumerate(coords):
if idx not in rejected_peaks_indices:
candidates = tree.query_ball_point(point,
r=min_distance,
p=p_norm)
candidates.remove(idx)
rejected_peaks_indices.update(candidates)
# Remove the peaks that are too close to each other
coords = np.delete(coords, tuple(rejected_peaks_indices),
axis=0)[:num_peaks]
coords = cp.asarray(coords)
if indices:
return coords
peaks = cp.zeros_like(image, dtype=bool)
peaks[tuple(coords.T)] = True
return peaks
def walsh_transform(self,keys=None):
if keys is None:
keys = ['kernel'] + list(self.constraints.keys())
else:
keys = keys
is_stored = dict()
for key in keys:
is_stored[key] = False
if os.path.exists(self.fname):
with h5py.File(self.fname,mode='r') as f:
for key in keys:
try:
if '3' in f[key].keys():
is_stored[key] = True
if key == 'depth':
res = f['depth']['constraint'][:] - self.constraints['depth']
res = np.linalg.norm(res)/np.linalg.norm(self.constraints['depth'])
if res > 1.0e-3:
is_stored[key] = False
except KeyError:
continue
self._gen_walsh_matrix()
logn = int(np.ceil(np.log2(self._nx*self._ny*self._nz)))
norm_walsh = 1./(np.sqrt(2)**logn)
blocks = ['0','1','2','3']
matvec_op = {'kernel':self.kernel_op.gtoep.matvec,
'dx': lambda x: self._dxyzvec(x,key='dx'),
'dy': lambda x: self._dxyzvec(x,key='dy'),
'dz': lambda x: self._dxyzvec(x,key='dz'),
'refer': lambda x: self._diagvec(x,diag=self.constraints['refer']),
'depth': lambda x: self._diagvec(x,diag=np.sqrt(self.constraints['depth']))
}
is_stored['refer'] = True
for key in keys:
if is_stored[key]:
print('walsh transformation of {} already exists.'.format(key))
continue
print('performing walsh transformation on {}.'.format(key))
step = self.nx*self.ny*self.nz // 4
if key == 'depth':
step = self._nz
with h5py.File(self.fname,mode='a') as f:
try:
del f[key]
except KeyError:
pass
dxyz_group = f.create_group(key)
walsh_group = f['walsh_matrix']
for i in range(4):
print("\t progress {}/4".format(i))
part_walsh = walsh_group[blocks[i]][:]
if key == 'depth':
part_walsh = walsh_group[blocks[i]][:self._nz]
part_walsh = matvec_op[key](part_walsh)
with cp.cuda.Device(2):
res = cp.zeros((step,step))
j = 0
while j*step < part_walsh.shape[1]:
tmp_block_gpu = cp.asarray(part_walsh[:,j*step:(j+1)*step])
res += tmp_block_gpu @ tmp_block_gpu.T
j += 1
res = cp.asnumpy(res)
if key in self._smooth_components:
res[np.abs(res)<1.0e-1*norm_walsh] = 0.
tmp_block_gpu = None
mempool = cp.get_default_memory_pool()
pinned_mempool = cp.get_default_pinned_memory_pool()
mempool.free_all_blocks()
pinned_mempool.free_all_blocks()
dxyz_group.create_dataset(blocks[i],data=res)
if ('depth' in keys) and (not is_stored['depth']):
with h5py.File(self.fname,mode='a') as f:
try:
del f['depth_constraint']
except KeyError:
pass
dxyz_group = f['depth']
dxyz_group.create_dataset('constraint',data=self.constraints['depth'])
def get_good_channels(raw_data=None, probe=None, params=None):
"""
of the channels indicated by the user as good (chanMap)
further subset those that have a mean firing rate above a certain value
(default is ops.minfr_goodchannels = 0.1Hz)
needs the same filtering parameters in ops as usual
also needs to know where to start processing batches (twind)
and how many channels there are in total (NchanTOT)
"""
fs = params.fs
fshigh = params.fshigh
fslow = params.fslow
Nbatch = get_Nbatch(raw_data, params)
NT = params.NT
spkTh = params.spkTh
nt0 = params.nt0
minfr_goodchannels = params.minfr_goodchannels
chanMap = probe.chanMap
# Nchan = probe.Nchan
NchanTOT = len(chanMap)
ich = []
k = 0
ttime = 0
# skip every 100 batches
for ibatch in tqdm(range(0, Nbatch, int(ceil(Nbatch / 100))),
desc="Finding good channels"):
i = NT * ibatch
buff = raw_data[:, i:i + NT]
assert np.isfortran(buff)
if buff.size == 0:
break
# Put on GPU.
buff = cp.asarray(buff, dtype=np.float32)
assert cp.isfortran(buff)
datr = gpufilter(buff,
chanMap=chanMap,
fs=fs,
fshigh=fshigh,
fslow=fslow)
# very basic threshold crossings calculation
s = cp.std(datr, axis=0)
datr = datr / s # standardize each channel ( but don't whiten)
mdat = my_min(
datr, 30,
0) # get local minima as min value in +/- 30-sample range
# take local minima that cross the negative threshold
xi, xj = cp.nonzero((datr < mdat + 1e-3) & (datr < spkTh))
# filtering may create transients at beginning or end. Remove those.
xj = xj[(xi >= nt0) & (xi <= NT - nt0)]
# collect the channel identities for the detected spikes
ich.append(xj)
k += xj.size
# keep track of total time where we took spikes from
ttime += datr.shape[0] / fs
ich = cp.concatenate(ich)
# count how many spikes each channel got
nc, _ = cp.histogram(ich, cp.arange(NchanTOT + 1))
# divide by total time to get firing rate
nc = nc / ttime
# keep only those channels above the preset mean firing rate
igood = cp.asnumpy(nc >= minfr_goodchannels)
logger.info('Found %d threshold crossings in %2.2f seconds of data.' %
(k, ttime))
logger.info('Found %d/%d bad channels.' % (np.sum(~igood), len(igood)))
return igood
print(
str(i + 1).zfill(6) + ", " + str(x_G) + ", " + str(y_G) + ", " +
str(z_G))
cp_dens_pre = cp.roll(cp_dens_pre, int(row / 2) - x_G, axis=2)
cp_dens_pre = cp.roll(cp_dens_pre, int(col / 2) - y_G, axis=1)
cp_dens_pre = cp.roll(cp_dens_pre, int(sta / 2) - z_G, axis=0)
cp_sup = cp.roll(cp_sup, int(row / 2) - x_G, axis=2)
cp_sup = cp.roll(cp_sup, int(row / 2) - y_G, axis=1)
cp_sup = cp.roll(cp_sup, int(sta / 2) - z_G, axis=0)
#電子密度の出力
if ((i + 1) % int(output_interval) == 0):
np_dens_pre_real = cp.asnumpy(cp_dens_pre_real) #cupy配列 ⇒ numpy配列に変換
with mrcfile.new(header + "_" + str(i + 1).zfill(6) + '_rdens.mrc',
overwrite=True) as mrc:
mrc.set_data(np_dens_pre_real)
mrc.close
# tifffile.imsave(header + "_" + str(i+1).zfill(6) + '_rdens.mrc' ,np_dens_pre_real)
#最後の電子密度とパターンの出力
if (i + 1 == int(iteration) + int(additional_iteration)):
R_dens = cp_dens_pre
R_dens.real = R_dens.real * cp_sup
R_dens.imag[:, :, :] = 0
R_structure_factor = cp.fft.fftn(R_dens, axes=(0, 1, 2), norm="ortho")
R_structure_factor = cp.fft.fftshift(R_structure_factor)
def test_compare_xp_gpu(self):
noisyimg_gpu = cp.asarray(self.noisyimg)
imgivar_gpu = cp.asarray(self.imgivar)
A4_gpu = cp.asarray(self.A4)
# Compare the "signal" decorrelation method
flux0, ivar0, R0 = ex2d_patch(self.noisyimg,
self.imgivar,
self.A4,
decorrelate='signal')
flux1_gpu, ivar1_gpu, R1_gpu = xp_ex2d_patch(noisyimg_gpu,
imgivar_gpu,
A4_gpu,
decorrelate='signal')
flux1 = cp.asnumpy(flux1_gpu)
ivar1 = cp.asnumpy(ivar1_gpu)
R1 = cp.asnumpy(R1_gpu)
eps_double = np.finfo(np.float64).eps
where = np.where(
~np.isclose(flux0, flux1, rtol=1e5 * eps_double, atol=0))
np.testing.assert_allclose(flux0,
flux1,
rtol=1e5 * eps_double,
atol=0,
err_msg=f"where: {where}")
self.assertTrue(
np.allclose(ivar0, ivar1, rtol=1e3 * eps_double, atol=0))
self.assertTrue(
np.allclose(np.diag(R0),
np.diag(R1),
rtol=1e2 * eps_double,
atol=1e3 * eps_double))
self.assertTrue(
np.allclose(
np.abs(flux0 - flux1) / np.sqrt(1. / ivar0 + 1. / ivar1),
np.zeros_like(flux0)))
# Compare the "noise" decorrelation method
flux0, ivar0, R0 = ex2d_patch(self.noisyimg,
self.imgivar,
self.A4,
decorrelate='noise')
flux1_gpu, ivar1_gpu, R1_gpu = xp_ex2d_patch(noisyimg_gpu,
imgivar_gpu,
A4_gpu,
decorrelate='noise')
flux1 = cp.asnumpy(flux1_gpu)
ivar1 = cp.asnumpy(ivar1_gpu)
R1 = cp.asnumpy(R1_gpu)
self.assertTrue(
np.allclose(flux0, flux1, rtol=1e5 * eps_double, atol=0))
self.assertTrue(
np.allclose(ivar0, ivar1, rtol=1e3 * eps_double, atol=0))
self.assertTrue(
np.allclose(np.diag(R0),
np.diag(R1),
rtol=1e2 * eps_double,
atol=0))
self.assertTrue(
np.allclose(
np.abs(flux0 - flux1) / np.sqrt(1. / ivar0 + 1. / ivar1),
np.zeros_like(flux0)))
def assert_array_almost_equal(x, y, decimal=6, err_msg='', verbose=True):
numpy.testing.assert_array_almost_equal(
cupy.asnumpy(x), cupy.asnumpy(y), decimal=decimal,
err_msg=err_msg, verbose=verbose)
def Allreduce_mean(self, x, **kwargs):
"""Multi-process multi-GPU based mean."""
src = self.pool.reduce_mean(x, **kwargs)
mean = self.mpi.Allreduce(cp.asnumpy(src)) / self.mpi.size
return cp.asarray(mean)
def assert_arrays_almost_equal_nulp(x, y, nulp=1):
numpy.testing.assert_arrays_almost_equal_nulp(
cupy.asnumpy(x), cupy.asnumpy(y), nulp=nulp)
)
print(output)
if before_output is not None:
min_batch_size = min(before_output.shape[1], output.shape[1])
print(
"equal",
numpy.all(before_output[:, :min_batch_size] == output[:, :min_batch_size]),
)
assert numpy.all(
before_output[:, :min_batch_size] == output[:, :min_batch_size]
)
before_output = output
expected = cupy.array(
fast_generate(
length=length,
x=model.xp.copy(base_x),
l_array=model.xp.copy(model.xp.transpose(base_l_array, [1, 0, 2])),
h=model.xp.copy(base_hidden),
**params,
)
)
if output.shape == expected.shape:
print("equal", numpy.all(output == cupy.asnumpy(expected)))
assert numpy.all(output == cupy.asnumpy(expected))
else:
print(expected)
def calc_log_prior_total_det(self):
self.log_prior_det_val = 0
self.log_total_det_val = 0
blocks = ['0','1','2','3']
prior_eigs = np.zeros(self._nx*self._ny*self._nz)
total_eigs = np.zeros(self._nx*self._ny*self._nz)
step = self._nx*self._ny*self._nz//4
try:
depth_weight = self._weights['depth']
except KeyError:
depth_weight = 1.
with h5py.File(self.fname,mode='r') as f:
if 'depth' in self._weights.keys():
depth_walsh = f['depth']['0'][:]
for i_b,block in enumerate(blocks):
tmp_block = np.zeros((step,step))
for dxyz_name in self._smooth_components:
try:
dxyz_walsh = f[dxyz_name][block][:].reshape(step//self._nz,
self._nz,
step//self._nz,
self._nz)
ein_path = np.einsum_path('mi,xiyj,jn->xmyn',
depth_walsh.T,
dxyz_walsh,
depth_walsh,
optimize='optimal')[0]
tmp_multi = np.einsum('mi,xiyj,jn->xmyn',
depth_walsh.T,
dxyz_walsh,
depth_walsh,
optimize=ein_path)
tmp_block += depth_weight*self._weights[dxyz_name]*tmp_multi.reshape(step,step)
except KeyError:
pass
if 'refer' in self._weights.keys():
tmp_multi_small = depth_walsh.T@depth_walsh
for i in range(step//self._nz):
tmp_block[i*self._nz:(i+1)*self._nz,
i*self._nz:(i+1)*self._nz] += depth_weight*self._weights['refer']*tmp_multi_small
with cp.cuda.Device(2):
tmp_block_gpu = cp.asarray(tmp_block,dtype=np.float32)
eigs = cp.linalg.eigvalsh(tmp_block_gpu)
prior_eigs[i_b*step:(i_b+1)*step] = cp.asnumpy(eigs)
self.log_prior_det_val += cp.asnumpy(cp.sum(cp.log(eigs)))
tmp_block_gpu = None
eigs = None
free_gpu()
tmp_block += self._weights['obs']*f['kernel'][block][:]
with cp.cuda.Device(2):
tmp_block_gpu = cp.asarray(tmp_block,dtype=np.float32)
eigs = cp.linalg.eigvalsh(tmp_block_gpu)
total_eigs[i_b*step:(i_b+1)*step] = cp.asnumpy(eigs)
self.log_total_det_val += cp.asnumpy(cp.sum(cp.log(eigs)))
tmp_block_gpu = None
eigs = None
free_gpu()
self.log_prior_det_val = cp.asnumpy(self.log_prior_det_val)
self.log_total_det_val = cp.asnumpy(self.log_total_det_val)
self.eigs = {'prior':prior_eigs,'total':total_eigs}
return self.log_prior_det_val,self.log_total_det_val
def to_numpy(a):
if isinstance(a, cupy.ndarray):
a = cupy.asnumpy(a)
return numpy.ascontiguousarray(a)
def dicToNumpy(self, dic):
for key in dic.keys():
if 'cupy' in str(type(dic[key])):
dic[key] = xp.asnumpy(dic[key])
return dic
