def one_module(p1, p2, flat=False):
"""
:param p1: actual coordinate of the point close to the origin
:param p2: actual coordinate of the point close to the end of first line
:param dx: number of pixel in a line
:param dy: number of pixel in a column
:param px: pixel size in x
:param py: pixel size in y
:return: 2x (dy+1)x(dx+1) array of corner position
"""
xyz1 = p1[1:4] / 1000.0 # in meter
xyz2 = p2[1:4] / 1000.0 # in meter
if flat:
xyz1[2] = 0
xyz2[2] = 0
x = xyz2 - xyz1
x /= numpy.linalg.norm(x)
z = numpy.array([0., 0., 1.])
y = numpy.cross(z, x)
z = numpy.cross(x, y)
m = pix * numpy.vstack((x, y, z))
vol_xyz = numpy.zeros((dy + 1, dx + 1, 3))
vol_xyz[:, :, 1] = numpy.outer(numpy.arange(0, dy + 1), numpy.ones(dx + 1))
vol_xyz[:, :, 0] = numpy.outer(numpy.ones(dy + 1), numpy.arange(dx + 1))
n = numpy.dot(vol_xyz, m) + xyz1
return numpy.ascontiguousarray(n[:, :, 1]), numpy.ascontiguousarray(n[:, :, 0]), numpy.ascontiguousarray(n[:, :, 2])
def filter2d_cv(src, kernel, anchor = (-1,-1) ):
if kernel.flags['C_CONTIGUOUS'] == False:
kernel = numpy.ascontiguousarray(kernel, 'f')
if src.flags['C_CONTIGUOUS'] == False:
src = numpy.ascontiguousarray(src)
dst = cv2.filter2D(src, -1, kernel, borderType=cv2.BORDER_CONSTANT)
return dst
def convert_leaves_all_probs_pred_old(image, leaves, all_probs, num_leaves, classifiers_fn=None):
global CLASSIFIERS, CLASSIFIER_FEATURE
if classifiers_fn is None:
classifiers_fn = os.environ['CLASSIFIERS_FN']
get_classifier_confidence = lambda x: x[0][0] * x[0][1]
if CLASSIFIERS is None:
all_classifiers = sorted(file_parse.load(classifiers_fn))
name_classifiers = []
for x in range(len(all_classifiers)):
if x < len(all_classifiers): # TODO(brandyn): Fix memory issue so that we can use the last classifier too
name_classifiers.append((all_classifiers[x][0],
classifiers.loads(all_classifiers[x][1])))
else:
name_classifiers.append((all_classifiers[x][0],
name_classifiers[-1][1]))
all_classifiers[x] = None # NOTE(Brandyn): This is done to save memory
print('ILP Classifiers %r' % ([x for x, _ in name_classifiers],))
CLASSIFIERS = [x for _, x in name_classifiers]
if CLASSIFIER_FEATURE is None:
CLASSIFIER_FEATURE = features.select_feature('bovw_hog')
feature = CLASSIFIER_FEATURE(np.ascontiguousarray(image[:, :, :3]))
preds = np.ascontiguousarray([get_classifier_confidence(classifier.predict(feature))
for classifier in CLASSIFIERS], dtype=np.float64)
out0 = imseg.convert_labels_to_integrals(leaves, num_leaves)
out1 = imseg.convert_all_probs_to_integrals(all_probs)
return preds, np.ascontiguousarray(np.dstack([out0, out1]))
def getPeakProperty(self, p_name):
"""
Return a numpy array containing the requested property.
"""
if not p_name in self.peak_properties:
raise MultiFitterException("No such property '" + p_name + "'")
# Properties that are calculated from other properties.
if(self.peak_properties[p_name] == "compound"):
# Return 0 length array if there are no localizations.
if(self.getNFit() == 0):
return numpy.zeros(0, dtype = numpy.float64)
# Peak significance calculation.
if(p_name == "significance"):
bg_sum = self.getPeakProperty("bg_sum")
fg_sum = self.getPeakProperty("fg_sum")
return fg_sum/numpy.sqrt(bg_sum)
# Floating point properties.
elif(self.peak_properties[p_name] == "float"):
values = numpy.ascontiguousarray(numpy.zeros(self.getNFit(), dtype = numpy.float64))
self.clib.mFitGetPeakPropertyDouble(self.mfit,
values,
ctypes.c_char_p(p_name.encode()))
return values
# Integer properties.
elif(self.peak_properties[p_name] == "int"):
values = numpy.ascontiguousarray(numpy.zeros(self.getNFit(), dtype = numpy.int32))
self.clib.mFitGetPeakPropertyInt(self.mfit,
values,
ctypes.c_char_p(p_name.encode()))
return values
def __init__(self,tracks,colors=None, line_width=2.,affine=None):
if affine==None:
self.affine=np.eye(4)
else: self.affine=affine
self.tracks_no=len(tracks)
self.tracks_len=[len(t) for t in tracks]
self.tracks=tracks
self.vertices = np.ascontiguousarray(np.concatenate(self.tracks).astype('f4'))
if colors==None:
self.colors = np.ascontiguousarray(np.ones((len(self.vertices),4)).astype('f4'))
else:
if isinstance(colors, (list, tuple)):
self.colors = np.tile(colors,(np.sum(self.tracks_len),1))
self.colors = np.ascontiguousarray(colors.astype('f4'))
self.vptr=self.vertices.ctypes.data
self.cptr=self.colors.ctypes.data
self.count=np.array(self.tracks_len, dtype=np.int32)
self.first=np.r_[0,np.cumsum(self.count)[:-1]].astype(np.int32)
self.firstptr=self.first.ctypes.data
self.countptr=self.count.ctypes.data
self.line_width=line_width
self.items=self.tracks_no
self.show_aabb = False
mn=self.vertices.min()
mx=self.vertices.max()
self.make_aabb((np.array([mn,mn,mn]),np.array([mx,mx,mx])),margin = 0)
def test_cmyk():
ref = imread(os.path.join(data_dir, 'color.png'))
img = Image.open(os.path.join(data_dir, 'color.png'))
img = img.convert('CMYK')
f = NamedTemporaryFile(suffix='.jpg')
fname = f.name
f.close()
img.save(fname)
try:
img.close()
except AttributeError: # `close` not available on PIL
pass
new = imread(fname)
ref_lab = rgb2lab(ref)
new_lab = rgb2lab(new)
for i in range(3):
newi = np.ascontiguousarray(new_lab[:, :, i])
refi = np.ascontiguousarray(ref_lab[:, :, i])
sim = ssim(refi, newi, dynamic_range=refi.max() - refi.min())
assert sim > 0.99
def test_mem_layout():
# Test with different memory layouts of X and y
X_ = np.asfortranarray(X)
clf = GradientBoostingClassifier(n_estimators=100, random_state=1)
clf.fit(X_, y)
assert_array_equal(clf.predict(T), true_result)
assert_equal(100, len(clf.estimators_))
X_ = np.ascontiguousarray(X)
clf = GradientBoostingClassifier(n_estimators=100, random_state=1)
clf.fit(X_, y)
assert_array_equal(clf.predict(T), true_result)
assert_equal(100, len(clf.estimators_))
y_ = np.asarray(y, dtype=np.int32)
y_ = np.ascontiguousarray(y_)
clf = GradientBoostingClassifier(n_estimators=100, random_state=1)
clf.fit(X, y_)
assert_array_equal(clf.predict(T), true_result)
assert_equal(100, len(clf.estimators_))
y_ = np.asarray(y, dtype=np.int32)
y_ = np.asfortranarray(y_)
clf = GradientBoostingClassifier(n_estimators=100, random_state=1)
clf.fit(X, y_)
assert_array_equal(clf.predict(T), true_result)
assert_equal(100, len(clf.estimators_))
def multiple_mahalanobis(effect, covariance):
"""Returns the squared Mahalanobis distance for a given set of samples
Parameters
----------
effect: array of shape (n_features, n_samples),
Each column represents a vector to be evaluated
covariance: array of shape (n_features, n_features, n_samples),
Corresponding covariance models stacked along the last axis
Returns
-------
sqd: array of shape (n_samples,)
the squared distances (one per sample)
"""
# check size
if effect.ndim == 1:
effect = effect[:, np.newaxis]
if covariance.ndim == 2:
covariance = covariance[:, :, np.newaxis]
if effect.shape[0] != covariance.shape[0]:
raise ValueError('Inconsistant shape for effect and covariance')
if covariance.shape[0] != covariance.shape[1]:
raise ValueError('Inconsistant shape for covariance')
# transpose and make contuguous for the sake of speed
Xt, Kt = np.ascontiguousarray(effect.T), np.ascontiguousarray(covariance.T)
# compute the inverse of the covariances
Kt = multiple_fast_inv(Kt)
# derive the squared Mahalanobis distances
sqd = np.sum(np.sum(Xt[:, :, np.newaxis] * Xt[:, np.newaxis] * Kt, 1), 1)
return sqd
def _compute_targets(rois, overlaps, labels):
"""Compute bounding-box regression targets for an image."""
# Indices of ground-truth ROIs
gt_inds = np.where(overlaps == 1)[0]
if len(gt_inds) == 0:
# Bail if the image has no ground-truth ROIs
return np.zeros((rois.shape[0], 5), dtype=np.float32)
# Indices of examples for which we try to make predictions
ex_inds = np.where(overlaps >= cfg.TRAIN.BBOX_THRESH)[0]
# Get IoU overlap between each ex ROI and gt ROI
ex_gt_overlaps = bbox_overlaps(
np.ascontiguousarray(rois[ex_inds, :], dtype=np.float),
np.ascontiguousarray(rois[gt_inds, :], dtype=np.float))
# Find which gt ROI each ex ROI has max overlap with:
# this will be the ex ROI's gt target
gt_assignment = ex_gt_overlaps.argmax(axis=1)
gt_rois = rois[gt_inds[gt_assignment], :]
ex_rois = rois[ex_inds, :]
targets = np.zeros((rois.shape[0], 5), dtype=np.float32)
targets[ex_inds, 0] = labels[ex_inds]
targets[ex_inds, 1:] = bbox_transform(ex_rois, gt_rois)
return targets
def propagate(image, labels, mask, weight):
"""Propagate the labels to the nearest pixels
image - gives the Z height when computing distance
labels - the labeled image pixels
mask - only label pixels within the mask
weight - the weighting of x/y distance vs z distance
high numbers favor x/y, low favor z
returns a label matrix and the computed distances
"""
if image.shape != labels.shape:
raise ValueError("Image shape %s != label shape %s"%(repr(image.shape),repr(labels.shape)))
if image.shape != mask.shape:
raise ValueError("Image shape %s != mask shape %s"%(repr(image.shape),repr(mask.shape)))
labels_out = np.zeros(labels.shape, np.int32)
distances = -np.ones(labels.shape,np.float64)
distances[labels > 0] = 0
labels_and_mask = np.logical_and(labels != 0, mask)
coords = np.argwhere(labels_and_mask)
i1,i2 = _propagate.convert_to_ints(0.0)
ncoords = coords.shape[0]
pq = np.column_stack((np.ones((ncoords,),int) * i1,
np.ones((ncoords,),int) * i2,
labels[labels_and_mask],
coords))
_propagate.propagate(np.ascontiguousarray(image,np.float64),
np.ascontiguousarray(pq,np.int32),
np.ascontiguousarray(mask,np.int8),
labels_out, distances, float(weight))
labels_out[labels > 0] = labels[labels > 0]
return labels_out,distances
def sorted_points_and_ids(xin, yin, zin, xperiod, yperiod, zperiod,
approx_xcell_size, approx_ycell_size, approx_zcell_size):
""" Determine the cell_id of every point, sort the points
according to cell_id, and return the sorted points as well as
the cell id indexing array.
Notes
-----
The x-coordinates of points with cell_id = icell are given by
xout[cell_id_indices[icell]:cell_id_indices[icell+1]].
"""
npts = len(xin)
num_xdivs, xcell_size = determine_cell_size(xperiod, approx_xcell_size)
num_ydivs, ycell_size = determine_cell_size(yperiod, approx_ycell_size)
num_zdivs, zcell_size = determine_cell_size(zperiod, approx_zcell_size)
ncells = num_xdivs*num_ydivs*num_zdivs
ix = digitized_position(xin, xcell_size, num_xdivs)
iy = digitized_position(yin, ycell_size, num_ydivs)
iz = digitized_position(zin, zcell_size, num_zdivs)
cell_ids = cell_id_from_cell_tuple(ix, iy, iz, num_ydivs, num_zdivs)
cell_id_sorting_indices = np.argsort(cell_ids)
cell_id_indices = np.searchsorted(cell_ids, np.arange(ncells),
sorter = cell_id_sorting_indices)
cell_id_indices = np.append(cell_id_indices, npts)
xout = np.ascontiguousarray(xin[cell_id_sorting_indices], dtype=np.float64)
yout = np.ascontiguousarray(yin[cell_id_sorting_indices], dtype=np.float64)
zout = np.ascontiguousarray(zin[cell_id_sorting_indices], dtype=np.float64)
cell_id_indices = np.ascontiguousarray(cell_id_indices, dtype=np.int64)
return xout, yout, zout, cell_id_indices
def dbscan(x, y, z, c, eps, min_points, z_factor = 0.5, verbose = True):
n_peaks = x.size
l = numpy.zeros(n_peaks, dtype = numpy.int32)
c_x = numpy.ascontiguousarray(x.astype(numpy.float32))
c_y = numpy.ascontiguousarray(y.astype(numpy.float32))
c_z = numpy.ascontiguousarray(z.astype(numpy.float32))*z_factor
c_c = numpy.ascontiguousarray(c.astype(numpy.int32))
c_l = numpy.ascontiguousarray(l)
lib_dbscan.dbscan(c_x,
c_y,
c_z,
c_c,
c_l,
n_peaks,
eps,
min_points,
int(verbose))
# Print number of clusters
if verbose:
n_clusters_ = len(set(c_l)) - (1 if -1 in c_l else 0)
print('Estimated number of clusters: %d' % n_clusters_)
return c_l
def _handle_input(image, selem, out, mask, out_dtype=None):
if image.dtype not in (np.uint8, np.uint16):
image = img_as_ubyte(image)
selem = np.ascontiguousarray(img_as_ubyte(selem > 0))
image = np.ascontiguousarray(image)
if mask is None:
mask = np.ones(image.shape, dtype=np.uint8)
else:
mask = img_as_ubyte(mask)
mask = np.ascontiguousarray(mask)
if out is None:
if out_dtype is None:
out_dtype = image.dtype
out = np.empty_like(image, dtype=out_dtype)
if image is out:
raise NotImplementedError("Cannot perform rank operation in place.")
is_8bit = image.dtype in (np.uint8, np.int8)
if is_8bit:
max_bin = 255
else:
max_bin = max(4, image.max())
bitdepth = int(np.log2(max_bin))
if bitdepth > 10:
warnings.warn("Bitdepth of %d may result in bad rank filter "
"performance due to large number of bins." % bitdepth)
return image, selem, out, mask, max_bin
def _set_data(self, coors, ngroups, conns, mat_ids, descs, nodal_bcs=None):
"""
Set mesh data.
Parameters
----------
coors : array
Coordinates of mesh nodes.
ngroups : array
Node groups.
conns : list of arrays
The array of mesh elements (connectivities) for each element group.
mat_ids : list of arrays
The array of material ids for each element group.
descs: list of strings
The element type for each element group.
nodal_bcs : dict of arrays, optional
The nodes defining regions for boundary conditions referred
to by the dict keys in problem description files.
"""
self.coors = nm.ascontiguousarray(coors)
if ngroups is None:
self.ngroups = nm.zeros((self.coors.shape[0],), dtype=nm.int32)
else:
self.ngroups = nm.ascontiguousarray(ngroups)
self.conns = [nm.asarray(conn, dtype=nm.int32) for conn in conns]
self.mat_ids = [nm.asarray(mat_id, dtype=nm.int32)
for mat_id in mat_ids]
self.descs = descs
self.nodal_bcs = get_default(nodal_bcs, {})
def __iter__(self):
''' This is were all the fun starts '''
x, y, z, g = self.a.shape
# for all seeds
for i in range(self.seed_no):
if self.seed_list == None:
rx = (x - 1) * np.random.rand()
ry = (y - 1) * np.random.rand()
rz = (z - 1) * np.random.rand()
seed = np.ascontiguousarray(
np.array([rx, ry, rz]), dtype=np.float64)
else:
seed = np.ascontiguousarray(
self.seed_list[i], dtype=np.float64)
# for all peaks
for ref in range(g):
track = eudx_both_directions(seed.copy(),
ref,
self.a,
self.ind,
self.odf_vertices,
self.a_low,
self.ang_thr,
self.step_sz,
self.total_weight,
self.max_points)
if track == None:
pass
else:
if track.shape[0] > 1:
yield track + self.voxel_shift
def connect_extrema(im_pos, target, markers, visualize=False): ''' im_pos : XYZ positions of each point in image formation (n x m x 3) ''' height, width,_ = im_pos.shape centroid = np.array(target) im_pos = np.ascontiguousarray(im_pos.astype(np.int16)) cost_map = np.ascontiguousarray(np.zeros([height, width], dtype=np.uint16)) extrema = dgn.geodesic_map_MPI(cost_map, im_pos, np.array(centroid, dtype=np.int16), 1, 1) cost_map = extrema[-1] trails = [] for m in markers: trail = dgn.geodesic_trail(cost_map.copy()+(32000*(im_pos[:,:,2]==0)).astype(np.uint16), np.array(m, dtype=np.int16)) trails += [trail.copy()] if visualize: cost_map = deepcopy(cost_map) circ = circle(markers[0][0],markers[0][1], 5) circ = np.array([np.minimum(circ[0], height-1), np.minimum(circ[1], width-1)]) circ = np.array([np.maximum(circ[0], 0), np.maximum(circ[1], 0)]) cost_map[circ[0], circ[1]] = 0 for i,t in enumerate(trails[1:]): # embed() cost_map[t[:,0], t[:,1]] = 0 circ = circle(markers[i+1][0],markers[i+1][1], 5) circ = np.array([np.minimum(circ[0], height-1), np.minimum(circ[1], width-1)]) circ = np.array([np.maximum(circ[0], 0), np.maximum(circ[1], 0)]) cost_map[circ[0], circ[1]] = 0 return trails, cost_map else: return trails
def read_sparse_array(self, hdr):
''' Read sparse matrix type
Matlab (TM) 4 real sparse arrays are saved in a N+1 by 3 array
format, where N is the number of non-zero values. Column 1 values
[0:N] are the (1-based) row indices of the each non-zero value,
column 2 [0:N] are the column indices, column 3 [0:N] are the
(real) values. The last values [-1,0:2] of the rows, column
indices are shape[0] and shape[1] respectively of the output
matrix. The last value for the values column is a padding 0. mrows
and ncols values from the header give the shape of the stored
matrix, here [N+1, 3]. Complex data is saved as a 4 column
matrix, where the fourth column contains the imaginary component;
the last value is again 0. Complex sparse data do _not_ have the
header imagf field set to True; the fact that the data are complex
is only detectable because there are 4 storage columns
'''
res = self.read_sub_array(hdr)
tmp = res[:-1,:]
dims = res[-1,0:2]
I = np.ascontiguousarray(tmp[:,0],dtype='intc') #fixes byte order also
J = np.ascontiguousarray(tmp[:,1],dtype='intc')
I -= 1 # for 1-based indexing
J -= 1
if res.shape[1] == 3:
V = np.ascontiguousarray(tmp[:,2],dtype='float')
else:
V = np.ascontiguousarray(tmp[:,2],dtype='complex')
V.imag = tmp[:,3]
return scipy.sparse.coo_matrix((V,(I,J)), dims)
def _get_cache_key(self, solver_fn, solver, neuron_type, gain, bias,
x, targets, rng, E):
h = hashlib.sha1()
if PY2:
h.update(str(Fingerprint(solver_fn)))
h.update(str(Fingerprint(solver)))
h.update(str(Fingerprint(neuron_type)))
else:
h.update(str(Fingerprint(solver_fn)).encode('utf-8'))
h.update(str(Fingerprint(solver)).encode('utf-8'))
h.update(str(Fingerprint(neuron_type)).encode('utf-8'))
h.update(np.ascontiguousarray(gain).data)
h.update(np.ascontiguousarray(bias).data)
h.update(np.ascontiguousarray(x).data)
h.update(np.ascontiguousarray(targets).data)
# rng format doc:
# noqa <http://docs.scipy.org/doc/numpy/reference/generated/numpy.random.RandomState.get_state.html#numpy.random.RandomState.get_state>
state = rng.get_state()
h.update(state[0].encode()) # string 'MT19937'
h.update(state[1].data) # 1-D array of 624 unsigned integer keys
h.update(struct.pack('q', state[2])) # integer pos
h.update(struct.pack('q', state[3])) # integer has_gauss
h.update(struct.pack('d', state[4])) # float cached_gaussian
if E is not None:
h.update(np.ascontiguousarray(E).data)
return h.hexdigest()
def equalize_lib_sizes(counts, groups, lib_size, dispersion=0, common_size=None):
""" Equalize the library sizes
"""
if not common_size:
common_size = np.exp(np.mean(np.log(lib_size)))
try: len(dispersion)
except TypeError: dispersion = np.repeat(dispersion,
counts.shape[0])
input_mean = np.empty(counts.shape, dtype=np.double)
output_mean = input_mean.copy()
for key, group in groups.iteritems():
beta = glm_one_group_numba(
np.ascontiguousarray(counts.ix[:,group].as_matrix(),
dtype=np.int32),
dispersion,
np.ascontiguousarray(np.log(lib_size[group]),
dtype=np.double)
)
beta = np.asarray(beta)
bn_lambda = np.exp(beta).T.reshape(len(beta),1)
temp_lib_size = np.array(lib_size[group]).reshape(1,
len(lib_size[group]))
out_size = np.repeat(common_size, len(group)).reshape(1,
len(group))
input_mean[:, group] = np.dot(bn_lambda, temp_lib_size)
output_mean[:, group] = np.dot(bn_lambda,
out_size)
pseudo = q2qnbinom(np.asarray(counts.as_matrix()),
input_mean, output_mean, dispersion)
pseudo[pseudo < 0] = 0
return pseudo, common_size
def fit(self,X,y,k=None,sorted=False):
"""
:param sorted:
:param X:
:param y:
:param k:
"""
X = np.asarray(X)
y = np.asarray(y)
if not X.flags['C_CONTIGUOUS']:
X = np.ascontiguousarray(X)
if not y.flags['C_CONTIGUOUS']:
y = np.ascontiguousarray(y)
assert y.ndim == 1
if X.ndim > 1:
assert X.ndim==2 and (X.shape[0] == y.shape[0] or X.shape[1] == y.shape[0])
if(X.shape[0] != y.shape[0]):
X = X.transpose()
self.N = X.shape[1]
else:
assert X.shape[0] == y.shape[0]
self.N = 1
if k == None:
k = []
for i in range(0,self.N):
k.append(3)
k = list(k)
assert len(k) == self.N
self.f.fit(X,y,k,sorted)
def DoubleInPointerFilter_assign_fail_test(): import numpy as np from ATK.Core import DoubleInPointerFilter d = np.ascontiguousarray(np.arange(1000, dtype=np.float64)[None,:]) filter = DoubleInPointerFilter(d, False) d = np.ascontiguousarray(np.arange(1000, dtype=np.float64).reshape(2,-1)) filter.set_pointer(d)
def hist_3d_index(x, y, z, shape):
"""
Fast 3d histogram of 3D indices with C++ inner loop optimization.
Is more than 2 orders faster than np.histogramdd() and uses less RAM.
The indices are given in x, y, z coordinates and have to fit into a histogram of the dimensions shape.
Parameters
----------
x : array like
y : array like
z : array like
shape : tuple
tuple with x,y,z dimensions: (x, y, z)
Returns
-------
np.ndarray with given shape
"""
if len(shape) != 3:
raise ValueError('The shape has to describe a 3-d histogram')
# change memory alignment for c++ library
x = np.ascontiguousarray(x.astype(np.int32))
y = np.ascontiguousarray(y.astype(np.int32))
z = np.ascontiguousarray(z.astype(np.int32))
result = np.zeros(shape=shape, dtype=np.uint32).ravel() # ravel hist in c-style, 3D --> 1D
compiled_analysis_functions.hist_3d(x, y, z, shape[0], shape[1], shape[2], result)
return np.reshape(result, shape) # rebuilt 3D hist from 1D hist
def __init__(self, x, y, z, Lbox, cell_size):
"""
Initialize the grid.
Parameters
----------
x, y, z : arrays
Length-Npts arrays containing the spatial position of the Npts points.
Lbox : float
Length scale defining the periodic boundary conditions
cell_size : float
The approximate cell size into which the box will be divided.
"""
self.cell_size = cell_size.astype(np.float)
self.Lbox = Lbox.astype(np.float)
self.num_divs = np.floor(Lbox/cell_size).astype(int)
self.dL = Lbox/self.num_divs
#build grid tree
idx_sorted, slice_array = self.compute_cell_structure(x, y, z)
self.x = np.ascontiguousarray(x[idx_sorted],dtype=np.float64)
self.y = np.ascontiguousarray(y[idx_sorted],dtype=np.float64)
self.z = np.ascontiguousarray(z[idx_sorted],dtype=np.float64)
self.slice_array = slice_array
self.idx_sorted = idx_sorted
def do_parameter_selection(argv):
path, test_path = argv;
params = {'n_estimators': 500, 'max_depth': 4, 'min_samples_split': 1,
'min_samples_leaf':1, 'random_state':None, 'do_consider_correct':1,
'learn_rate': 0.2, 'n1': 2000, 'n2': 1, 'tau': 0.01};
print 'loading data...'
X, dr, sr, groups = load_dataset(path)
test_X, test_rd, test_rs, test_groups = load_dataset(test_path);
# test_X = np.asfortranarray(test_X, dtype=DTYPE);
test_rd = np.ascontiguousarray(test_rd);
test_rs = np.ascontiguousarray(test_rs);
test_docpair_samples = DocPairSampler(np.random.RandomState()).sample(test_rd, test_groups, 20000);
from sklearn.grid_search import IterGrid;
param_grid = IterGrid({'n_estimators':[200,400,600,800,1000], 'n1':[1000,2000,5000], 'learn_rate':[.1,.2,.3] });
for param in param_grid:
print param;
params.update(param);
ranker = GradientBoostingRanker(**params);
ranker.fit(X, dr, sr, groups);
test_y_pred = ranker.predict(test_X);
test_pred_sort_groups = PredictSortGroups(test_y_pred, test_groups);
test_loss = ranker.loss_(test_rd, test_rs, test_y_pred, test_groups, test_pred_sort_groups, ranker.random_state, test_docpair_samples);
print ranker.train_score_[-1], test_loss;
def read(self):
"""Read the visibilities and return as a (data,weight) tuple. """
print "Reading " + str(self.data_size()) + " samples..."
data = numpy.ascontiguousarray(numpy.zeros(self._imagingdata.dataSize, dtype=numpy.complex128))
weights = numpy.ascontiguousarray(numpy.zeros(self._imagingdata.dataSize, dtype=numpy.float64))
_wsclean.read(self._userdata, data, weights)
return data, weights
def initializeC(self, image):
super(MultiFitterZ, self).initializeC(image)
self.clib.daoInitializeZ(self.mfit,
numpy.ascontiguousarray(self.wx_params),
numpy.ascontiguousarray(self.wy_params),
self.min_z,
self.max_z)
def tucker_als(idx, val, shape, core_shape, iters=25, growth_tol=0.01, batch_run=False):
'''
The function computes Tucker ALS decomposition of sparse tensor
provided in COO format. Usage:
u0, u1, u2, g = newtuck(idx, val, shape, core_shape)
'''
def log_status(msg):
if not batch_run:
print msg
if not (idx.flags.c_contiguous and val.flags.c_contiguous):
raise ValueError('Warning! Imput arrays must be C-contigous.')
#TODO: it's better to implement check for future
#if np.any(idx[1:, 0]-idx[:-1, 0]) < 0):
# print 'Warning! Index array must be sorted by first column in ascending order.'
r0, r1, r2 = core_shape
u1 = np.random.rand(shape[1], r1)
u1 = np.linalg.qr(u1, mode='reduced')[0]
u2 = np.random.rand(shape[2], r2)
u2 = np.linalg.qr(u2, mode='reduced')[0]
u1 = np.ascontiguousarray(u1)
u2 = np.ascontiguousarray(u2)
g_norm_old = 0
for i in xrange(iters):
log_status('Step %i of %i' % (i+1, iters))
u0 = tensordot2(idx, val, shape, u2, u1, ((2, 0), (1, 0)))\
.reshape(shape[0], r1*r2)
uu = np.linalg.svd(u0, full_matrices=0)[0]
u0 = np.ascontiguousarray(uu[:, :r0])
u1 = tensordot2(idx, val, shape, u2, u0, ((2, 0), (0, 0)))\
.reshape(shape[1], r0*r2)
uu = np.linalg.svd(u1, full_matrices=0)[0]
u1 = np.ascontiguousarray(uu[:, :r1])
u2 = tensordot2(idx, val, shape, u1, u0, ((1, 0), (0, 0)))\
.reshape(shape[2], r0*r1)
uu, ss, vv = np.linalg.svd(u2, full_matrices=0)
u2 = np.ascontiguousarray(uu[:, :r2])
g_norm_new = np.linalg.norm(np.diag(ss[:r2]))
g_growth = (g_norm_new - g_norm_old) / g_norm_new
g_norm_old = g_norm_new
log_status('growth of the core: %f' % g_growth)
if g_growth < growth_tol:
log_status('Core is no longer growing. Norm of the core: %f' % g_norm_old)
break
g = np.diag(ss[:r2]).dot(vv[:r2, :])
g = g.reshape(r2, r1, r0).transpose(2, 1, 0)
log_status('Done')
return u0, u1, u2, g
def optimum_reparam_pair(q, time, q1, q2, lam=0.0):
"""
calculates the warping to align srsf pair q1 and q2 to q
:param q: vector of size N or array of NxM samples of first SRSF
:param time: vector of size N describing the sample points
:param q1: vector of size N or array of NxM samples samples of second SRSF
:param q2: vector of size N or array of NxM samples samples of second SRSF
:param lam: controls the amount of elasticity (default = 0.0)
:rtype: vector
:return gam: describing the warping function used to align q2 with q1
"""
if q1.ndim == 1 and q2.ndim == 1:
q_c = column_stack((q1, q2))
gam = orN.coptimum_reparam_pair(ascontiguousarray(q), time,
ascontiguousarray(q_c), lam)
if q1.ndim == 2 and q2.ndim == 2:
gam = orN.coptimum_reparamN2_pair(ascontiguousarray(q), time,
ascontiguousarray(q1),
ascontiguousarray(q2), lam)
return gam
def initializeC(self, image):
"""
This initializes the C fitting library.
It needs the image in order to know what size arrays to create
as we won't always have SCMOS calibration data.
"""
if self.scmos_cal is None:
if self.verbose:
print("Using zeros for sCMOS calibration data.")
self.scmos_cal = numpy.ascontiguousarray(numpy.zeros(image.shape), dtype = numpy.float64)
else:
self.scmos_cal = numpy.ascontiguousarray(self.scmos_cal, dtype = numpy.float64)
if self.rqe is None:
if self.verbose:
print("Using ones for relative quantum efficiency data.")
self.rqe = numpy.ascontiguousarray(numpy.ones(image.shape), dtype = numpy.float64)
else:
self.rqe = numpy.ascontiguousarray(self.rqe, dtype = numpy.float64)
if (image.shape[0] != self.scmos_cal.shape[0]) or (image.shape[1] != self.scmos_cal.shape[1]):
raise MultiFitterException("Image shape and sCMOS calibration shape do not match.")
if (image.shape[0] != self.rqe.shape[0]) or (image.shape[1] != self.rqe.shape[1]):
raise MultiFitterException("Image shape and RQE shape do not match.")
self.im_shape = self.scmos_cal.shape
if self.verbose and self.als_fit:
print("Anscombe least squares fitting requested.")
def lasso(X, Y, B=None, lam=1., max_iter=None, tol=None):
'''
B = lasso(X, Y, B={np.zeros()}, lam=1. max_iter={1024}, tol={1e-5})
Solve LASSO Optimisation
B* = arg min_B ½/n || Y - BX ||₂² + λ||B||₁
where $n$ is the number of samples.
Milk uses coordinate descent, looping through the coordinates in order
(with an active set strategy to update only non-zero βs, if possible). The
problem is convex and the solution is guaranteed to be optimal (within
floating point accuracy).
Parameters
----------
X : ndarray
Design matrix
Y : ndarray
Matrix of outputs
B : ndarray, optional
Starting values for approximation. This can be used for a warm start if
you have an estimate of where the solution should be. If used, the
solution might be written in-place (if the array has the right format).
lam : float, optional
λ (default: 1.0)
max_iter : int, optional
Maximum nr of iterations (default: 1024)
tol : float, optional
Tolerance. Whenever a parameter is to be updated by a value smaller
than ``tolerance``, that is considered a null update. Be careful that
if the value is too small, performance will degrade horribly.
(default: 1e-5)
Returns
-------
B : ndarray
'''
X = np.ascontiguousarray(X, dtype=np.float32)
Y = np.ascontiguousarray(Y, dtype=np.float32)
if B is None:
B = np.zeros((Y.shape[0],X.shape[0]), np.float32)
else:
B = np.ascontiguousarray(B, dtype=np.float32)
if max_iter is None:
max_iter = 1024
if tol is None:
tol = 1e-5
if X.shape[0] != B.shape[1] or \
Y.shape[0] != B.shape[0] or \
X.shape[1] != Y.shape[1]:
raise ValueError('milk.supervised.lasso: Dimensions do not match')
if np.any(np.isnan(X)) or np.any(np.isnan(B)):
raise ValueError('milk.supervised.lasso: NaNs are only supported in the ``Y`` matrix')
W = np.ascontiguousarray(~np.isnan(Y), dtype=np.float32)
Y = np.nan_to_num(Y)
n = Y.size
_lasso.lasso(X, Y, W, B, max_iter, float(2*n*lam), float(tol))
return B
def array_like(obj,
name,
dtype=np.double,
ndim=1,
maxdim=None,
shape=None,
order='C',
contiguous=False,
optional=False):
"""
Convert array-like to a ndarray and check conditions
Parameters
----------
obj : array_like
An array, any object exposing the array interface, an object whose
__array__ method returns an array, or any (nested) sequence.
name : str
Name of the variable to use in exceptions
dtype : {None, numpy.dtype, str}
Required dtype. Default is double. If None, does not change the dtype
of obj (if present) or uses NumPy to automatically detect the dtype
ndim : {int, None}
Required number of dimensions of obj. If None, no check is performed.
If the numebr of dimensions of obj is less than ndim, additional axes
are inserted on the right. See examples.
maxdim : {int, None}
Maximum allowed dimension. Use ``maxdim`` instead of ``ndim`` when
inputs are allowed to have ndim 1, 2, ..., or maxdim.
shape : {tuple[int], None}
Required shape obj. If None, no check is performed. Partially
restricted shapes can be checked using None. See examples.
order : {'C', 'F'}
Order of the array
contiguous : bool
Ensure that the array's data is contiguous with order ``order``
optional : bool
Flag indicating whether None is allowed
Returns
-------
ndarray
The converted input.
Examples
--------
Convert a list or pandas series to an array
>>> import pandas as pd
>>> x = [0, 1, 2, 3]
>>> a = array_like(x, 'x', ndim=1)
>>> a.shape
(4,)
>>> a = array_like(pd.Series(x), 'x', ndim=1)
>>> a.shape
(4,)
>>> type(a.orig)
pandas.core.series.Series
Squeezes singleton dimensions when required
>>> x = np.array(x).reshape((4, 1))
>>> a = array_like(x, 'x', ndim=1)
>>> a.shape
(4,)
Right-appends when required size is larger than actual
>>> x = [0, 1, 2, 3]
>>> a = array_like(x, 'x', ndim=2)
>>> a.shape
(4, 1)
Check only the first and last dimension of the input
>>> x = np.arange(4*10*4).reshape((4, 10, 4))
>>> y = array_like(x, 'x', ndim=3, shape=(4, None, 4))
Check only the first two dimensions
>>> z = array_like(x, 'x', ndim=3, shape=(4, 10))
Raises ValueError if constraints are not satisfied
>>> z = array_like(x, 'x', ndim=2)
Traceback (most recent call last):
...
ValueError: x is required to have ndim 2 but has ndim 3
>>> z = array_like(x, 'x', shape=(10, 4, 4))
Traceback (most recent call last):
...
ValueError: x is required to have shape (10, 4, 4) but has shape (4, 10, 4)
>>> z = array_like(x, 'x', shape=(None, 4, 4))
Traceback (most recent call last):
...
ValueError: x is required to have shape (*, 4, 4) but has shape (4, 10, 4)
"""
if optional and obj is None:
return None
arr = np.asarray(obj, dtype=dtype, order=order)
if maxdim is not None:
if arr.ndim > maxdim:
msg = '{0} must have ndim <= {1}'.format(name, maxdim)
raise ValueError(msg)
elif ndim is not None:
if arr.ndim > ndim:
arr = _right_squeeze(arr, stop_dim=ndim)
elif arr.ndim < ndim:
arr = np.reshape(arr, arr.shape + (1, ) * (ndim - arr.ndim))
if arr.ndim != ndim:
msg = '{0} is required to have ndim {1} but has ndim {2}'
raise ValueError(msg.format(name, ndim, arr.ndim))
if shape is not None:
for actual, req in zip(arr.shape, shape):
if req is not None and actual != req:
req_shape = str(shape).replace('None, ', '*, ')
msg = '{0} is required to have shape {1} but has shape {2}'
raise ValueError(msg.format(name, req_shape, arr.shape))
if contiguous:
arr = np.ascontiguousarray(arr, dtype=dtype)
return arr
def as_floatX(x):
if not hasattr(x, '__len__'):
return np.array(x, dtype=floatX)
return np.ascontiguousarray(x, dtype=floatX)
if ret_val:
ret_val = cv2.imwrite(filename, img)
print(ret_val)
pyyolo.init(datacfg, cfgfile, weightfile)
# from file
print('----- test original C using a file')
outputs = pyyolo.test(filename, thresh, hier_thresh)
for output in outputs:
print(output)
# camera
print('----- test python API using a file')
i = 1
while i < 2:
# ret_val, img = cam.read()
img = cv2.imread(filename)
img = img.transpose(2, 0, 1)
c, h, w = img.shape[0], img.shape[1], img.shape[2]
# print w, h, c
data = img.ravel() / 255.0
data = np.ascontiguousarray(data, dtype=np.float32)
outputs = pyyolo.detect(w, h, c, data, thresh, hier_thresh)
for output in outputs:
print(output)
i = i + 1
# free model
pyyolo.cleanup()
def _call(self):
# compute the sampling frequency if necessary
if 'dt' in self.data['Sensors']['Lumbar']:
dt = self.data['Sensors']['Lumbar']['dt'][()]
else:
dt = mean(diff(self.data['Sensors']['Lumbar']['Unix Time'][:100]))
self.data = ('Sensors/Lumbar/dt', dt) # save for future use
# set-up the filter that will be used
sos = butter(self.lp_ord,
2 * self.lp_cut * dt,
btype='low',
output='sos')
if 'Processed' in self.data:
days = [
i for i in self.data['Processed']['Sit2Stand'].keys()
if 'Day' in i
]
else:
days = ['Day 1']
for iday, day in enumerate(days):
try:
start, stop = self.data['Processed']['Sit2Stand'][day][
'Indices']
except KeyError:
start, stop = 0, self.data['Sensors']['Lumbar'][
'Accelerometer'].shape[0]
# compute the magnitude of the acceleration
m_acc = norm(
self.data['Sensors']['Lumbar']['Accelerometer'][start:stop],
axis=1)
f_acc = ascontiguousarray(sosfiltfilt(sos, m_acc))
# reconstructed acceleration
if self.method == 'dwt':
# deconstruct the filtered acceleration magnitude
coefs = pywt.wavedec(f_acc, self.dwave, mode=self.ext_mode)
# set all but the desired level of coefficients to be 0s
if (len(coefs) - self.recon_level) < 1:
warn(
f'Chosen reconstruction level is too high, setting to {len(coefs) - 1}',
UserWarning)
ind = 1
else:
ind = len(coefs) - self.recon_level
for i in range(1, len(coefs)):
if i != ind:
coefs[i][:] = 0
r_acc = pywt.waverec(coefs, self.dwave, mode=self.ext_mode)
elif self.method == 'moving average':
n_window = int(around(self.window / dt))
r_acc, *_ = mov_stats(f_acc, n_window)
# CWT power peak detection
coefs, freqs = pywt.cwt(r_acc,
arange(1, 65),
self.cwave,
sampling_period=dt)
# sum the coefficients over the frequencies in the power band
f_mask = (freqs <= self.power_end_f) & (freqs >=
self.power_start_f)
power = sum(coefs[f_mask, :], axis=0)
# find the peaks in the power data
if self.std_height:
if self.std_trim != 0:
trim = int(self.std_trim / dt)
self.power_peak_kw['height'] = std(power[trim:-trim],
ddof=1)
else:
self.power_peak_kw['height'] = std(power, ddof=1)
power_peaks, _ = find_peaks(power, **self.power_peak_kw)
self.data = (PROC.format(day_n=iday + 1,
value='Filtered Acceleration'), f_acc)
self.data = (PROC.format(day_n=iday + 1,
value='Reconstructed Acceleration'),
r_acc[:m_acc.size])
self.data = (PROC.format(day_n=iday + 1, value='Power'), power)
self.data = (PROC.format(day_n=iday + 1,
value='Power Peaks'), power_peaks)
def cythonize(*matrices):
return tuple(np.ascontiguousarray(matrix, dtype=np.float64) for matrix in matrices)
def __next__(self):
self.count += 1
if self.count == self.nB:
raise StopIteration
ia = self.count * self.batch_size
ib = min((self.count + 1) * self.batch_size, self.nF)
img_all, labels_all, img_paths, img_shapes = [], [], [], []
for index, files_index in enumerate(range(ia, ib)):
img_path = self.img_files[self.shuffled_vector[files_index]]
label_path = self.label_files[self.shuffled_vector[files_index]]
img = cv2.imread(img_path) # BGR
assert img is not None, 'File Not Found ' + img_path
augment_hsv = True
if self.augment and augment_hsv:
# SV augmentation by 50%
fraction = 0.50
img_hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
S = img_hsv[:, :, 1].astype(np.float32)
V = img_hsv[:, :, 2].astype(np.float32)
a = (random.random() * 2 - 1) * fraction + 1
S *= a
if a > 1:
np.clip(S, a_min=0, a_max=255, out=S)
a = (random.random() * 2 - 1) * fraction + 1
V *= a
if a > 1:
np.clip(V, a_min=0, a_max=255, out=V)
img_hsv[:, :, 1] = S.astype(np.uint8)
img_hsv[:, :, 2] = V.astype(np.uint8)
cv2.cvtColor(img_hsv, cv2.COLOR_HSV2BGR, dst=img)
h, w, _ = img.shape
img, ratio, padw, padh = letterbox(img, height=self.img_size)
# Load labels
if os.path.isfile(label_path):
labels0 = np.loadtxt(label_path,
dtype=np.float32).reshape(-1, 5)
# Normalized xywh to pixel xyxy format
labels = labels0.copy()
labels[:, 1] = ratio * w * (labels0[:, 1] -
labels0[:, 3] / 2) + padw
labels[:, 2] = ratio * h * (labels0[:, 2] -
labels0[:, 4] / 2) + padh
labels[:, 3] = ratio * w * (labels0[:, 1] +
labels0[:, 3] / 2) + padw
labels[:, 4] = ratio * h * (labels0[:, 2] +
labels0[:, 4] / 2) + padh
else:
labels = np.array([])
# Augment image and labels
if self.augment:
img, labels, M = random_affine(img,
labels,
degrees=(-5, 5),
translate=(0.10, 0.10),
scale=(0.90, 1.10))
plotFlag = False
if plotFlag:
import matplotlib.pyplot as plt
plt.figure(figsize=(10, 10)) if index == 0 else None
plt.subplot(4, 4, index + 1).imshow(img[:, :, ::-1])
plt.plot(labels[:, [1, 3, 3, 1, 1]].T,
labels[:, [2, 2, 4, 4, 2]].T, '.-')
plt.axis('off')
nL = len(labels)
if nL > 0:
# convert xyxy to xywh
labels[:,
1:5] = xyxy2xywh(labels[:, 1:5].copy()) / self.img_size
if self.augment:
# random left-right flip
lr_flip = True
if lr_flip & (random.random() > 0.5):
img = np.fliplr(img)
if nL > 0:
labels[:, 1] = 1 - labels[:, 1]
# random up-down flip
ud_flip = False
if ud_flip & (random.random() > 0.5):
img = np.flipud(img)
if nL > 0:
labels[:, 2] = 1 - labels[:, 2]
if nL > 0:
labels = np.concatenate((np.zeros(
(nL, 1), dtype='float32') + index, labels), 1)
labels_all.append(labels)
img_all.append(img)
img_paths.append(img_path)
img_shapes.append((h, w))
# Normalize
img_all = np.stack(img_all)[:, :, :, ::-1].transpose(
0, 3, 1, 2) # BGR to RGB and cv2 to pytorch
img_all = np.ascontiguousarray(img_all, dtype=np.float32)
img_all /= 255.0
labels_all = torch.from_numpy(np.concatenate(labels_all, 0))
return torch.from_numpy(img_all), labels_all, img_paths, img_shapes
def imread(fname):
im = np.asarray(Image.open(fname).convert('RGB'))
if im.ndim == 3 and im.shape[2] > 3:
im = im[:, :, :3]
return np.ascontiguousarray(im)
#img = bpy.data.images["DummyImage4"]
#img_array = [1 for pix in range(len(img.pixels))] # fill with zeros
#img.pixels = img_array
##tex.image = img
#for i in range(3):
# tex.voxel_data.resolution[i] = -512
for i in range(3):
tex.voxel_data.resolution[i] = rez
vdp = ctypes.cast(bpy.data.textures[tex_name].voxel_data.as_pointer(),
ctypes.POINTER(VoxelData))
import yt
ds = yt.load("~/data/IsolatedGalaxy/galaxy0030/galaxy0030")
cg = ds.covering_grid(level, [0, 0, 0], (rez, rez, rez))
rho = np.log10(cg["density"])
rho = (rho - rho.min()) / (rho.max() - rho.min())
rho = rho.astype("float32").copy()
rho = np.ascontiguousarray(rho)
arr = (ctypes.c_float * rho.size)()
arr2 = np.ctypeslib.as_array(arr, rho.shape)
arr2[:] = rho.flat[:]
vdp.contents.dataset = ctypes.cast(arr, ctypes.POINTER(ctypes.c_float))
vdp.contents.ok = 1
vdp.contents.cachedframe = bpy.context.scene.frame_current
print(rho.strides)
# NOTE for a pretty image you want this setup with "over sampling" checked in the materials panel: http://blender.stackexchange.com/questions/15010/rendering-a-3d-volume
def pointsToPolyData(points, copy_z=False):
"""Create ``vtkPolyData`` from a numpy array of XYZ points. If the points
have more than 3 dimensions, then all dimensions after the third will be
added as attributes. Assume the first three dimensions are the XYZ
coordinates.
Args:
points (np.ndarray or pandas.DataFrame): The points and pointdata
copy_z (bool): A flag on whether to append the z values as a PointData
array
Return:
vtkPolyData : points with point-vertex cells
"""
__displayname__ = 'Points to PolyData'
__category__ = 'filter'
# This prevents an error that occurs when only one point is passed
if points.ndim < 2:
points = points.reshape((1,-1))
keys = ['Field %d' % i for i in range(points.shape[1] - 3)]
# Check if input is anything other than a NumPy array and cast it
# e.g. you could send a Pandas dataframe
if not isinstance(points, np.ndarray):
if isinstance(points, pd.DataFrame):
# If a pandas data frame, lets grab the keys
keys = points.keys()[3::]
points = np.array(points)
# If points are not 3D
if points.shape[1] < 2:
raise RuntimeError('Points must be 3D. Try adding a third dimension of zeros.')
atts = points[:, 3::]
points = points[:, 0:3]
npoints = points.shape[0]
# Make VTK cells array
cells = np.hstack((np.ones((npoints, 1)),
np.arange(npoints).reshape(-1, 1)))
cells = np.ascontiguousarray(cells, dtype=np.int64)
cells = np.reshape(cells, (2*npoints))
vtkcells = vtk.vtkCellArray()
vtkcells.SetCells(npoints, nps.numpy_to_vtk(cells, deep=True, array_type=vtk.VTK_ID_TYPE))
# Convert points to vtk object
pts = vtk.vtkPoints()
pts.SetData(convertArray(points))
# Create polydata
pdata = vtk.vtkPolyData()
pdata.SetPoints(pts)
pdata.SetVerts(vtkcells)
# Add attributes if given
scalSet = False
for i, key in enumerate(keys):
data = convertArray(atts[:, i], name=key)
pdata.GetPointData().AddArray(data)
if not scalSet:
pdata.GetPointData().SetActiveScalars(key)
scalSet = True
if copy_z:
z = convertArray(points[:, 2], name='Elevation')
pdata.GetPointData().AddArray(z)
return wrapvtki(pdata)
def create_batches(self, batch_size, shuffle=True):
# 1 batch = [(image, [([x, y, w, h], id), ([x, y, w, h], id), ...]), ...]
batch = []
counter_samples=0
while True:
indices = range(len(self.img_ids))
if shuffle:
indices = np.random.permutation(indices)
for index in indices:
index += 1
try:
img = self.coco.loadImgs(self.img_ids[index])[0]
except:
print(index)
continue
path = os.path.join(self.image_dir, self.get_image_path(id=img['id'], name=img['file_name']))
I = cv2.imread(path).astype(np.uint8)[:, :, ::-1]
I = np.ascontiguousarray(I)
try:
if len(I.shape) != 3:
continue
except:
print("no image exist")
continue
ann_ids = self.coco.getAnnIds(imgIds=img['id'], catIds=self.cat_ids, iscrowd=None)
anns = self.coco.loadAnns(ann_ids)
ann_list = []
rles = []
for ann in anns:
bb = [f for f in ann["bbox"]]
try:
rle = frPyObjects(ann['segmentation'], I.shape[0], I.shape[1])[0]
bbb = toBbox(rle)
except:
continue
# make sure we dont include unknown classes
if self.id2i[ann["category_id"]] < 0 or self.id2i[ann["category_id"]] > config.TOTAL_CLASSES:
print("This class cannot be processed %d ...", self.id2i[ann["category_id"]])
continue
rles.append(rle)
ann_list.append((decode(rle).astype(np.float), bb, self.id2i[ann["category_id"]]))
# print("------- RLEs-SHAPE")
# print(len(rles))
if len(rles) == 0:
print("NO RLE was extracted continue with next picture ...")
continue
mask = decode(rles).astype(np.float)
batch.append((I, mask, ann_list))
# print("------- MASKS-SHAPE")
# print(len(mask))
# print(np.max(mask), np.min(mask))
if len(batch) >= batch_size:
self.counter_samples += len(batch)
print("Getting new batch with %d elements ..." % (len(batch)))
yield batch
del mask
del ann_list
del rles
del batch
batch = []
def get_img_label(img_name, img_dir, gt_dir):
# get the labels、targets for the input image
name = img_name.split('.')[0]
img_path = os.path.join(img_dir, img_name)
gt_path = os.path.join(gt_dir, name + '.txt')
img = cv2.imread(img_path)
img_size = img.shape
h_feat = img_size[0] // 2 // 2 // 2 // 2 #feature map height
w_feat = img_size[1] // 2 // 2 // 2 // 2 #feature map width
#get the image data in bytes
with open(img_path, 'rb') as ff:
img_bytes = ff.read()
###read gt_box
with open(gt_path, 'r') as ff:
lines = ff.readlines()
box_list = []
side_flag_list = []
for line in lines:
line = line.strip().split(',')
side_flag_list.append(int(line[-1]))
line = line[0:-1]
x1, y1, x2, y2 = map(int, line)
box_list.append(np.array([x1, y1, x2, y2]))
gt_boxes = np.stack(box_list)
gt_side_flag = np.stack(side_flag_list)
###get all anchors
base_anchors = gen_base_anchors()
A = base_anchors.shape[0]
K = h_feat * w_feat
base_anchors = base_anchors.reshape(1, A, 4)
shift_x = np.arange(w_feat) * _stripe #对于feature map上每个点，x方向anchor偏移量
shift_y = np.arange(h_feat) * _stripe #对于feature map上每个点，y方向anchor偏移量
shift_x, shift_y = np.meshgrid(shift_x, shift_y) #生成二维点阵的x,y方向偏移量
shift_x = shift_x.ravel() #二维变一维
shift_y = shift_y.ravel()
shift = np.stack([shift_x, shift_y, shift_x, shift_y]).transpose()
shift = shift.reshape(K, 1, 4)
all_anchors = base_anchors + shift
all_anchors = all_anchors.reshape((K * A, 4))
total_anchors = K * A
#remove the anchors that are out of the image
index_inside = np.where((all_anchors[:, 0] >= 0) & (all_anchors[:, 1] >= 0)
& (all_anchors[:, 2] < img_size[1])
& (all_anchors[:, 3] < img_size[0]))[0]
anchors = all_anchors[index_inside, :]
### get labels, 1=positive, 0=negetive, -1=don't care
labels = np.ones(anchors.shape[0], dtype=np.int) * -1
labels2 = np.ones(anchors.shape[0], dtype=np.int) * -1
overlaps = bbox_overlaps(np.ascontiguousarray(anchors, dtype=np.float),
np.ascontiguousarray(gt_boxes, dtype=np.float))
max_ol_gt_index = overlaps.argmax(axis=1)
max_ol_gt = overlaps[np.arange(anchors.shape[0]), max_ol_gt_index]
max_ol_anchor_index = overlaps.argmax(axis=0)
max_ol_anchor = overlaps[max_ol_anchor_index, np.arange(gt_boxes.shape[0])]
labels[max_ol_gt < OVERLAP_NEGATIVE_THR] = 0
labels[max_ol_gt >= OVERLAP_POSITIVE_THR] = 1
labels2[
max_ol_anchor_index] = 1 #there is at least one positive anchor for each gt_box
spectial_anchor_index = np.where(
(labels != 1) & (labels2 == 1)
)[0] #spectial anchor is the anchor that is the max overlap anchor of gt_box A,
# but this anchor's max overlaps with all the gt_box is gt_box B and <OVERLAP_NEGATIVE_THR
spectial_anchor_gt_index = np.zeros(spectial_anchor_index.shape[0],
dtype=np.int)
for ii in range(spectial_anchor_gt_index.shape[0]):
spectial_anchor_gt_index[ii] = np.where(
max_ol_anchor_index == spectial_anchor_index[ii])[0][0]
labels[spectial_anchor_index] = 1
### side label:the anchor if is the left side or right side of the text area
side_labels = np.zeros(anchors.shape[0], dtype=np.int)
side_labels = gt_side_flag[max_ol_gt_index]
side_labels[spectial_anchor_index] = gt_side_flag[spectial_anchor_gt_index]
side_labels[np.where(labels != 1)[0]] = 0
### get the targets
gt_for_anchors = gt_boxes[max_ol_gt_index]
gt_for_anchors[spectial_anchor_index] = gt_boxes[spectial_anchor_gt_index]
box_targets_y, box_targets_offset = target_calc(anchors, gt_for_anchors,
side_labels)
box_weights_y = np.zeros((anchors.shape[0], 2))
box_weights_y[labels == 1, :] = 1
box_targets_y = box_targets_y * box_weights_y
### get the anchor that are out of the image back
labels = _unmap(labels, total_anchors, index_inside, fill=-1)
side_labels = _unmap(side_labels, total_anchors, index_inside, fill=0)
box_targets_y = _unmap(box_targets_y, total_anchors, index_inside, fill=0)
box_targets_offset = _unmap(box_targets_offset,
total_anchors,
index_inside,
fill=0)
### reshape
labels = labels.reshape((1, h_feat, w_feat, A))
side_labels = side_labels.reshape(1, h_feat, w_feat, A)
box_targets_y = box_targets_y.reshape((1, h_feat, w_feat, A * 2))
box_targets_offset = box_targets_offset.reshape(1, h_feat, w_feat, A)
return img_bytes, labels, side_labels, box_targets_y, box_targets_offset
def __getitem__(self, index):
if self.image_weights:
index = self.indices[index]
hyp = self.hyp
if self.mosaic:
# Load mosaic
img, labels = load_mosaic(self, index)
shapes = None
# MixUp https://arxiv.org/pdf/1710.09412.pdf
# if random.random() < 0.5:
# img2, labels2 = load_mosaic(self, random.randint(0, len(self.labels) - 1))
# r = np.random.beta(0.3, 0.3) # mixup ratio, alpha=beta=0.3
# img = (img * r + img2 * (1 - r)).astype(np.uint8)
# labels = np.concatenate((labels, labels2), 0)
else:
# Load image
img, (h0, w0), (h, w) = load_image(self, index)
# Letterbox
shape = self.batch_shapes[self.batch[
index]] if self.rect else self.img_size # final letterboxed shape
img, ratio, pad = letterbox(img,
shape,
auto=False,
scaleup=self.augment)
shapes = (h0, w0), (
(h / h0, w / w0), pad) # for COCO mAP rescaling
# Load labels
labels = []
x = self.labels[index]
if x.size > 0:
# Normalized xywh to pixel xyxy format
labels = x.copy()
labels[:,
1] = ratio[0] * w * (x[:, 1] -
x[:, 3] / 2) + pad[0] # pad width
labels[:,
2] = ratio[1] * h * (x[:, 2] -
x[:, 4] / 2) + pad[1] # pad height
labels[:, 3] = ratio[0] * w * (x[:, 1] + x[:, 3] / 2) + pad[0]
labels[:, 4] = ratio[1] * h * (x[:, 2] + x[:, 4] / 2) + pad[1]
if self.augment:
# Augment imagespace
if not self.mosaic:
img, labels = random_affine(img,
labels,
degrees=hyp['degrees'],
translate=hyp['translate'],
scale=hyp['scale'],
shear=hyp['shear'])
# Augment colorspace
augment_hsv(img,
hgain=hyp['hsv_h'],
sgain=hyp['hsv_s'],
vgain=hyp['hsv_v'])
# Apply cutouts
# if random.random() < 0.9:
# labels = cutout(img, labels)
nL = len(labels) # number of labels
if nL:
# convert xyxy to xywh
labels[:, 1:5] = xyxy2xywh(labels[:, 1:5])
# Normalize coordinates 0 - 1
labels[:, [2, 4]] /= img.shape[0] # height
labels[:, [1, 3]] /= img.shape[1] # width
if self.augment:
# random left-right flip
lr_flip = True
if lr_flip and random.random() < 0.5:
img = np.fliplr(img)
if nL:
labels[:, 1] = 1 - labels[:, 1]
# random up-down flip
ud_flip = False
if ud_flip and random.random() < 0.5:
img = np.flipud(img)
if nL:
labels[:, 2] = 1 - labels[:, 2]
labels_out = torch.zeros((nL, 6))
if nL:
labels_out[:, 1:] = torch.from_numpy(labels)
# Convert
img = img[:, :, ::-1].transpose(2, 0, 1) # BGR to RGB, to 3x416x416
img = np.ascontiguousarray(img)
return torch.from_numpy(img), labels_out, self.img_files[index], shapes
def load_from_table(self, data, b0_thr=0):
"""Build the structure from an input matrix.
The first three columns represent the gradient directions.
Then, we accept two formats to describe each gradient:
- if the shape of data is Nx4, the 4^ column is the b-value;
- if the shape of data is Nx7, the last 4 columns are, respectively, the gradient strength, big delta, small delta and TE.
Parameters
----------
data : numpy.ndarray
Matrix containing tall the values.
b0_thr : float
The threshold on the b-values to identify the b0 images (default: 0)
"""
if data.ndim == 1:
data = np.expand_dims(data, axis=0)
self.raw = data
# number of samples
# self.nS = self.raw.shape[0] JL: incomplete getter/setter incompatible with 3.6; this is never used any as getter always returns derived value
# set/calculate the b-values
if self.raw.shape[1] == 4:
self.version = 0
self.b = self.raw[:, 3]
elif self.raw.shape[1] == 7:
self.version = 1
self.b = (267.513e6 * self.raw[:, 3] * self.raw[:, 5])**2 * (
self.raw[:, 4] - self.raw[:, 5] / 3.0) * 1e-6 # in mm^2/s
else:
ERROR('Unrecognized scheme format')
# store information about the volumes
self.b0_thr = b0_thr
self.b0_idx = np.where(self.b <= b0_thr)[0]
self.b0_count = len(self.b0_idx)
self.dwi_idx = np.where(self.b > b0_thr)[0]
self.dwi_count = len(self.dwi_idx)
# ensure the directions are in the spherical range [0,180]x[0,180]
idx = np.where(self.raw[:, 1] < 0)[0]
self.raw[idx, 0:3] = -self.raw[idx, 0:3]
# store information about each shell in a dictionary
self.shells = []
tmp = np.ascontiguousarray(self.raw[:, 3:])
schemeUnique, schemeUniqueInd = np.unique(tmp.view([('', tmp.dtype)] *
tmp.shape[1]),
return_index=True)
schemeUnique = schemeUnique.view(tmp.dtype).reshape(
(schemeUnique.shape[0], tmp.shape[1]))
schemeUnique = [tmp[index] for index in sorted(schemeUniqueInd)]
bUnique = [self.b[index] for index in sorted(schemeUniqueInd)]
for i in range(len(schemeUnique)):
if bUnique[i] <= b0_thr:
continue
shell = {}
shell['b'] = bUnique[i]
if self.version == 0:
shell['G'] = None
shell['Delta'] = None
shell['delta'] = None
shell['TE'] = None
else:
shell['G'] = schemeUnique[i][0]
shell['Delta'] = schemeUnique[i][1]
shell['delta'] = schemeUnique[i][2]
shell['TE'] = schemeUnique[i][3]
shell['idx'] = np.where((tmp == schemeUnique[i]).all(axis=1))[0]
shell['grad'] = self.raw[shell['idx'], 0:3]
self.shells.append(shell)
def denoise_nl_means(image,
patch_size=7,
patch_distance=11,
h=0.1,
multichannel=False,
fast_mode=True,
sigma=0.,
*,
preserve_range=None,
channel_axis=None):
"""Perform non-local means denoising on 2-D or 3-D grayscale images, and
2-D RGB images.
Parameters
----------
image : 2D or 3D ndarray
Input image to be denoised, which can be 2D or 3D, and grayscale
or RGB (for 2D images only, see ``multichannel`` parameter).
patch_size : int, optional
Size of patches used for denoising.
patch_distance : int, optional
Maximal distance in pixels where to search patches used for denoising.
h : float, optional
Cut-off distance (in gray levels). The higher h, the more permissive
one is in accepting patches. A higher h results in a smoother image,
at the expense of blurring features. For a Gaussian noise of standard
deviation sigma, a rule of thumb is to choose the value of h to be
sigma of slightly less.
multichannel : bool, optional
Whether the last axis of the image is to be interpreted as multiple
channels or another spatial dimension. This argument is deprecated:
specify `channel_axis` instead.
fast_mode : bool, optional
If True (default value), a fast version of the non-local means
algorithm is used. If False, the original version of non-local means is
used. See the Notes section for more details about the algorithms.
sigma : float, optional
The standard deviation of the (Gaussian) noise. If provided, a more
robust computation of patch weights is computed that takes the expected
noise variance into account (see Notes below).
preserve_range : bool, optional
Whether to keep the original range of values. Otherwise, the input
image is converted according to the conventions of `img_as_float`.
Also see https://scikit-image.org/docs/dev/user_guide/data_types.html
channel_axis : int or None, optional
If None, the image is assumed to be a grayscale (single channel) image.
Otherwise, this parameter indicates which axis of the array corresponds
to channels.
.. versionadded:: 0.19
``channel_axis`` was added in 0.19.
Returns
-------
result : ndarray
Denoised image, of same shape as `image`.
Notes
-----
The non-local means algorithm is well suited for denoising images with
specific textures. The principle of the algorithm is to average the value
of a given pixel with values of other pixels in a limited neighbourhood,
provided that the *patches* centered on the other pixels are similar enough
to the patch centered on the pixel of interest.
In the original version of the algorithm [1]_, corresponding to
``fast=False``, the computational complexity is::
image.size * patch_size ** image.ndim * patch_distance ** image.ndim
Hence, changing the size of patches or their maximal distance has a
strong effect on computing times, especially for 3-D images.
However, the default behavior corresponds to ``fast_mode=True``, for which
another version of non-local means [2]_ is used, corresponding to a
complexity of::
image.size * patch_distance ** image.ndim
The computing time depends only weakly on the patch size, thanks to
the computation of the integral of patches distances for a given
shift, that reduces the number of operations [1]_. Therefore, this
algorithm executes faster than the classic algorithm
(``fast_mode=False``), at the expense of using twice as much memory.
This implementation has been proven to be more efficient compared to
other alternatives, see e.g. [3]_.
Compared to the classic algorithm, all pixels of a patch contribute
to the distance to another patch with the same weight, no matter
their distance to the center of the patch. This coarser computation
of the distance can result in a slightly poorer denoising
performance. Moreover, for small images (images with a linear size
that is only a few times the patch size), the classic algorithm can
be faster due to boundary effects.
The image is padded using the `reflect` mode of `skimage.util.pad`
before denoising.
If the noise standard deviation, `sigma`, is provided a more robust
computation of patch weights is used. Subtracting the known noise variance
from the computed patch distances improves the estimates of patch
similarity, giving a moderate improvement to denoising performance [4]_.
It was also mentioned as an option for the fast variant of the algorithm in
[3]_.
When `sigma` is provided, a smaller `h` should typically be used to
avoid oversmoothing. The optimal value for `h` depends on the image
content and noise level, but a reasonable starting point is
``h = 0.8 * sigma`` when `fast_mode` is `True`, or ``h = 0.6 * sigma`` when
`fast_mode` is `False`.
References
----------
.. [1] A. Buades, B. Coll, & J-M. Morel. A non-local algorithm for image
denoising. In CVPR 2005, Vol. 2, pp. 60-65, IEEE.
:DOI:`10.1109/CVPR.2005.38`
.. [2] J. Darbon, A. Cunha, T.F. Chan, S. Osher, and G.J. Jensen, Fast
nonlocal filtering applied to electron cryomicroscopy, in 5th IEEE
International Symposium on Biomedical Imaging: From Nano to Macro,
2008, pp. 1331-1334.
:DOI:`10.1109/ISBI.2008.4541250`
.. [3] Jacques Froment. Parameter-Free Fast Pixelwise Non-Local Means
Denoising. Image Processing On Line, 2014, vol. 4, pp. 300-326.
:DOI:`10.5201/ipol.2014.120`
.. [4] A. Buades, B. Coll, & J-M. Morel. Non-Local Means Denoising.
Image Processing On Line, 2011, vol. 1, pp. 208-212.
:DOI:`10.5201/ipol.2011.bcm_nlm`
Examples
--------
>>> a = np.zeros((40, 40))
>>> a[10:-10, 10:-10] = 1.
>>> rng = np.random.default_rng()
>>> a += 0.3 * rng.standard_normal(a.shape)
>>> denoised_a = denoise_nl_means(a, 7, 5, 0.1)
"""
if image.ndim == 2:
image = image[..., np.newaxis]
channel_axis = -1
if image.ndim != 3:
raise NotImplementedError("Non-local means denoising is only \
implemented for 2D grayscale and RGB images or 3-D grayscale images.")
if preserve_range is None and np.issubdtype(image.dtype, np.integer):
warn(
'Image dtype is not float. By default denoise_nl_means will '
'assume you want to preserve the range of your image '
'(preserve_range=True). In scikit-image 0.19 this behavior will '
'change to preserve_range=False. To avoid this warning, '
'explicitly specify the preserve_range parameter.',
stacklevel=2)
preserve_range = True
image = convert_to_float(image, preserve_range)
if not image.flags.c_contiguous:
image = np.ascontiguousarray(image)
kwargs = dict(s=patch_size, d=patch_distance, h=h, var=sigma * sigma)
if channel_axis is not None: # 2-D images
if fast_mode:
return _fast_nl_means_denoising_2d(image, **kwargs)
else:
return _nl_means_denoising_2d(image, **kwargs)
else: # 3-D grayscale
if fast_mode:
return _fast_nl_means_denoising_3d(image, **kwargs)
else:
return _nl_means_denoising_3d(image, **kwargs)
def train(train_A_dir, train_B_dir, model_dir, model_name, random_seed, validation_A_dir, validation_B_dir, output_dir, tensorboard_log_dir):
np.random.seed(random_seed)
num_epochs = 5000
mini_batch_size = 1 # mini_batch_size = 1 is better
generator_learning_rate = 0.0002
generator_learning_rate_decay = generator_learning_rate / 200000
discriminator_learning_rate = 0.0001
discriminator_learning_rate_decay = discriminator_learning_rate / 200000
sampling_rate = 16000
num_mcep = 24
frame_period = 5.0
n_frames = 128
lambda_cycle = 10
lambda_identity = 5
max_samples = 1000
print('Data Preprocessing...')
start_time = time.time()
wavs_A = load_wavs(wav_dir = train_A_dir, sr = sampling_rate)
wavs_B = load_wavs(wav_dir = train_B_dir, sr = sampling_rate)
f0s_A, timeaxes_A, sps_A, aps_A, coded_sps_A = world_encode_data(wavs = wavs_A, fs = sampling_rate, frame_period = frame_period, coded_dim = num_mcep)
f0s_B, timeaxes_B, sps_B, aps_B, coded_sps_B = world_encode_data(wavs = wavs_B, fs = sampling_rate, frame_period = frame_period, coded_dim = num_mcep)
log_f0s_mean_A, log_f0s_std_A = logf0_statistics(f0s_A)
log_f0s_mean_B, log_f0s_std_B = logf0_statistics(f0s_B)
print('Log Pitch A: Mean %f, Std %f' %(log_f0s_mean_A, log_f0s_std_A))
print('Log Pitch B: Mean %f, Std %f' %(log_f0s_mean_B, log_f0s_std_B))
coded_sps_A_transposed = transpose_in_list(lst = coded_sps_A)
coded_sps_B_transposed = transpose_in_list(lst = coded_sps_B)
coded_sps_A_norm, coded_sps_A_mean, coded_sps_A_std = coded_sps_normalization_fit_transoform(coded_sps = coded_sps_A_transposed)
coded_sps_B_norm, coded_sps_B_mean, coded_sps_B_std = coded_sps_normalization_fit_transoform(coded_sps = coded_sps_B_transposed)
print("Input data loaded.")
if not os.path.exists(model_dir):
os.makedirs(model_dir)
np.savez(os.path.join(model_dir, 'logf0s_normalization.npz'), mean_A = log_f0s_mean_A, std_A = log_f0s_std_A, mean_B = log_f0s_mean_B, std_B = log_f0s_std_B)
np.savez(os.path.join(model_dir, 'mcep_normalization.npz'), mean_A = coded_sps_A_mean, std_A = coded_sps_A_std, mean_B = coded_sps_B_mean, std_B = coded_sps_B_std)
if validation_A_dir is not None:
validation_A_output_dir = os.path.join(output_dir, 'converted_A')
if not os.path.exists(validation_A_output_dir):
os.makedirs(validation_A_output_dir)
if validation_B_dir is not None:
validation_B_output_dir = os.path.join(output_dir, 'converted_B')
if not os.path.exists(validation_B_output_dir):
os.makedirs(validation_B_output_dir)
end_time = time.time()
time_elapsed = end_time - start_time
print('Preprocessing Done.')
print('Time Elapsed for Data Preprocessing: %02d:%02d:%02d' % (time_elapsed // 3600, (time_elapsed % 3600 // 60), (time_elapsed % 60 // 1)))
model = CycleGAN(num_features = num_mcep)
num_iterations = 0
for epoch in range(num_epochs):
print('Epoch: %d' % epoch)
'''
if epoch > 60:
lambda_identity = 0
if epoch > 1250:
generator_learning_rate = max(0, generator_learning_rate - 0.0000002)
discriminator_learning_rate = max(0, discriminator_learning_rate - 0.0000001)
'''
start_time_epoch = time.time()
pool_A, pool_B = list(coded_sps_A_norm), list(coded_sps_B_norm)
f0sA, f0sB = list(f0s_A), list(f0s_B)
dataset_A, dataset_B = sample_train_data(pool_A=pool_A, pool_B=pool_B, f0s_A=f0sA, f0s_B=f0sB, n_frames=n_frames, max_samples=max_samples)
# dataset_A, dataset_B = sample_train_data(dataset_A = coded_sps_A_norm, dataset_B = coded_sps_B_norm, n_frames = n_frames)
print('dataset_A', np.shape(dataset_A), 'dataset_B', np.shape(dataset_B))
n_samples = dataset_A.shape[0]
for i in range(n_samples // mini_batch_size):
num_iterations += 1
if num_iterations > 10000:
lambda_identity = 0
if num_iterations > 200000:
generator_learning_rate = max(0, generator_learning_rate - generator_learning_rate_decay)
discriminator_learning_rate = max(0, discriminator_learning_rate - discriminator_learning_rate_decay)
start = i * mini_batch_size
end = (i + 1) * mini_batch_size
generator_loss, discriminator_loss = model.train(input_A = dataset_A[start:end], input_B = dataset_B[start:end], lambda_cycle = lambda_cycle, lambda_identity = lambda_identity, generator_learning_rate = generator_learning_rate, discriminator_learning_rate = discriminator_learning_rate)
if i % 50 == 0:
#print('Iteration: %d, Generator Loss : %f, Discriminator Loss : %f' % (num_iterations, generator_loss, discriminator_loss))
print('Iteration: {:07d}, Generator Learning Rate: {:.7f}, Discriminator Learning Rate: {:.7f}, Generator Loss : {:.3f}, Discriminator Loss : {:.3f}'.format(num_iterations, generator_learning_rate, discriminator_learning_rate, generator_loss, discriminator_loss))
model.save(directory = model_dir, filename = model_name)
end_time_epoch = time.time()
time_elapsed_epoch = end_time_epoch - start_time_epoch
print('Time Elapsed for This Epoch: %02d:%02d:%02d' % (time_elapsed_epoch // 3600, (time_elapsed_epoch % 3600 // 60), (time_elapsed_epoch % 60 // 1)))
if generator_learning_rate <= 0:
print('training end')
break
if validation_A_dir is not None:
if epoch % 50 == 0:
print('Generating Validation Data B from A...')
for file in os.listdir(validation_A_dir):
filepath = os.path.join(validation_A_dir, file)
wav, _ = librosa.load(filepath, sr = sampling_rate, mono = True)
wav = wav_padding(wav = wav, sr = sampling_rate, frame_period = frame_period, multiple = 4)
f0, timeaxis, sp, ap = world_decompose(wav = wav, fs = sampling_rate, frame_period = frame_period)
f0_converted = pitch_conversion(f0 = f0, mean_log_src = log_f0s_mean_A, std_log_src = log_f0s_std_A, mean_log_target = log_f0s_mean_B, std_log_target = log_f0s_std_B)
coded_sp = world_encode_spectral_envelop(sp = sp, fs = sampling_rate, dim = num_mcep)
coded_sp_transposed = coded_sp.T
coded_sp_norm = (coded_sp_transposed - coded_sps_A_mean) / coded_sps_A_std
coded_sp_converted_norm = model.test(inputs = np.array([coded_sp_norm]), direction = 'A2B')[0]
coded_sp_converted = coded_sp_converted_norm * coded_sps_B_std + coded_sps_B_mean
coded_sp_converted = coded_sp_converted.T
coded_sp_converted = np.ascontiguousarray(coded_sp_converted)
decoded_sp_converted = world_decode_spectral_envelop(coded_sp = coded_sp_converted, fs = sampling_rate)
wav_transformed = world_speech_synthesis(f0 = f0_converted, decoded_sp = decoded_sp_converted, ap = ap, fs = sampling_rate, frame_period = frame_period)
librosa.output.write_wav(os.path.join(validation_A_output_dir, os.path.basename(file)), wav_transformed, sampling_rate)
if validation_B_dir is not None:
if epoch % 50 == 0:
print('Generating Validation Data A from B...')
for file in os.listdir(validation_B_dir):
filepath = os.path.join(validation_B_dir, file)
wav, _ = librosa.load(filepath, sr = sampling_rate, mono = True)
wav = wav_padding(wav = wav, sr = sampling_rate, frame_period = frame_period, multiple = 4)
f0, timeaxis, sp, ap = world_decompose(wav = wav, fs = sampling_rate, frame_period = frame_period)
f0_converted = pitch_conversion(f0 = f0, mean_log_src = log_f0s_mean_B, std_log_src = log_f0s_std_B, mean_log_target = log_f0s_mean_A, std_log_target = log_f0s_std_A)
coded_sp = world_encode_spectral_envelop(sp = sp, fs = sampling_rate, dim = num_mcep)
coded_sp_transposed = coded_sp.T
coded_sp_norm = (coded_sp_transposed - coded_sps_B_mean) / coded_sps_B_std
coded_sp_converted_norm = model.test(inputs = np.array([coded_sp_norm]), direction = 'B2A')[0]
coded_sp_converted = coded_sp_converted_norm * coded_sps_A_std + coded_sps_A_mean
coded_sp_converted = coded_sp_converted.T
coded_sp_converted = np.ascontiguousarray(coded_sp_converted)
decoded_sp_converted = world_decode_spectral_envelop(coded_sp = coded_sp_converted, fs = sampling_rate)
wav_transformed = world_speech_synthesis(f0 = f0_converted, decoded_sp = decoded_sp_converted, ap = ap, fs = sampling_rate, frame_period = frame_period)
librosa.output.write_wav(os.path.join(validation_B_output_dir, os.path.basename(file)), wav_transformed, sampling_rate)
def _find_binning_thresholds(data, max_bins, subsample, random_state):
"""Extract feature-wise quantiles from numerical data.
Missing values are ignored for finding the thresholds.
Parameters
----------
data : array-like, shape (n_samples, n_features)
The data to bin.
max_bins: int
The maximum number of bins to use for non-missing values. If for a
given feature the number of unique values is less than ``max_bins``,
then those unique values will be used to compute the bin thresholds,
instead of the quantiles.
subsample : int or None
If ``n_samples > subsample``, then ``sub_samples`` samples will be
randomly chosen to compute the quantiles. If ``None``, the whole data
is used.
random_state: int or numpy.random.RandomState or None
Pseudo-random number generator to control the random sub-sampling.
See :term:`random_state`.
Return
------
binning_thresholds: list of arrays
For each feature, stores the increasing numeric values that can
be used to separate the bins. Thus ``len(binning_thresholds) ==
n_features``.
"""
rng = check_random_state(random_state)
if subsample is not None and data.shape[0] > subsample:
subset = rng.choice(np.arange(data.shape[0]), subsample, replace=False)
data = data.take(subset, axis=0)
binning_thresholds = []
for f_idx in range(data.shape[1]):
col_data = data[:, f_idx]
# ignore missing values when computing bin thresholds
missing_mask = np.isnan(col_data)
if missing_mask.any():
col_data = col_data[~missing_mask]
col_data = np.ascontiguousarray(col_data, dtype=X_DTYPE)
distinct_values = np.unique(col_data)
if len(distinct_values) <= max_bins:
midpoints = distinct_values[:-1] + distinct_values[1:]
midpoints *= .5
else:
# We sort again the data in this case. We could compute
# approximate midpoint percentiles using the output of
# np.unique(col_data, return_counts) instead but this is more
# work and the performance benefit will be limited because we
# work on a fixed-size subsample of the full data.
percentiles = np.linspace(0, 100, num=max_bins + 1)
percentiles = percentiles[1:-1]
midpoints = np.percentile(col_data,
percentiles,
interpolation='midpoint').astype(X_DTYPE)
assert midpoints.shape[0] == max_bins - 1
# We avoid having +inf thresholds: +inf thresholds are only allowed in
# a "split on nan" situation.
np.clip(midpoints, a_min=None, a_max=ALMOST_INF, out=midpoints)
binning_thresholds.append(midpoints)
return binning_thresholds
def hash_array(s):
s = np.ascontiguousarray(s)
byte_view = s.view(np.uint8)
return hashlib.sha1(byte_view).hexdigest()
def dict_learning_online(X, n_components=2, alpha=1, n_iter=100,
return_code=True, dict_init=None, callback=None,
batch_size=3, verbose=False, shuffle=True, n_jobs=1,
method='lars', iter_offset=0, random_state=None,
n_atoms=None, chunk_size=None):
"""Solves a dictionary learning matrix factorization problem online.
Finds the best dictionary and the corresponding sparse code for
approximating the data matrix X by solving::
(U^*, V^*) = argmin 0.5 || X - U V ||_2^2 + alpha * || U ||_1
(U,V)
with || V_k ||_2 = 1 for all 0 <= k < n_components
where V is the dictionary and U is the sparse code. This is
accomplished by repeatedly iterating over mini-batches by slicing
the input data.
Parameters
----------
X: array of shape (n_samples, n_features)
Data matrix.
n_components : int,
Number of dictionary atoms to extract.
alpha : int,
Sparsity controlling parameter.
n_iter : int,
Number of iterations to perform.
return_code : boolean,
Whether to also return the code U or just the dictionary V.
dict_init : array of shape (n_components, n_features),
Initial value for the dictionary for warm restart scenarios.
callback :
Callable that gets invoked every five iterations.
batch_size : int,
The number of samples to take in each batch.
verbose :
Degree of output the procedure will print.
shuffle : boolean,
Whether to shuffle the data before splitting it in batches.
n_jobs : int,
Number of parallel jobs to run, or -1 to autodetect.
method : {'lars', 'cd'}
lars: uses the least angle regression method to solve the lasso problem
(linear_model.lars_path)
cd: uses the coordinate descent method to compute the
Lasso solution (linear_model.Lasso). Lars will be faster if
the estimated components are sparse.
iter_offset : int, default 0
Number of previous iterations completed on the dictionary used for
initialization.
random_state : int or RandomState
Pseudo number generator state used for random sampling.
Returns
-------
code : array of shape (n_samples, n_components),
the sparse code (only returned if `return_code=True`)
dictionary : array of shape (n_components, n_features),
the solutions to the dictionary learning problem
See also
--------
dict_learning
DictionaryLearning
MiniBatchDictionaryLearning
SparsePCA
MiniBatchSparsePCA
"""
if n_atoms is not None:
n_components = n_atoms
warnings.warn("Parameter n_atoms has been renamed to "
"'n_components' and will be removed in release 0.14.",
DeprecationWarning, stacklevel=2)
if chunk_size is not None:
chunk_size = batch_size
warnings.warn("Parameter chunk_size has been renamed to "
"'batch_size' and will be removed in release 0.14.",
DeprecationWarning, stacklevel=2)
if method not in ('lars', 'cd'):
raise ValueError('Coding method not supported as a fit algorithm.')
method = 'lasso_' + method
t0 = time.time()
n_samples, n_features = X.shape
# Avoid integer division problems
alpha = float(alpha)
random_state = check_random_state(random_state)
if n_jobs == -1:
n_jobs = cpu_count()
# Init V with SVD of X
if dict_init is not None:
dictionary = dict_init
else:
_, S, dictionary = randomized_svd(X, n_components)
dictionary = S[:, np.newaxis] * dictionary
r = len(dictionary)
if n_components <= r:
dictionary = dictionary[:n_components, :]
else:
dictionary = np.r_[dictionary,
np.zeros((n_components - r, dictionary.shape[1]))]
dictionary = np.ascontiguousarray(dictionary.T)
if verbose == 1:
print '[dict_learning]',
n_batches = floor(float(len(X)) / batch_size)
if shuffle:
X_train = X.copy()
random_state.shuffle(X_train)
else:
X_train = X
batches = np.array_split(X_train, n_batches)
batches = itertools.cycle(batches)
# The covariance of the dictionary
A = np.zeros((n_components, n_components))
# The data approximation
B = np.zeros((n_features, n_components))
for ii, this_X in itertools.izip(xrange(iter_offset, iter_offset + n_iter),
batches):
dt = (time.time() - t0)
if verbose == 1:
sys.stdout.write(".")
sys.stdout.flush()
elif verbose:
if verbose > 10 or ii % ceil(100. / verbose) == 0:
print ("Iteration % 3i (elapsed time: % 3is, % 4.1fmn)"
% (ii, dt, dt / 60))
this_code = sparse_encode(this_X, dictionary.T, algorithm=method,
alpha=alpha).T
# Update the auxiliary variables
if ii < batch_size - 1:
theta = float((ii + 1) * batch_size)
else:
theta = float(batch_size ** 2 + ii + 1 - batch_size)
beta = (theta + 1 - batch_size) / (theta + 1)
A *= beta
A += np.dot(this_code, this_code.T)
B *= beta
B += np.dot(this_X.T, this_code.T)
# Update dictionary
dictionary = _update_dict(dictionary, B, A, verbose=verbose,
random_state=random_state)
# XXX: Can the residuals be of any use?
# Maybe we need a stopping criteria based on the amount of
# modification in the dictionary
if callback is not None:
callback(locals())
if return_code:
if verbose > 1:
print 'Learning code...',
elif verbose == 1:
print '|',
code = sparse_encode(X, dictionary.T, algorithm=method, alpha=alpha,
n_jobs=n_jobs)
if verbose > 1:
dt = (time.time() - t0)
print 'done (total time: % 3is, % 4.1fmn)' % (dt, dt / 60)
return code, dictionary.T
return dictionary.T
def get_mnist_images(max_images=0, fold='train'):
"""Returns mnist images, batch dimension last."""
import gzip
from tensorflow.contrib.learn.python.learn.datasets import base
import numpy
def extract_images(f):
"""Extract the images into a 4D uint8 numpy array [index, y, x, depth].
Args:
f: A file object that can be passed into a gzip reader.
Returns:
data: A 4D uint8 numpy array [index, y, x, depth].
Raises:
ValueError: If the bytestream does not start with 2051.
"""
# print('Extracting', f.name) # todo: remove
with gzip.GzipFile(fileobj=f) as bytestream:
magic = _read32(bytestream)
if magic != 2051:
raise ValueError('Invalid magic number %d in MNIST image file: %s' %
(magic, f.name))
num_images = _read32(bytestream)
if max_images:
num_images = max_images
rows = _read32(bytestream)
cols = _read32(bytestream)
buf = bytestream.read(rows * cols * num_images)
data = numpy.frombuffer(buf, dtype=numpy.uint8)
data = data.reshape(num_images, rows, cols, 1)
return data
def _read32(bytestream):
dt = numpy.dtype(numpy.uint32).newbyteorder('>')
return numpy.frombuffer(bytestream.read(4), dtype=dt)[0]
if fold == 'train': # todo: rename
TRAIN_IMAGES = 'train-images-idx3-ubyte.gz'
elif fold == 'test':
TRAIN_IMAGES = 't10k-images-idx3-ubyte.gz'
else:
assert False, 'unknown fold %s'%(fold)
source_url = 'https://storage.googleapis.com/cvdf-datasets/mnist/'
local_file = base.maybe_download(TRAIN_IMAGES, '/tmp',
source_url + TRAIN_IMAGES)
train_images = extract_images(open(local_file, 'rb'))
dsize = train_images.shape[0]
if fold == 'train':
if not max_images:
dsize == 60000
else:
dsize = max_images
assert dsize <= 60000
else:
if not max_images:
dsize == 60000
else:
dsize = max_images
assert dsize <= 10000
train_images = train_images.reshape(dsize, 28**2).T.astype(np.float64)/255
train_images = np.ascontiguousarray(train_images)
return train_images.astype(default_np_dtype)
def tensor_from_rgb_image(image: np.ndarray) -> torch.Tensor:
image = np.moveaxis(image, -1, 0)
image = np.ascontiguousarray(image)
image = torch.from_numpy(image)
return image
def _pose_flip_lr(x):
"""Flip `x` horizontally."""
if isinstance(x, Pose):
return x.flip_lr()
return tensor(np.ascontiguousarray(np.array(x)[..., ::-1]))
node.outputs[0].shape= ['B',t1,'t4',t0]
graph.inputs = [node.outputs[0]]
if node.op == 'Add' and node.name == 'Add_62':
graph.outputs = [node.outputs[0]]
graph.cleanup()
onnx.save(gs.export_onnx(graph), onnxFile0)
'''
graph = gs.import_onnx(onnx.load(onnxFile0))
for node in graph.nodes:
if node.op == "MatMul" and node.name == 'MatMul_61':
convKernel = node.inputs[1].values.transpose(1, 0).reshape(
256, t1, 1, t0).astype(np.float32)
convKernelV = gs.Constant("ConvKernelV",
np.ascontiguousarray(convKernel))
continue
if node.op == "Add" and node.name == 'Add_62':
convBias = node.inputs[0].values
convBiasV = gs.Constant("ConvBiasV", np.ascontiguousarray(convBias))
continue
convV = gs.Variable("ConvV", np.dtype(np.float32), ['B', t1, 't4', 1])
convN = gs.Node("Conv",
"ConvN",
inputs=[graph.inputs[0], convKernelV, convBiasV],
outputs=[convV])
convN.attrs = OrderedDict([
('dilations', [1, 1]),
('kernel_shape', [1, t0]),
def _sample_rois(all_rois, gt_boxes, fg_rois_per_image, rois_per_image,
num_classes):
"""Generate a random sample of RoIs comprising foreground and background
examples.
"""
_bbox_para_num = 5
# overlaps: (rois x gt_boxes)
overlaps = rbbx_overlaps( # D
np.ascontiguousarray(all_rois[:, 1:_bbox_para_num + 1],
dtype=np.float32), # D
np.ascontiguousarray(gt_boxes[:, :_bbox_para_num], dtype=np.float32),
cfg.GPU_ID) # D
an_gt_diffs = angle_distance(all_rois[:, 1:_bbox_para_num + 1],
gt_boxes[:, :_bbox_para_num]) # D
gt_assignment = overlaps.argmax(axis=1)
max_overlaps = overlaps.max(axis=1)
max_overlaps_angle_diff = an_gt_diffs[np.arange(len(gt_assignment)),
gt_assignment] # D
labels = gt_boxes[gt_assignment, 5] # D: label is in the last column
# Select foreground RoIs as those with >= FG_THRESH overlap
#################### angle filter
# print np.shape(max_overlaps_angle_diff)
fg_inds = np.where((max_overlaps >= cfg.TRAIN.FG_THRESH) & (
max_overlaps_angle_diff <= cfg.TRAIN.R_POSITIVE_ANGLE_FILTER))[0] # D
####################
# print 'anglediff',max_overlaps_angle_diff[fg_inds]
# print gt_boxes
# Guard against the case when an image has fewer than fg_rois_per_image
# foreground RoIs
fg_rois_per_this_image = int(min(fg_rois_per_image, fg_inds.size))
# Sample foreground regions without replacement
if fg_inds.size > 0:
# print(type(fg_inds))
fg_inds = npr.choice(fg_inds,
size=fg_rois_per_this_image,
replace=False)
# Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI)
####################
bg_inds = np.where(
((max_overlaps < cfg.TRAIN.BG_THRESH_HI)
& (max_overlaps >= cfg.TRAIN.BG_THRESH_LO))
| ((max_overlaps >= cfg.TRAIN.FG_THRESH)
& (max_overlaps_angle_diff > cfg.TRAIN.R_NEGATIVE_ANGLE_FILTER)))[0]
####################
# print 'proposal fg',len(fg_inds),'bg',len(bg_inds)
# print
# Compute number of background RoIs to take from this image (guarding
# against there being fewer than desired)
bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image
bg_rois_per_this_image = int(min(bg_rois_per_this_image, bg_inds.size))
# Sample background regions without replacement
if bg_inds.size > 0:
bg_inds = npr.choice(bg_inds,
size=bg_rois_per_this_image,
replace=False)
# The indices that we're selecting (both fg and bg)
keep_inds = np.append(fg_inds, bg_inds)
# Select sampled values from various arrays:
labels = labels[keep_inds]
# Clamp labels for the background RoIs to 0
labels[fg_rois_per_this_image:] = 0
rois = all_rois[keep_inds]
bbox_target_data = _compute_targets(
rois[:, 1:_bbox_para_num + 1],
gt_boxes[gt_assignment[keep_inds], :_bbox_para_num], labels) # D
bbox_targets, bbox_inside_weights = \
_get_bbox_regression_labels(bbox_target_data, num_classes)
return labels, rois, bbox_targets, bbox_inside_weights
def extrude_triangulation(vertices, faces, height, transform=None, **kwargs):
"""
Extrude a 2D triangulation into a watertight mesh.
Parameters
----------
vertices : (n, 2) float
2D vertices
faces : (m, 3) int
Triangle indexes of vertices
height : float
Distance to extrude triangulation
**kwargs : dict
Passed to Trimesh constructor
Returns
---------
mesh : trimesh.Trimesh
Mesh created from extrusion
"""
vertices = np.asanyarray(vertices, dtype=np.float64)
height = float(height)
faces = np.asanyarray(faces, dtype=np.int64)
if not util.is_shape(vertices, (-1, 2)):
raise ValueError('Vertices must be (n,2)')
if not util.is_shape(faces, (-1, 3)):
raise ValueError('Faces must be (n,3)')
if np.abs(height) < tol.merge:
raise ValueError('Height must be nonzero!')
# make sure triangulation winding is pointing up
normal_test = triangles.normals([util.stack_3D(vertices[faces[0]])])[0]
normal_dot = np.dot(normal_test, [0.0, 0.0, np.sign(height)])[0]
# make sure the triangulation is aligned with the sign of
# the height we've been passed
if normal_dot < 0.0:
faces = np.fliplr(faces)
# stack the (n,3) faces into (3*n, 2) edges
edges = faces_to_edges(faces)
edges_sorted = np.sort(edges, axis=1)
# edges which only occur once are on the boundary of the polygon
# since the triangulation may have subdivided the boundary of the
# shapely polygon, we need to find it again
edges_unique = grouping.group_rows(edges_sorted, require_count=1)
# (n, 2, 2) set of line segments (positions, not references)
boundary = vertices[edges[edges_unique]]
# we are creating two vertical triangles for every 2D line segment
# on the boundary of the 2D triangulation
vertical = np.tile(boundary.reshape((-1, 2)), 2).reshape((-1, 2))
vertical = np.column_stack(
(vertical, np.tile([0, height, 0, height], len(boundary))))
vertical_faces = np.tile([3, 1, 2, 2, 1, 0], (len(boundary), 1))
vertical_faces += np.arange(len(boundary)).reshape((-1, 1)) * 4
vertical_faces = vertical_faces.reshape((-1, 3))
# stack the (n,2) vertices with zeros to make them (n, 3)
vertices_3D = util.stack_3D(vertices)
# a sequence of zero- indexed faces, which will then be appended
# with offsets to create the final mesh
faces_seq = [faces[:, ::-1], faces.copy(), vertical_faces]
vertices_seq = [
vertices_3D,
vertices_3D.copy() + [0.0, 0, height], vertical
]
# append sequences into flat nicely indexed arrays
vertices, faces = util.append_faces(vertices_seq, faces_seq)
if transform is not None:
# apply transform here to avoid later bookkeeping
vertices = tf.transform_points(vertices, transform)
# if the transform flips the winding flip faces back
# so that the normals will be facing outwards
if tf.flips_winding(transform):
# fliplr makes arrays non-contiguous
faces = np.ascontiguousarray(np.fliplr(faces))
# create mesh object with passed keywords
mesh = Trimesh(vertices=vertices, faces=faces, **kwargs)
# only check in strict mode (unit tests)
if tol.strict:
assert mesh.volume > 0.0
return mesh
def smooth(self,
positions,
fields=None,
index_fields=None,
method=None,
create_octree=False,
nneighbors=64,
kernel_name='cubic'):
r"""Operate on the mesh, in a particle-against-mesh fashion, with
non-local input.
This uses the octree indexing system to call a "smoothing" operation
(defined in yt/geometry/particle_smooth.pyx) that can take input from
several (non-local) particles and construct some value on the mesh.
The canonical example is to conduct a smoothing kernel operation on the
mesh.
Parameters
----------
positions : array_like (Nx3)
The positions of all of the particles to be examined. A new
indexed octree will be constructed on these particles.
fields : list of arrays
All the necessary fields for computing the particle operation. For
instance, this might include mass, velocity, etc.
index_fields : list of arrays
All of the fields defined on the mesh that may be used as input to
the operation.
method : string
This is the "method name" which will be looked up in the
`particle_smooth` namespace as `methodname_smooth`. Current
methods include `volume_weighted`, `nearest`, `idw`,
`nth_neighbor`, and `density`.
create_octree : bool
Should we construct a new octree for indexing the particles? In
cases where we are applying an operation on a subset of the
particles used to construct the mesh octree, this will ensure that
we are able to find and identify all relevant particles.
nneighbors : int, default 64
The number of neighbors to examine during the process.
kernel_name : string, default 'cubic'
This is the name of the smoothing kernel to use. Current supported
kernel names include `cubic`, `quartic`, `quintic`, `wendland2`,
`wendland4`, and `wendland6`.
Returns
-------
List of fortran-ordered, mesh-like arrays.
"""
# Here we perform our particle deposition.
positions.convert_to_units("code_length")
if create_octree:
morton = compute_morton(positions[:, 0], positions[:, 1],
positions[:, 2], self.ds.domain_left_edge,
self.ds.domain_right_edge)
morton.sort()
particle_octree = ParticleOctreeContainer(
[1, 1, 1],
self.ds.domain_left_edge,
self.ds.domain_right_edge,
over_refine=self._oref)
# This should ensure we get everything within one neighbor of home.
particle_octree.n_ref = nneighbors * 2
particle_octree.add(morton)
particle_octree.finalize()
pdom_ind = particle_octree.domain_ind(self.selector)
else:
particle_octree = self.oct_handler
pdom_ind = self.domain_ind
if fields is None: fields = []
if index_fields is None: index_fields = []
cls = getattr(particle_smooth, "%s_smooth" % method, None)
if cls is None:
raise YTParticleDepositionNotImplemented(method)
nz = self.nz
mdom_ind = self.domain_ind
nvals = (nz, nz, nz, (mdom_ind >= 0).sum())
op = cls(nvals, len(fields), nneighbors, kernel_name)
op.initialize()
mylog.debug("Smoothing %s particles into %s Octs", positions.shape[0],
nvals[-1])
# Pointer operations within 'process_octree' require arrays to be
# contiguous cf. https://bitbucket.org/yt_analysis/yt/issues/1079
fields = [np.ascontiguousarray(f, dtype="float64") for f in fields]
op.process_octree(self.oct_handler, mdom_ind, positions, self.fcoords,
fields, self.domain_id, self._domain_offset,
self.ds.periodicity, index_fields, particle_octree,
pdom_ind, self.ds.geometry)
# If there are 0s in the smoothing field this will not throw an error,
# but silently return nans for vals where dividing by 0
# Same as what is currently occurring, but suppressing the div by zero
# error.
with np.errstate(invalid='ignore'):
vals = op.finalize()
if vals is None: return
if isinstance(vals, list):
vals = [np.asfortranarray(v) for v in vals]
else:
vals = np.asfortranarray(vals)
return vals
def agruparTabla(self, tab):
#tab = np.array(tab)
tab = np.ascontiguousarray(tab)
unique_a = np.unique(tab.view([('', tab.dtype)] * tab.shape[1]))
return unique_a.view(tab.dtype).reshape(
(unique_a.shape[0], tab.shape[1]))
def test_ascontiguousarray():
a = afnumpy.random.random((2, 3))
b = numpy.array(a)
fassert(afnumpy.ascontiguousarray(a), numpy.ascontiguousarray(b))
# 8.99218
# 5.38 Earth, 5.13
# a=0.08036 au
if __name__ == '__main__':
with fits.open(
"hlsp_everest_k2_llc_205071984-c02_kepler_v2.0_lc.fits") as hdus:
data = hdus[1].data
t = data["TIME"]
y = data["FLUX"]
q = data["QUALITY"]
# Remove flagged EVEREST data points
m = numpy.isfinite(t) & numpy.isfinite(y)
for b in [1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 12, 13, 14, 16, 17]:
m &= (q & (2**(b - 1))) == 0
t = numpy.ascontiguousarray(t[m], dtype=numpy.float64)
y = numpy.ascontiguousarray(y[m], dtype=numpy.float64)
y = y / numpy.median(y)
skip = 64
t = t[skip:]
y = y[skip:]
#t, y = cleaned_array(t, y)
print(min(t), max(t), max(t) - min(t), len(t))
trend = scipy.signal.medfilt(y, 31)
trend = scipy.signal.savgol_filter(trend, 25, 2)
y_filt = y / trend
y_filt = sigma_clip(y_filt, sigma_upper=2, sigma_lower=float('inf'))
#for i in range(len(y_filt)):
# print(t[i], ',', y_filt[i])
# Periods
# Period 8.99198 d 2067.92701
def greycomatrix_with_nan(image,
distances,
angles,
levels=None,
symmetric=False,
normed=False):
"""Calculate the grey-level co-occurrence matrix with nan values."""
assert_nD(image, 2)
assert_nD(distances, 1, 'distances')
assert_nD(angles, 1, 'angles')
image = np.ascontiguousarray(image)
image_max = image.max()
if levels is None:
levels = 256
if image_max >= levels:
raise ValueError("The maximum grayscale value in the image should be "
"smaller than the number of levels.")
distances = np.ascontiguousarray(distances, dtype=np.float64)
angles = np.ascontiguousarray(angles, dtype=np.float64)
P = np.zeros((levels, levels, len(distances), len(angles)),
dtype=np.uint32,
order='C')
# count co-occurences
rows = image.shape[0]
cols = image.shape[1]
for a_idx in range(angles.shape[0]):
angle = angles[a_idx]
for d_idx in range(distances.shape[0]):
distance = distances[d_idx]
offset_row = round(np.sin(angle) * distance)
offset_col = round(np.cos(angle) * distance)
start_row = np.uint(max(0, -offset_row))
end_row = np.uint(min(rows, rows - offset_row))
start_col = np.uint(max(0, -offset_col))
end_col = np.uint(min(cols, cols - offset_col))
for r in range(start_row, end_row):
for c in range(start_col, end_col):
i = image[r, c]
# compute the location of the offset pixel
row = np.uint(r + offset_row)
col = np.uint(c + offset_col)
j = image[row, col]
if 0 <= i < levels and 0 <= j < levels and not np.isnan(
i) and not np.isnan(j):
P[np.uint8(i), np.uint8(j), d_idx, a_idx] += 1
# make each GLMC symmetric
if symmetric:
Pt = np.transpose(P, (1, 0, 2, 3))
P = P + Pt
# normalize each GLMC
if normed:
P = P.astype(np.float64)
glcm_sums = np.apply_over_axes(np.sum, P, axes=(0, 1))
glcm_sums[glcm_sums == 0] = 1
P /= glcm_sums
return P
mesh_refine_end = time.time()
mesh_refine_time = run_time(mesh_refine_end - mesh_refine_start, 'Mesh Refinement')
section_times.append(mesh_refine_time)
print('Mesh refined in {} s \n'.format(mesh_refine_time.time))
mesh = new_mesh
# Gathering all the data from the mesh AFTER having done the mesh refinement and defined the mesh for plotting
print('Rearranging mesh data\n')
rearrange_start = time.time()
V, vertex_number, x_coords, y_coords, z_coords, r_coords, sorting_index, x_sorted, y_sorted, z_sorted, r_sorted = rearrange_mesh_data(mesh, center_of_mass, degree_PDE)
#To be able to gather the coordinate arrays with MPI, the coordinates need to be C_contiguous
x_coords, y_coords, z_coords, r_coords = [np.ascontiguousarray(coord_array) for coord_array in [x_coords, y_coords, z_coords, r_coords]]
rearrange_end = time.time()
rearrange_time = run_time(rearrange_end - rearrange_start, 'Mesh data rearrange')
section_times.append(rearrange_time)
print('Mesh data rearranged in {} s \n'.format(rearrange_time.time))
# Defining a few BVP from combinations we use often. Naming scheme: 'weak form_source'
#BVPs for a discrete dirac mass distribution, for Newton and MOND with/out interpolations
newton_dirac = BVP(F_Newton, u_Newton, f_multiple_dirac, 'Newton, discrete dirac')
mond_deep_dirac = BVP(F_MOND_deep, u_displaced_cpp, f_multiple_dirac, 'Deep MOND, discrete dirac')
mond_simple_dirac = BVP(F_MOND_simple, u_displaced_cpp, f_multiple_dirac, 'Simple MOND, discrete dirac')
mond_standard_dirac = BVP(F_MOND_standard, u_displaced_cpp, f_multiple_dirac, 'Standard MOND, discrete dirac')
#BVPs for a discrete gauss mass distribution.