def _lazywhere(cond, arrays, f, fillvalue=None, f2=None):
"""
np.where(cond, x, fillvalue) always evaluates x even where cond is False.
This one only evaluates f(arr1[cond], arr2[cond], ...).
For example,
>>> a, b = np.array([1, 2, 3, 4]), np.array([5, 6, 7, 8])
>>> def f(a, b):
return a*b
>>> _lazywhere(a > 2, (a, b), f, np.nan)
array([ nan, nan, 21., 32.])
Notice it assumes that all `arrays` are of the same shape, or can be
broadcasted together.
"""
if fillvalue is None:
if f2 is None:
raise ValueError("One of (fillvalue, f2) must be given.")
else:
fillvalue = np.nan
else:
if f2 is not None:
raise ValueError("Only one of (fillvalue, f2) can be given.")
arrays = np.broadcast_arrays(*arrays)
temp = tuple(np.extract(cond, arr) for arr in arrays)
tcode = np.mintypecode([a.dtype.char for a in arrays])
out = _valarray(np.shape(arrays[0]), value=fillvalue, typecode=tcode)
np.place(out, cond, f(*temp))
if f2 is not None:
temp = tuple(np.extract(~cond, arr) for arr in arrays)
np.place(out, ~cond, f2(*temp))
return out
def update_strip_pattern(self, new_pos=False):
with self.visuals_changed.hold_and_emit():
if not self.block_strip:
self.block_strip = True
if new_pos:
# calculate average starting point y value
condition = (self.data_x >= self.strip_startx - 0.1) & (self.data_x <= self.strip_startx + 0.1)
section = np.extract(condition, self.data_y[:, 0])
self.avg_starty = np.average(section)
noise_starty = 2 * np.std(section) / self.avg_starty
# calculate average ending point y value
condition = (self.data_x >= self.strip_endx - 0.1) & (self.data_x <= self.strip_endx + 0.1)
section = np.extract(condition, self.data_y[:, 0])
self.avg_endy = np.average(section)
noise_endy = 2 * np.std(section) / self.avg_starty
# Calculate new slope and noise level
self.strip_slope = (self.avg_starty - self.avg_endy) / (self.strip_startx - self.strip_endx)
self.noise_level = (noise_starty + noise_endy) * 0.5
# Get the x-values in between start and end point:
condition = (self.data_x >= self.strip_startx) & (self.data_x <= self.strip_endx)
section_x = np.extract(condition, self.data_x)
# Calculate the new y-values, add noise according to noise_level
noise = self.avg_endy * 2 * (np.random.rand(*section_x.shape) - 0.5) * self.noise_level
section_y = (self.strip_slope * (section_x - self.strip_startx) + self.avg_starty) + noise
self.stripped_pattern = (section_x, section_y)
self.block_strip = False
def plotterIterations(X,nX,nIt,y,ny) :
#On se ramène à des journées
EltIteration = X[:,nIt]
Iterations = np.unique(EltIteration)
print(Iterations)
#Gérer plusieurs couleurs
colors = cm.rainbow(np.linspace(0, 1, len(Iterations)))
i = 0
for k,c in zip(Iterations,colors) :
#On prend que le jour
condition = EltIteration == k #entre 60 et 153
xPlot = np.extract(condition, X[:,nX])
yPlot = np.extract(condition, y[:,ny])
plt.scatter(xPlot,yPlot, color=c,s=2)
if i==10 :
break
i+=1
#sort
#print(np.concatenate(xPlot,yPlot))
#z = np.sort(np.concatenate(xPlot,yPlot),0)
#plt.plot(z[:,0], z[:,1], color=c,linewidth=2)
#print(k)
#print(xPlot)
#print(yPlot)
#print(z[:,0])
#print(z[:,1])
#break
#plt.plot(X_test, regr.predict(X_test), color='blue',linewidth=3)
plt.xticks(())
plt.yticks(())
plt.title('')
def diric(x,n):
"""Returns the periodic sinc function also called the dirichlet function:
diric(x) = sin(x *n / 2) / (n sin(x / 2))
where n is a positive integer.
"""
x,n = asarray(x), asarray(n)
n = asarray(n + (x-x))
x = asarray(x + (n-n))
if issubdtype(x.dtype, inexact):
ytype = x.dtype
else:
ytype = float
y = zeros(x.shape,ytype)
mask1 = (n <= 0) | (n <> floor(n))
place(y,mask1,nan)
z = asarray(x / 2.0 / pi)
mask2 = (1-mask1) & (z == floor(z))
zsub = extract(mask2,z)
nsub = extract(mask2,n)
place(y,mask2,pow(-1,zsub*(nsub-1)))
mask = (1-mask1) & (1-mask2)
xsub = extract(mask,x)
nsub = extract(mask,n)
place(y,mask,sin(nsub*xsub/2.0)/(nsub*sin(xsub/2.0)))
return y
def parzen(M, sym=True):
"""Return a Parzen window.
Parameters
----------
M : int
Number of points in the output window. If zero or less, an empty
array is returned.
sym : bool, optional
When True, generates a symmetric window, for use in filter design.
When False, generates a periodic window, for use in spectral analysis.
Returns
-------
w : ndarray
The window, with the maximum value normalized to 1 (though the value 1
does not appear if the number of samples is even and sym is True).
Examples
--------
Plot the window and its frequency response:
>>> from scipy import signal
>>> from scipy.fftpack import fft, fftshift
>>> import matplotlib.pyplot as plt
>>> window = signal.parzen(51)
>>> plt.plot(window)
>>> plt.title("Parzen window")
>>> plt.ylabel("Amplitude")
>>> plt.xlabel("Sample")
>>> plt.figure()
>>> A = fft(window, 2048) / (len(window)/2.0)
>>> freq = np.linspace(-0.5, 0.5, len(A))
>>> response = 20 * np.log10(np.abs(fftshift(A / abs(A).max())))
>>> plt.plot(freq, response)
>>> plt.axis([-0.5, 0.5, -120, 0])
>>> plt.title("Frequency response of the Parzen window")
>>> plt.ylabel("Normalized magnitude [dB]")
>>> plt.xlabel("Normalized frequency [cycles per sample]")
"""
if M < 1:
return np.array([])
if M == 1:
return np.ones(1, 'd')
odd = M % 2
if not sym and not odd:
M = M + 1
n = np.arange(-(M - 1) / 2.0, (M - 1) / 2.0 + 0.5, 1.0)
na = np.extract(n < -(M - 1) / 4.0, n)
nb = np.extract(abs(n) <= (M - 1) / 4.0, n)
wa = 2 * (1 - np.abs(na) / (M / 2.0)) ** 3.0
wb = (1 - 6 * (np.abs(nb) / (M / 2.0)) ** 2.0 +
6 * (np.abs(nb) / (M / 2.0)) ** 3.0)
w = np.r_[wa, wb, wa[::-1]]
if not sym and not odd:
w = w[:-1]
return w
def log_distances(m, sampleix=0, doplot=True, quiet=True, logv=None,
plotstyle=None, return_ixs=False):
"""Log distances (L2-norm) of data points from a reference point (sampleix).
For multiple calls to this function on the same data, it's faster to
pass logv precomputed.
return_ixs=True also returns the point indices from the sort to
be returned in this argument in a third return value.
"""
npts = size(m,0)
assert sampleix >= 0
d = zeros((npts-1,), 'd')
if rank(m) == 3:
for i in range(npts):
if sampleix != i:
try:
d[i] = norm(m[sampleix,:,:]-m[i,:,:])
except IndexError:
# catch case when index is too large for npts-1
# so use the empty i=pix position (ordering doesn't matter)
d[sampleix] = norm(m[sampleix,:,:]-m[i,:,:])
elif rank(m) == 2:
for i in range(npts):
if sampleix != i:
try:
d[i] = norm(m[sampleix,:]-m[i,:])
except IndexError:
# catch case when index is too large for npts-1
# so use the empty i=pix position (ordering doesn't matter)
d[sampleix] = norm(m[sampleix,:]-m[i,:])
else:
raise ValueError("Rank of input data must be 2 or 3")
if return_ixs:
# return sorted indices in that list argument
# (assumed to be an empty list)
ixs = array(argsort(d))
d = d[ixs]
else:
# just sort
d.sort()
if not quiet:
print("Chose reference point %i"%sampleix)
print("Min distance = %f, max distance = %f"%(d[0], d[-1]))
logd = log(d).ravel()
if logv is None:
logv = log(list(range(1,len(d)+1)))
nan_ixs = isfinite(logd) # mask, not a list of indices
logd = extract(nan_ixs, logd)
logv = extract(nan_ixs, logv)
if doplot:
if plotstyle is None:
plot(logd,logv)
else:
plot(logd,logv,plotstyle)
if return_ixs:
ixs = extract(nan_ixs, ixs)
return (logv, logd, ixs)
else:
return (logv, logd)
def reduce_grid_points(mesh_divisors,
grid_address,
dense_grid_points,
dense_grid_weights=None,
coarse_mesh_shifts=None):
divisors = np.array(mesh_divisors, dtype='intc')
if (divisors == 1).all():
coarse_grid_points = np.array(dense_grid_points, dtype='intc')
if dense_grid_weights is not None:
coarse_grid_weights = np.array(dense_grid_weights, dtype='intc')
else:
grid_weights = []
if coarse_mesh_shifts is None:
shift = [0, 0, 0]
else:
shift = np.where(coarse_mesh_shifts, divisors // 2, [0, 0, 0])
modulo = grid_address[dense_grid_points] % divisors
condition = (modulo == shift).all(axis=1)
coarse_grid_points = np.extract(condition, dense_grid_points)
if dense_grid_weights is not None:
coarse_grid_weights = np.extract(condition, dense_grid_weights)
if dense_grid_weights is None:
return coarse_grid_points
else:
return coarse_grid_points, coarse_grid_weights
def lsofcsr(coo3, dtype=float, shape=None, axis=0):
"""
Generate a list of csr matrices out of a 3-dimensional coo format
Args:
coo3 : must be a tuple (data, (i1,i2,i3)) in analogy to the tuple (data, (rows,cols)) for a common coo format
shape : a tuple of dimensions if they are known or cannot be guessed correctly from the data
axis : index (0,1 or 2) along which to construct the list of sparse arrays
Returns:
list of csr matrices
"""
(d, it) = coo3
assert len(it)==3
for ia in it: assert len(d)==len(ia)
shape = (max(ia) for ia in it) if shape is None else shape
#print( len(d) )
#print( shape )
iir = [i for i in range(len(shape)) if i!=axis]
#print(iir)
lsofcsr = [0] * shape[axis]
sh = [shape[i] for i in iir]
#print(sh, shape)
for i in range(shape[axis]):
mask = it[axis]==i
csrm = csr_matrix( (extract(mask,d), (extract(mask,it[iir[0]]),extract(mask,it[iir[1]]) )), shape=sh, dtype=dtype)
csrm.eliminate_zeros()
lsofcsr[i] = csrm
return lsofcsr
def get_linear_idxs_of_diff_bits(SortedDeltaIdxs, dists_vec, cfg):
# Identifies spots in matrix where the indices of diff bits will be.
IsPossDistLteActual = calc_IsPossDistLteActual(dists_vec, cfg)
# Columnar linear indices for the whole mtx.
UniqSortedDeltaIdxs = uniq_SortedDeltaIdxs(SortedDeltaIdxs, cfg)
# >> Which bits were actually different! (Pts were on opposite sides of HP.) <<
#
# Hack:
# We +1 to the idxs, so that none has the value 0.
# Then we replace all the idxs we don't want with 0.
# For those left, we shift back down one.
LinearIdxsOfDiffBits = (UniqSortedDeltaIdxs + 1) * IsPossDistLteActual
my_assert (lambda : LinearIdxsOfDiffBits.shape == cfg.shape)
# print 'LinearIdxsOfDiffBits (+1)'; print LinearIdxsOfDiffBits
diff_bits = np.extract(LinearIdxsOfDiffBits > 0, LinearIdxsOfDiffBits)
diff_bits -= 1
# --------------------------------
# extract(): returns non-zero vals by going along ROWs,
# not down COLs (as in Matlab). So the ORDER of these may be wacky.
# Sorting gives us the same order that Matlab would.
# --------------------------------
diff_bits.sort()
my_assert (lambda :
len(diff_bits) ==
len(np.extract(IsPossDistLteActual == True, IsPossDistLteActual)))
print 'diff_bits:', diff_bits
return diff_bits
def asymptotic_pdf(t, tres, tau, area):
"""
Calculate asymptotic probabolity density function.
Parameters
----------
t : ndarray.
Time.
tres : float
Time resolution.
tau : ndarray, shape(k, 1)
Time constants.
area : ndarray, shape(k, 1)
Component relative area.
Returns
-------
apdf : ndarray.
"""
t1 = np.extract(t[:] < tres, t)
t2 = np.extract(t[:] >= tres, t)
apdf2 = t2 * pdfs.expPDF(t2 - tres, tau, area)
apdf = np.append(t1 * 0.0, apdf2)
return apdf
def factorial2(n,exact=0):
"""n!! = special.gamma(n/2+1)*2**((m+1)/2)/sqrt(pi) n odd
= 2**(n) * n! n even
If exact==0, then floating point precision is used, otherwise
exact long integer is computed.
Notes:
- Array argument accepted only for exact=0 case.
- If n<0, the return value is 0.
"""
if exact:
if n < -1:
return 0L
if n <= 0:
return 1L
val = 1L
for k in xrange(n,0,-2):
val *= k
return val
else:
from scipy import special
n = asarray(n)
vals = zeros(n.shape,'d')
cond1 = (n % 2) & (n >= -1)
cond2 = (1-(n % 2)) & (n >= -1)
oddn = extract(cond1,n)
evenn = extract(cond2,n)
nd2o = oddn / 2.0
nd2e = evenn / 2.0
place(vals,cond1,special.gamma(nd2o+1)/sqrt(pi)*pow(2.0,nd2o+0.5))
place(vals,cond2,special.gamma(nd2e+1) * pow(2.0,nd2e))
return vals
def count_Ave_CorrCoef(NeuCountArray):
"""
FUNCTION DESCRIPTION
This function takes as input a N-D numpy.array of spike times, and outputs a spike count vector; the spike
counts are averaged over a user defined interval
:param NeuCountArray: N-D numpy.array, units are seconds, neuron spike times for each neuron stored in an numpy.array
"""
#Spike times array turned into a numpy array
NeuCountArray = np.array(NeuCountArray)
print(NeuCountArray)
#Generate array of normalized correlation coefficients (makes a symmetrical matrix)
CountCorrCoef = np.corrcoef(NeuCountArray, rowvar = True)
print(CountCorrCoef)
#Keep the upper triangle of symmetrical corr. coef. matrix and the diagnol of ones (the redunate information is converted to zeros)
UpTriCorrCoef = np.triu(CountCorrCoef)
print (UpTriCorrCoef)
#Eliminate the zeros (redunant info from symmetrical matrix) from the symmetrical corr. coef matrix
NoZero = np.extract(abs(UpTriCorrCoef) > 0, UpTriCorrCoef)
print(NoZero)
#Eliminate the ones from the symmetrical corr. coef matrix (eliminate the variances and keep covariances)
CovariancesOnly = np.extract(abs(NoZero) < 1, NoZero)
print(CovariancesOnly)
#Average the covariances
AveCov = np.mean(abs(CovariancesOnly))
return (AveCov)
def find_next_set_images(location_x,location_y,heading,file_database_sorted,picture_name_list):
image_found=0
heading,direction_vector=phase_wrap_heading(heading)
# Convert heading
phase_wrap=np.array([3, 0, 1, 2, 3, 0],dtype='u1')
heading_array=np.array([phase_wrap[heading], phase_wrap[heading+1],phase_wrap[heading+2]])
# find x values
matched_x_loc=np.extract(file_database_sorted['x_loc']==location_x,file_database_sorted)
# Check values found!!!!!
if matched_x_loc.size<4:
print "Not enough images at this x location!!, number img=\t", matched_x_loc.size
return (0,0,heading,direction_vector,0)
# find y values
matched_y_loc=np.extract(matched_x_loc['y_loc']==location_y,matched_x_loc)
# Check values found!!!!!
if matched_y_loc.size<4:
print "Not enough images at this y location!!, number img=\t", matched_y_loc.size
return (0,0,heading,direction_vector,0)
images_2_display=matched_y_loc['file_id'][heading_array]
combined_img = np.concatenate(img_file[images_2_display] , axis=1) #file_database_north_sortx[0,:]
resized_img = cv2.resize(combined_img, (image_display_width, image_display_height))
image_found=1
picture_name=picture_name_list[images_2_display[1]]
return (resized_img,image_found,heading,direction_vector,picture_name)
def get_Ic(qNow, sFinal, qMax, x, interp, extrap, weighted_transition=True):
'''return the corrected intensity based on circular symmetry'''
u = numpy.sqrt(qNow*qNow + x*x) # circular-symmetric
# divide integrand into different regions
# interpolate from existing data
u_in = numpy.extract(u <= sFinal, u)
Ic_in = numpy.exp(interp(u_in))
condition = numpy.multiply(sFinal < u, u <= qMax)
u_mid = numpy.extract(condition, u)
if u_mid.size < 2 or not weighted_transition:
Ic_mid = numpy.exp(interp(u_mid))
else:
# make smooth transition between sFinal < q < qMax
#weight = (u_mid - u_mid.min()) / (u_mid.max() - u_mid.min())
weight = numpy.linspace(0, 1.0, u_mid.size)
Ic_mid_in = numpy.exp(interp(u_mid))
Ic_mid_ex = extrap.calc(u_mid)
Ic_mid = (1-weight) * Ic_mid_in + weight * Ic_mid_ex
# extrapolate from model beyond range of available data
u_ex = numpy.extract(qMax < u, u)
Ic_ex = extrap.calc(u_ex)
# join the parts of the integrand
return numpy.concatenate((Ic_in, Ic_mid, Ic_ex))
def getClusterXYZ(RecHits, clusterID):
'''
Computes the log-energy-weighted xyzt coordinates of a given cluster. The energy is initially
reduced by a threshold amount to discard noisy hits, and then log'd to account for energy
collection fluctuations in the material.
'''
extractMask = np.logical_and.reduce((RecHits['t'] > 0,
RecHits['clusterID'] == clusterID,
RecHits['isIn3x3'])) #|Extraction mask for cluster t and xyz processing; decides initially which hits to be counted
#minLayer = np.min(newRecHits['layerID'])
#newRecHits = np.compress(newRecHits['layerID'] == minLayer, newRecHits)
x = np.extract(extractMask, RecHits['x'])
y = np.extract(extractMask, RecHits['y'])
z = np.extract(extractMask, RecHits['z'])
E = np.extract(extractMask, RecHits['en'])
return (np.mean(x), np.mean(y), np.mean(z)) #|Flat average
# Energy weight
w0 = 7 #|Arbitrary weighting term that works well, as found by Geoffrey Monet
Eweight = np.maximum(np.log(E/np.sum(E)) + w0, 0)
xw, yw, zw = (np.dot(x,Eweight)/np.sum(Eweight),
np.dot(y,Eweight)/np.sum(Eweight),
np.dot(z,Eweight)/np.sum(Eweight)) #|Weighted average
return xw, yw, zw
def makeHistogram(coadd, numBins, numImages):
"""Generate a histogram for a given coadd maskedImage
Inputs:
- coadd: a chiSquared coadd MaskedImage
- numBins: number of bins for histogram
- numImages: number of images that went into the coadd
Returns:
- histX: x values for histogram of coadd data (counts)
- histY: y values for histogram of coadd data (number of pixels)
- chiSqY: chi squared distribution values corresponding to histX
"""
coaddData = coadd.getImage().getArray()
# undo normalization
coaddData *= float(numImages)
# get rid of nans and infs
goodData = np.extract(np.isfinite(coaddData.flat), coaddData.flat)
goodData = np.extract(goodData < 50, goodData)
# compute histogram
histY, binEdges = np.histogram(goodData, bins=numBins)
histX = binEdges[0:-1]
histY = np.array(histY, dtype=float) # convert from int to float
histY /= histY.sum()
# compute chiSq probability distribution; chi squared order = numImages
chiSqY = np.power(histX, (numImages / 2.0) - 1) * np.exp(-histX / 2.0)
chiSqY /= chiSqY.sum()
return (histX, histY, chiSqY)
def sub_arr(array, lim, con_array = None, min=None, max=None, boundaries=True):
"""Purpose: Extract sub array of values between min and max limits
arguements:
array var array to take subset of
lim var array containing [min, max]
boundaries bool include boundaries ie <= and >=
keywords:
con_array var condition array to apply min/max check on
Outputs:
array of values in array with indices satisfying min < con_array < max
Call example:
function()
TODO: implement only using max or min
"""
array = check_array(array) # check array is a numpy array
assert lim[1] >= lim[0], 'min > max'
if con_array == None: # If no separate array supplied use same array for min/max
con_array = array
else:
assert np.size(con_array) != np.size(array), 'WARNING: size(con_array) != size(array)'
if boundaries == True:
sub = np.extract( (con_array>=lim[0]) * (con_array<=lim[1]), array)
else:
sub = np.extract( (con_array>lim[0]) * (con_array<lim[1]), array)
return sub
def chi_profs(bulge_prof, disk_prof, mask, weight, resid, hrad,
smoothed = False, smooth_scale = 2.0):
galprof = bulge_prof + disk_prof
if smoothed:
resid = filters.gaussian_filter(resid**2, smooth_scale)
weight = filters.gaussian_filter(weight**2, smooth_scale)
else:
resid = resid**2
weight = weight**2
new_mask = np.where(galprof > np.max(galprof)/100.0, 1,0)
galprof = image_info(galprof, mask=new_mask)
profile = image_info(resid/weight, mask = mask, x_ctr = galprof.x_ctr, y_ctr = galprof.y_ctr, ell = galprof.ba, pa= galprof.pa, zoom =-1 )
profile.profile()
rads = np.array(profile.rads)*0.396/hrad
profs = np.array(profile.prof)
cum_profs = np.array(profile.aperflux)/np.array(profile.included_pix)
cum_profs = np.extract(rads>0, cum_profs)
profs = np.extract(rads>0, profs)
rads = np.extract(rads>0, rads)
x = np.arange(0.1, 4.0,0.1)
s = interp.InterpolatedUnivariateSpline(rads,cum_profs)
ynew = s(x)
return x, ynew
def extract_velocity_cells_non_sphere( vlsvReader, cellid, origin, radius ):
''' Retrieves the velocity cells within a given sphere and returns the population outside the given sphere
:param vlsvReader: Some VlsvFile with a file open
:param cellid: The cellid whose rho to calculate
:param origin: Origin for the sphere
:param radius: Radius for the sphere
:returns: Non backstream velocity cells and their avgs values as [vcellids, avgs]
'''
# Read the velocity cells:
velocity_cell_data = vlsvReader.read_velocity_cells(cellid)
# Get cells:
vcellids = velocity_cell_data.keys()
# Get avgs data:
avgs = velocity_cell_data.values()
# Get a list of velocity coordinates shifted by the solar wind bulk velocity:
origin = np.array(origin)
v = vlsvReader.get_velocity_cell_coordinates(vcellids) - origin
# Get sum of radiuses:
radiuses = np.sum(np.abs(v)**2,axis=-1)
# Check radius condition
radius2 = radius**2
condition = (radiuses > radius2)
# Get the velocity cells of nonsphere
vcellids_nonsphere = np.extract(condition, vcellids)
# Get the avgs
avgs_nonsphere = np.extract(condition, avgs)
# Return
return [vcellids_nonsphere, avgs_nonsphere]
def dataExtraction(data = 'train', class1 = 1, class0 = 0):
import pickle, gzip
# Load the dataset
f = gzip.open('mnist.pkl.gz', 'rb')
train_set, valid_set, test_set = pickle.load(f, encoding='latin1')
f.close()
if data is 'test':
[data, labels] = test_set
else:
data = np.concatenate((train_set[0], valid_set[0]), axis = 0)
labels = np.concatenate((train_set[1], valid_set[1]), axis = 0)
y1 = np.extract(labels == class1, labels)
X1 = data[labels == class1, :]
y0 = np.extract(labels == class0, labels)
X0 = data[labels == class0, :]
y = np.concatenate((y1, y0), axis = 0)
X = np.concatenate((X1, X0), axis = 0)
#X = (X - np.mean(X, axis = 0)) / (1 + np.std(X, axis = 0)) # Data Normalization
y[y == class1] = 1
y[y == class0] = 0
y = np.reshape(y, (np.shape(X)[0], 1))
return y, X
def calculateStaticBiases (data):
## calculate global bias
# calculate mean of all the ratings
globalBias = numpy.zeros([1,1])
globalBias[0,0] = numpy.mean(data[:,2])
numpy.savetxt(globalBiasResult,globalBias,delimiter=';')
# prepare the users' biases matrix
numberOfUsers = len(numpy.unique(data[:,0]))
userMatIdBias = numpy.zeros([numberOfUsers, 2])
# prepare the items' biases matrix
numberOfItems = len(numpy.unique(data[:,1]))
itemMatIdBias = numpy.zeros([numberOfItems, 2])
## calculate user bias
# calculate mean rating for the user in the condition
for i, usrIndex in enumerate(numpy.unique(data[:,0])):
condition = data[:,0]==usrIndex
f=numpy.mean(numpy.extract(condition, data[:,2]))
f=f-globalBias[0,0]
userMatIdBias[i,:] = [usrIndex,f]
numpy.savetxt(userBiasesResult,userMatIdBias,delimiter=';')
## calculate item bias
# calculate mean rating for the item in the condition
for i, itmIndex in enumerate(numpy.unique(data[:,1])):
condition = data[:,1]==itmIndex
f=numpy.mean(numpy.extract(condition, data[:,2]))
f = f-globalBias[0,0]
itemMatIdBias[i,:] = [itmIndex,f]
numpy.savetxt(itemBiasesResult,itemMatIdBias,delimiter=';')
def normalize_player_name(img_name, debug=False):
img_name_w = matcher.MM_WHITE(sat=(0, 96), visibility=(48, 255))(img_name)
img_name_x_hist = np.extract(np.sum(img_name_w, axis=0) > 128, np.arange(img_name_w.shape[1]))
img_name_y_hist = np.extract(np.sum(img_name_w, axis=1) > 128, np.arange(img_name_w.shape[0]))
if (len(img_name_x_hist) == 0) or (len(img_name_y_hist) == 0):
# In some cases, we can't find any pixels.
return None
img_name_left = np.min(img_name_x_hist)
img_name_right = np.max(img_name_x_hist)
img_name_top = np.min(img_name_y_hist)
img_name_bottom = np.max(img_name_y_hist)
# Cropping error? should be handled gracefully.
if not (img_name_left < img_name_right):
return None
img_name_w = img_name_w[img_name_top:img_name_bottom, img_name_left:img_name_right]
img_name_w_norm = np.zeros((15, 250), dtype=np.uint8)
img_name_w_norm[:, 0 : img_name_w.shape[1]] = cv2.resize(img_name_w, (img_name_w.shape[1], 15))
if debug:
print(img_name_w_norm.shape)
cv2.imshow("name", img_name_w_norm)
cv2.waitKey(1)
return img_name_w_norm
def boxoverlap(regions_a, region_b, thre):
# (x1,y1) top-left coord, (x2,y2) bottom-right coord, (w,h) size
TP=NP=0;
TP_all=NP_all=0
N=len(region_b);
for (xb,yb,wb,hb) in region_b:
x1=np.maximum(regions_a[:,0],xb);
y1=np.maximum(regions_a[:,1],yb);
x2=np.minimum((regions_a[:,2]+regions_a[:,0]),(xb+wb));
y2=np.minimum((regions_a[:,3]+regions_a[:,1]),(yb+hb));
print x1,y1,x2,y2
w=x2-x1+1;
h=y2-y1+1;
inter=w*h;
aarea=(regions_a[:,2]+1)*(regions_a[:,3]+1);
barea=(wb+1)*(hb+1);
#intersection over union overlap
o=inter/(aarea+float(barea)-inter);
#set invalid entries to 0 overlap
o[w<=0]=0
o[h<=0]=0
TP=len(np.extract(o>=thre, o))
NP=len(np.extract(o<thre, o))
TP_all=TP_all+TP
NP_all=NP-TP_all
if NP_all<0:
NP_all=0
return TP_all, NP_all, N;
def computeMcNemarSignificance(self, truth, predictions1, predictions2): condition = (truth == 1) truth = numpy.extract(condition, truth) predictions1 = numpy.extract(condition, predictions1) predictions2 = numpy.extract(condition, predictions2) evals1 = (predictions1 == truth) evals2 = (predictions2 == truth) # Misclassified by the first model only: c01. # Misclassified by the second model only: c10. c01, c10 = 0, 0 for i, eval1 in enumerate(evals1): eval2 = evals2[i] if eval1 == 0 and eval2 == 1: c01 += 1 if eval1 == 1 and eval2 == 0: c10 += 1 if c01 + c10 < 20: print "Unreliable conclusion:", c01, c10 return 0.0 else: return math.pow(abs(c01 - c10) - 1, 2) / (c01 + c10)
def running_ave(self, rad_pix, good_image, edges):
rad_out = []
prof_out = []
proferr_out = []
aperflux = []
included_pix = []
for curr_edge in np.arange(len(edges)-1):
#print 'edges ', edges[curr_edge],edges[curr_edge+1]
tmp_im = np.extract(rad_pix < edges[curr_edge+1], good_image)
tmp_rad = np.extract(rad_pix < edges[curr_edge+1], rad_pix)
#print 'rads1'
#print tmp_rad[:10]
#raw_input()
if len(tmp_rad)>0:
aper_tmp = np.sum(tmp_im)
inc_tmp = float(tmp_im.size)
tmp_im = np.extract(tmp_rad >= edges[curr_edge], tmp_im)
tmp_rad = np.extract(tmp_rad >= edges[curr_edge], tmp_rad)
#print 'rads2'
#print tmp_rad[:10]
#raw_input()
if len(tmp_rad) > 0:
rad_out.append(np.mean(tmp_rad))
prof_out.append(np.mean(tmp_im))
proferr_out.append(np.std(tmp_im))#/(tmp_im.size-1))
aperflux.append(aper_tmp)
included_pix.append(inc_tmp)
return rad_out, prof_out,proferr_out, aperflux, included_pix
def rain_split(qlwell, channel_num=0, threshold=None, pct_boundary=0.3, split_all_peaks=False):
"""
Splits between rain and non-rain. If you want the well's auto threshold to be used,
use None as a threshold parameter (the default).
If you do not want a threshold to be calculated, use '0'. (little unclear from the spec)
Returns tuple (rain, non-rain)
"""
if threshold is None:
threshold = qlwell.channels[channel_num].statistics.threshold
ok_peaks = accepted_peaks(qlwell)
prain, rain, nrain, p_thresh, mh_thresh, ml_thresh, l_thresh = \
rain_pvalues_thresholds(ok_peaks, channel_num=channel_num, threshold=threshold, pct_boundary=pct_boundary)
if split_all_peaks:
peaks = qlwell.peaks
else:
peaks = ok_peaks
# this would be useful as a standalone, but for efficiency's sake will cut out for now
rain_condition_arr = [channel_amplitudes(peaks, channel_num) > p_thresh]
if mh_thresh and ml_thresh:
rain_condition_arr.append(np.logical_and(channel_amplitudes(peaks, channel_num) > ml_thresh,
channel_amplitudes(peaks, channel_num) < mh_thresh))
rain_condition_arr.append(channel_amplitudes(peaks, channel_num) < l_thresh)
rain_condition = reduce(np.logical_or, rain_condition_arr)
nonrain_condition = np.logical_not(rain_condition)
rain = np.extract(rain_condition, peaks)
nonrain = np.extract(nonrain_condition, peaks)
return rain, nonrain
def gap_rain(qlwell, channel_num=0, threshold=None, pct_boundary=0.3, gap_size=10000):
"""
Return the rain in the gaps between non-rain droplets.
"""
rain, nonrain = rain_split(qlwell,
channel_num=channel_num,
threshold=threshold,
pct_boundary=pct_boundary)
# ok, now identify the gaps in the gates.
times = peak_times(nonrain)
if nonrain is None or len(nonrain) < 2:
return np.ndarray([0],dtype=peak_dtype(2))
intervals = np.ediff1d(times, to_begin=0, to_end=0)
big_intervals = intervals > gap_size
# find beginning of gaps with extract
beginnings = np.extract(big_intervals[1:], times)
ends = np.extract(big_intervals[:-1], times)
gap_intervals = zip(beginnings, ends)
gap_intervals.insert(0, (0, times[0]))
gap_intervals.append((times[-1], times[-1]*100))
# count the rain in the intervals
gap_drops = np.extract(reduce(np.logical_or, [np.logical_and(peak_times(rain) > b,
peak_times(rain) < e) for b, e in gap_intervals]),
rain)
return gap_drops
def auroc(y_prob, y_true):
#threshold = [0, 0.2, 0.4, 0.6, 0.7, 0.8, 0.9, 0.99, 0.9999, 0.99999, 0.999999, 1.01]
threshold = np.sort(np.unique(y_prob))
threshold[0] = 0
threshold[threshold.shape[0]-1] = 1
tpr = np.empty_like(threshold)
fpr = np.empty_like(threshold)
for i in range(0, len(threshold)):
#print threshold[i]
predicted = np.empty_like(y_prob)
for j in range(0,y_prob.shape[0]):
if threshold[len(threshold)-1-i] != 1:
predicted[j] = int(y_prob[j] >= threshold[len(threshold)-1-i])
else:
predicted[j] = 0
a = np.extract(y_true == 1, predicted-y_true)
tpr[i] = float(a.shape[0] - np.count_nonzero(a))/np.count_nonzero(y_true)
b = np.extract(y_true == 0, predicted-y_true)
fpr[i] = np.count_nonzero(b)/float(y_true.shape[0]-np.count_nonzero(y_true))
#roc = interp1d(fpr, tpr, kind='linear')
roc_auc = trapz(tpr, fpr)
#print 'tpr', tpr
#print 'fpr', fpr
#print 'roc_auc', roc_auc
return tpr, fpr, roc_auc
def explore_transformation(dirty_dir, clean_dir, num_stds=0):
images = load_images(dirty_dir)
cleaned_images = load_images(clean_dir)
for key in images.keys():
image = images[key]
image_c = third_pass_filter(image)
cv2.imshow('original', image)
cv2.moveWindow('original', 0, 0)
cv2.imshow('cleaned by me', image_c)
cv2.moveWindow('cleaned by me', 500, 0)
them = cleaned_images[key]
cv2.imshow('clean', them)
cv2.moveWindow('clean', 500, 300)
consider_white = 200
important_me = np.extract(image_c.flatten() < consider_white, image_c.flatten())
important_them = np.extract(them.flatten() < consider_white, them.flatten())
bins = 20
plt.hist(important_them, bins, label='them')
plt.hist(important_me, bins, label='me')
plt.legend(loc='upper right')
#plt.show()
cv2.waitKey(0)
plt.close()
cv2.destroyAllWindows()
def plot_dec_boundary(theta, x, y, mapFeat=False):
theta = np.matrix(theta).T
if mapFeat is True:
u = np.linspace(-1, 1.5, 50)
v = np.linspace(-1, 1.5, 50)
else:
u = np.linspace(np.min(x[:,1]), np.max(x[:,1]), 50)
v = np.linspace(np.min(x[:,2]), np.max(x[:,2]), 50)
z = np.zeros((u.size, v.size))
#% Evaluate z = theta*x over the grid
for i in range(0,u.size):
for j in range(0,v.size):
if mapFeat is True:
x_m = mapfeat(u[i], v[j])
else:
#x_m = np.matrix([1,u[i],v[j],u[i]**2,u[i]**4])
x_m = np.matrix([1,u[i],v[j],u[i]**2])
z[i,j] = x_m * theta
z = z.T # important to transpose z before calling contour
print z, z.shape
# Plot z = 0
# Notice you need to specify the range [0, 0]
plt.figure()
#plt.scatter(x[:, 1], x[:, 2], c=y, cmap=plt.cm.Paired)
plt.scatter(np.extract(y==1, x[:, 1]), np.extract(y==1, x[:, 2]), c='b', marker='o', label='admitted')
plt.scatter(np.extract(y==0, x[:, 1]), np.extract(y==0, x[:, 2]), c='r', marker='o', label='declined')
#ax = fig.add_subplot(111)
plt.xlabel('Test 1 scores')
plt.ylabel('Test 2 scores')
plt.legend()
plt.contour(u, v, z, [0,0], linewidth = 2, cmap=plt.cm.Paired)
plt.show()
def versuch_auswerten(versuch_werte, versuch_name, header):
# Werte verarbeitbar machen
delta_l_values = pd.to_numeric(versuch_werte.delta_l_t).values
delta_r_values = pd.to_numeric(versuch_werte.delta_r_t).values
delta_m_values = pd.to_numeric(versuch_werte.delta_m_t).values
geschwindigkeit_l_values = pd.to_numeric(
versuch_werte.geschwindigkeit_l).values
geschwindigkeit_r_values = pd.to_numeric(
versuch_werte.geschwindigkeit_r).values
geschwindigkeit_m_values = pd.to_numeric(
versuch_werte.geschwindigkeit_m).values
tendenz_l_values = pd.to_numeric(versuch_werte.tendenz_l).values
tendenz_r_values = pd.to_numeric(versuch_werte.tendenz_r).values
tendenz_m_values = pd.to_numeric(versuch_werte.tendenz_m).values
blick_l_x_values = pd.to_numeric(versuch_werte.blick_l_x).values
blick_l_y_values = pd.to_numeric(versuch_werte.blick_l_y).values
blick_r_x_values = pd.to_numeric(versuch_werte.blick_r_x).values
blick_r_y_values = pd.to_numeric(versuch_werte.blick_r_y).values
sacc_m_values = pd.to_numeric(versuch_werte.sacc_m).values
sacc_l_values = pd.to_numeric(versuch_werte.sacc_l).values
sacc_r_values = pd.to_numeric(versuch_werte.sacc_r).values
# Mittelkwerte bestimmen
# Kein Exceptionhandling, da ein leeres Array dazu fuehrt, dass np.mean() nan zurueckgibt und keine Exception
if delta_l_values[np.nonzero(delta_l_values)].size == 0:
mean_delta_l = -1
else:
mean_delta_l = np.mean(delta_l_values[np.nonzero(delta_l_values)])
if delta_r_values[np.nonzero(delta_r_values)].size == 0:
mean_delta_r = -1
else:
mean_delta_r = np.mean(delta_r_values[np.nonzero(delta_r_values)])
if delta_m_values[np.nonzero(delta_m_values)].size == 0:
mean_delta_m = -1
else:
mean_delta_m = np.mean(delta_m_values[np.nonzero(delta_m_values)])
if geschwindigkeit_l_values[np.nonzero(
geschwindigkeit_l_values)].size == 0:
mean_geschwindigkeit_l = -1
else:
mean_geschwindigkeit_l = np.mean(
geschwindigkeit_l_values[np.nonzero(geschwindigkeit_l_values)])
if geschwindigkeit_r_values[np.nonzero(
geschwindigkeit_r_values)].size == 0:
mean_geschwindigkeit_r = -1
else:
mean_geschwindigkeit_r = np.mean(
geschwindigkeit_r_values[np.nonzero(geschwindigkeit_r_values)])
if geschwindigkeit_m_values[np.nonzero(
geschwindigkeit_m_values)].size == 0:
mean_geschwindigkeit_m = -1
else:
mean_geschwindigkeit_m = np.mean(
geschwindigkeit_m_values[np.nonzero(geschwindigkeit_m_values)])
header = np.append(header, [
versuch_name + '_mean_delta_l', versuch_name + '_mean_delta_r',
versuch_name + '_mean_delta_m', versuch_name +
'_mean_geschwindigkeit_l', versuch_name + '_mean_geschwindigkeit_r',
versuch_name + '_mean_geschwindigkeit_m'
])
# Maxima bestimmen
try:
max_delta_l = np.max(delta_l_values[np.nonzero(delta_l_values)])
except ValueError:
max_delta_l = -1
try:
max_delta_r = np.max(delta_r_values[np.nonzero(delta_r_values)])
except ValueError:
max_delta_r = -1
try:
max_delta_m = np.max(delta_m_values[np.nonzero(delta_m_values)])
except ValueError:
max_delta_m = -1
try:
max_geschwindigkeit_l = np.max(
geschwindigkeit_l_values[np.nonzero(geschwindigkeit_l_values)])
except ValueError:
max_geschwindigkeit_l = -1
try:
max_geschwindigkeit_r = np.max(
geschwindigkeit_r_values[np.nonzero(geschwindigkeit_r_values)])
except ValueError:
max_geschwindigkeit_r = -1
try:
max_geschwindigkeit_m = np.max(
geschwindigkeit_m_values[np.nonzero(geschwindigkeit_m_values)])
except ValueError:
max_geschwindigkeit_m = -1
header = np.append(header, [
versuch_name + '_max_delta_l', versuch_name + '_max_delta_r',
versuch_name + '_max_delta_m', versuch_name + '_max_geschwindigkeit_l',
versuch_name + '_max_geschwindigkeit_r',
versuch_name + '_max_geschwindigkeit_m'
])
# Minima bestimmen
#Exceptionhandling fuer die Versuchspersonen, bei denen nur ein Auge gemessen wurde
try:
min_delta_l = np.min(delta_l_values[np.nonzero(delta_l_values)])
except ValueError:
min_delta_l = -1
try:
min_delta_r = np.min(delta_r_values[np.nonzero(delta_r_values)])
except ValueError:
min_delta_r = -1
try:
min_delta_m = np.min(delta_m_values[np.nonzero(delta_m_values)])
except ValueError:
min_delta_m = -1
try:
min_geschwindigkeit_l = np.min(
geschwindigkeit_l_values[np.nonzero(geschwindigkeit_l_values)])
except ValueError:
min_geschwindigkeit_l = -1
try:
min_geschwindigkeit_r = np.min(
geschwindigkeit_r_values[np.nonzero(geschwindigkeit_r_values)])
except ValueError:
min_geschwindigkeit_r = -1
try:
min_geschwindigkeit_m = np.min(
geschwindigkeit_m_values[np.nonzero(geschwindigkeit_m_values)])
except ValueError:
min_geschwindigkeit_m = -1
header = np.append(header, [
versuch_name + '_min_delta_l', versuch_name + '_min_delta_r',
versuch_name + '_min_delta_m', versuch_name + '_min_geschwindigkeit_l',
versuch_name + '_min_geschwindigkeit_r',
versuch_name + '_min_geschwindigkeit_m'
])
# Standardabweichungen berechnen
if delta_l_values[np.nonzero(delta_l_values)].size == 0:
std_delta_l = -1
else:
std_delta_l = np.std(delta_l_values[np.nonzero(delta_l_values)])
if delta_r_values[np.nonzero(delta_r_values)].size == 0:
std_delta_r = -1
else:
std_delta_r = np.std(delta_r_values[np.nonzero(delta_r_values)])
if delta_m_values[np.nonzero(delta_m_values)].size == 0:
std_delta_m = -1
else:
std_delta_m = np.std(delta_m_values[np.nonzero(delta_m_values)])
if geschwindigkeit_l_values[np.nonzero(
geschwindigkeit_l_values)].size == 0:
std_geschwindigkeit_l = -1
else:
std_geschwindigkeit_l = np.std(
geschwindigkeit_l_values[np.nonzero(geschwindigkeit_l_values)])
if geschwindigkeit_r_values[np.nonzero(
geschwindigkeit_r_values)].size == 0:
std_geschwindigkeit_r = -1
else:
std_geschwindigkeit_r = np.std(
geschwindigkeit_r_values[np.nonzero(geschwindigkeit_r_values)])
if geschwindigkeit_m_values[np.nonzero(
geschwindigkeit_m_values)].size == 0:
std_geschwindigkeit_m = -1
else:
std_geschwindigkeit_m = np.std(
geschwindigkeit_m_values[np.nonzero(geschwindigkeit_m_values)])
header = np.append(header, [
versuch_name + '_standardabweichung_delta_l',
versuch_name + '_standardabweichung_delta_r',
versuch_name + '_standardabweichung_delta_m',
versuch_name + '_standardabweichung_geschwindigkeit_l',
versuch_name + '_standardabweichung_geschwindigkeit_r',
versuch_name + '_standardabweichung_geschwindigkeit_m'
])
# Varianzen berechnen
if delta_l_values[np.nonzero(delta_l_values)].size == 0:
var_delta_l = -1
else:
var_delta_l = np.var(delta_l_values[np.nonzero(delta_l_values)])
if delta_r_values[np.nonzero(delta_r_values)].size == 0:
var_delta_r = -1
else:
var_delta_r = np.var(delta_r_values[np.nonzero(delta_r_values)])
if delta_m_values[np.nonzero(delta_m_values)].size == 0:
var_delta_m = -1
else:
var_delta_m = np.var(delta_m_values[np.nonzero(delta_m_values)])
if geschwindigkeit_l_values[np.nonzero(
geschwindigkeit_l_values)].size == 0:
var_geschwindigkeit_l = -1
else:
var_geschwindigkeit_l = np.var(
geschwindigkeit_l_values[np.nonzero(geschwindigkeit_l_values)])
if geschwindigkeit_r_values[np.nonzero(
geschwindigkeit_r_values)].size == 0:
var_geschwindigkeit_r = -1
else:
var_geschwindigkeit_r = np.var(
geschwindigkeit_r_values[np.nonzero(geschwindigkeit_r_values)])
if geschwindigkeit_m_values[np.nonzero(
geschwindigkeit_m_values)].size == 0:
var_geschwindigkeit_m = -1
else:
var_geschwindigkeit_m = np.var(
geschwindigkeit_m_values[np.nonzero(geschwindigkeit_m_values)])
header = np.append(header, [
versuch_name + '_varianz_delta_l', versuch_name + '_varianz_delta_r',
versuch_name + '_varianz_delta_m',
versuch_name + '_varianz_geschwindigkeit_l',
versuch_name + '_varianz_geschwindigkeit_r',
versuch_name + '_varianz_geschwindigkeit_m'
])
# Tendenz auswerten
condition_voraus_l = np.equal(tendenz_l_values, 1)
num_voraus_l = len(np.extract(condition_voraus_l, tendenz_l_values))
condition_voraus_r = np.equal(tendenz_r_values, 1)
num_voraus_r = len(np.extract(condition_voraus_r, tendenz_r_values))
condition_voraus_m = np.equal(tendenz_m_values, 1)
num_voraus_m = len(np.extract(condition_voraus_m, tendenz_m_values))
condition_hinter_l = np.equal(tendenz_l_values, -1)
num_hinter_l = len(np.extract(condition_hinter_l, tendenz_l_values))
condition_hinter_r = np.equal(tendenz_r_values, -1)
num_hinter_r = len(np.extract(condition_hinter_r, tendenz_r_values))
condition_hinter_m = np.equal(tendenz_m_values, -1)
num_hinter_m = len(np.extract(condition_hinter_m, tendenz_m_values))
# -100 steht fuer keinen errechneten Wert, sondern fuer nich vorhanden.
if num_voraus_l == 0 and num_hinter_l == 0:
tendenz_l = -100
else:
if num_voraus_l > num_hinter_l:
tendenz_l = 1
else:
if num_hinter_l > num_voraus_l:
tendenz_l = -1
else:
tendenz_l = 0
if num_voraus_r == 0 and num_hinter_r == 0:
tendenz_r = -100
else:
if num_voraus_r > num_hinter_r:
tendenz_r = 1
else:
if num_hinter_r > num_voraus_r:
tendenz_r = -1
else:
tendenz_r = 0
if num_voraus_m == 0 and num_hinter_m == 0:
tendenz_m = -100
else:
if num_voraus_m > num_hinter_m:
tendenz_m = 1
else:
if num_hinter_m > num_voraus_m:
tendenz_m = -1
else:
tendenz_m = 0
header = np.append(header, [
versuch_name + '_tendenz_l', versuch_name + '_tendenz_r',
versuch_name + '_tendenz_m'
])
# Berechnung der Kovarianz vom linken und rechten Auge
cov_x = np.cov(blick_l_x_values, blick_r_x_values)[0][1]
cov_y = np.cov(blick_l_y_values, blick_r_y_values)[0][1]
header = np.append(header, [
versuch_name + '_Kovarianz_blick_x',
versuch_name + '_Kovarianz_blick_y'
])
verhaeltnis_l_x_da = blick_l_x_values[np.nonzero(
blick_l_x_values)].size / blick_l_x_values.size
verhaeltnis_l_y_da = blick_l_y_values[np.nonzero(
blick_l_y_values)].size / blick_l_y_values.size
verhaeltnis_r_x_da = blick_r_x_values[np.nonzero(
blick_r_x_values)].size / blick_r_x_values.size
verhaeltnis_r_y_da = blick_r_y_values[np.nonzero(
blick_r_y_values)].size / blick_r_y_values.size
sacc_m = np.sum(sacc_m_values)
sacc_l = np.sum(sacc_l_values)
sacc_r = np.sum(sacc_r_values)
if versuch_name == 'Horizontal' or versuch_name == 'Liegende_8_schnell':
sacc_rate_m = sacc_m / (999 * 4)
sacc_rate_l = sacc_l / (999 * 4)
sacc_rate_r = sacc_r / (999 * 4)
else:
sacc_rate_m = sacc_m / (999 * 5)
sacc_rate_l = sacc_l / (999 * 5)
sacc_rate_r = sacc_r / (999 * 5)
header = np.append(header, [
versuch_name + '_links_verhaeltnis_x', versuch_name +
'_links_verhaeltnis_y', versuch_name + '_rechts_verhaeltnis_x',
versuch_name + '_rechts_verhaeltnis_y', versuch_name + '_sacc_m',
versuch_name + '_sacc_rate_m', versuch_name + '_sacc_l', versuch_name +
'_sacc_rate_l', versuch_name + '_sacc_r', versuch_name + '_sacc_rate_r'
])
yield [[
mean_delta_l, mean_delta_r, mean_delta_m, mean_geschwindigkeit_l,
mean_geschwindigkeit_r, mean_geschwindigkeit_m, max_delta_l,
max_delta_r, max_delta_m, max_geschwindigkeit_l, max_geschwindigkeit_r,
max_geschwindigkeit_m, min_delta_l, min_delta_r, min_delta_m,
min_geschwindigkeit_l, min_geschwindigkeit_r, min_geschwindigkeit_m,
std_delta_l, std_delta_r, std_delta_m, std_geschwindigkeit_l,
std_geschwindigkeit_r, std_geschwindigkeit_m, var_delta_l, var_delta_r,
var_delta_m, var_geschwindigkeit_l, var_geschwindigkeit_r,
var_geschwindigkeit_m, tendenz_l, tendenz_r, tendenz_m, cov_x, cov_y,
verhaeltnis_l_x_da, verhaeltnis_l_y_da, verhaeltnis_r_x_da,
verhaeltnis_r_y_da, sacc_m, sacc_rate_m, sacc_l, sacc_rate_l, sacc_r,
sacc_rate_r
]]
yield header
def categorical_crossentropy(preds, labels):
return np.mean(-np.log(np.extract(labels, preds)))
def sawtooth(t, width=1):
"""
Return a periodic sawtooth or triangle waveform.
The sawtooth waveform has a period ``2*pi``, rises from -1 to 1 on the
interval 0 to ``width*2*pi``, then drops from 1 to -1 on the interval
``width*2*pi`` to ``2*pi``. `width` must be in the interval [0, 1].
Note that this is not band-limited. It produces an infinite number
of harmonics, which are aliased back and forth across the frequency
spectrum.
Parameters
----------
t : array_like
Time.
width : array_like, optional
Width of the rising ramp as a proportion of the total cycle.
Default is 1, producing a rising ramp, while 0 produces a falling
ramp. `t` = 0.5 produces a triangle wave.
If an array, causes wave shape to change over time, and must be the
same length as t.
Returns
-------
y : ndarray
Output array containing the sawtooth waveform.
Examples
--------
A 5 Hz waveform sampled at 500 Hz for 1 second:
>>> from scipy import signal
>>> import matplotlib.pyplot as plt
>>> t = np.linspace(0, 1, 500)
>>> plt.plot(t, signal.sawtooth(2 * np.pi * 5 * t))
"""
t, w = asarray(t), asarray(width)
w = asarray(w + (t - t))
t = asarray(t + (w - w))
if t.dtype.char in ['fFdD']:
ytype = t.dtype.char
else:
ytype = 'd'
y = zeros(t.shape, ytype)
# width must be between 0 and 1 inclusive
mask1 = (w > 1) | (w < 0)
place(y, mask1, nan)
# take t modulo 2*pi
tmod = mod(t, 2 * pi)
# on the interval 0 to width*2*pi function is
# tmod / (pi*w) - 1
mask2 = (1 - mask1) & (tmod < w * 2 * pi)
tsub = extract(mask2, tmod)
wsub = extract(mask2, w)
place(y, mask2, tsub / (pi * wsub) - 1)
# on the interval width*2*pi to 2*pi function is
# (pi*(w+1)-tmod) / (pi*(1-w))
mask3 = (1 - mask1) & (1 - mask2)
tsub = extract(mask3, tmod)
wsub = extract(mask3, w)
place(y, mask3, (pi * (wsub + 1) - tsub) / (pi * (1 - wsub)))
return y
def do_error(self, line):
"""Override the error calculation for TTS
The error is calculated as the vertical distance between theory points, in the current view,\
calculated over all possible pairs of theory tables, when the theories overlap in the horizontal direction and\
they correspond to files with the same Mw (if the parameters Mw2 and phi exist, their values are also
used to classify the error). 1/2 of the error is added to each file.
Report the error of the current theory on all the files.\n\
File error is calculated as the mean square of the residual, averaged over all calculated points in the shifted tables.\n\
Total error is the mean square of the residual, averaged over all points considered in all files.
"""
total_error = 0
npoints = 0
view = self.parent_dataset.parent_application.current_view
nfiles = len(self.parent_dataset.files)
file_error = np.zeros(nfiles)
file_points = np.zeros(nfiles, dtype=np.int)
xth = []
yth = []
xmin = np.zeros((nfiles, view.n))
xmax = np.zeros((nfiles, view.n))
for i in range(nfiles):
Filei = self.parent_dataset.files[i]
xthi, ythi, success = view.view_proc(
self.tables[Filei.file_name_short], Filei.file_parameters
)
# We need to sort arrays
for k in range(view.n):
x = xthi[:, k]
p = x.argsort()
xthi[:, k] = xthi[p, k]
ythi[:, k] = ythi[p, k]
xth.append(xthi)
yth.append(ythi)
xmin[i, :] = np.amin(xthi, 0)
xmax[i, :] = np.amax(xthi, 0)
# Mwset, Mw, Tdict = self.get_cases()
MwUnique = {}
for o in self.Mwset:
MwUnique[o] = [0.0, 0]
for i in range(nfiles):
for j in range(i + 1, nfiles):
if self.Mw[i] != self.Mw[j]:
continue
for k in range(view.n):
condition = (xth[j][:, k] > xmin[i, k]) * (
xth[j][:, k] < xmax[i, k]
)
x = np.extract(condition, xth[j][:, k])
y = np.extract(condition, yth[j][:, k])
yinterp = interp(x, xth[i][:, k], yth[i][:, k])
error = np.sum((yinterp - y) ** 2)
npt = len(y)
total_error += error
npoints += npt
MwUnique[self.Mw[i]][0] += error
MwUnique[self.Mw[i]][1] += npt
if line == "":
# table='''<table border="1" width="100%">'''
# table+='''<tr><th>Mw</th><th>Mw2</th><th>phi</th><th>phi2</th><th>Error</th><th># Pts.</th></tr>'''
table = [
[
"%-12s" % "Mw",
"%-12s" % "Mw2",
"%-12s" % "phi",
"%-12s" % "phi2",
"%-12s" % "Error",
"%-12s" % "# Pts.",
],
]
p = list(MwUnique.keys())
p.sort()
for o in p:
if MwUnique[o][1] > 0:
# table+='''<tr><td>%4g</td><td>%4g</td><td>%4g</td><td>%4g</td><td>%8.3g</td><td>(%5d)</td></tr>'''%(o[0], o[1], o[2], o[3], MwUnique[o][0] / MwUnique[o][1], MwUnique[o][1])
table.append(
[
"%-12.4g" % o[0],
"%-12.4g" % o[1],
"%-12.4g" % o[2],
"%-12.4g" % o[3],
"%-12.3g" % (MwUnique[o][0] / MwUnique[o][1]),
"%-12d" % MwUnique[o][1],
]
)
else:
# table+='''<tr><td>%4g</td><td>%4g</td><td>%4g</td><td>%4g</td><td>%s</td><td>(%5d)</td></tr>'''%(o[0], o[1], o[2], o[3], "-", MwUnique[o][1])
table.append(
[
"%-12.4g" % o[0],
"%-12.4g" % o[1],
"%-12.4g" % o[2],
"%-12.4g" % o[3],
"%-12s" % "-",
"%-12d" % MwUnique[o][1],
]
)
# table+='''</table><br>'''
self.Qprint(table)
if npoints > 0:
total_error /= npoints
else:
total_error = 1e10
if line == "":
self.Qprint("<b>TOTAL ERROR</b>: %12.5g (%6d)<br>" % (total_error, npoints))
return total_error
def calcularArea(
self, clasesMuestras
): #Calcula el area de la superficie del mapa de covertura del suelo (Raster o Vector)
error = True
self.reultados.append("Inicia proceso de cálculo de áreas")
idLayerQg = self.itemSuperficie.currentData()
if (idLayerQg == 1):
if str(os.path.splitext(
self.rutaCSVSuperficie)[1]) == ".tif" or str(
os.path.splitext(self.rutaCSVSuperficie)[1]) == ".TIF":
layerArea = QgsRasterLayer(self.rutaCSVSuperficie,
"aleatorios", "gdal")
else:
layerArea = QgsVectorLayer(self.rutaCSVSuperficie,
"aleatorios", "ogr")
else:
layerArea = QgsProject.instance().mapLayer(str(idLayerQg))
if str(layerArea.type()) == "QgsMapLayerType.VectorLayer":
features = layerArea.getFeatures()
clases = self.columClase.currentText()
features = layerArea.getFeatures()
field = [f[clases] for f in features]
Clasesunicas = np.unique(np.array(field))
matrizAreaClass = np.empty((len(Clasesunicas), 2)).astype(str)
if (len(clasesMuestras) == len(Clasesunicas)):
#print(sorted(np.array(clasesMuestras).astype(str)))
#print(sorted(np.array(Clasesunicas).astype(str)))
classMapa = sorted(np.array(Clasesunicas).astype(str))
classMuestras = sorted(np.array(clasesMuestras).astype(str))
x = np.array_equal(classMuestras, classMapa)
if (x):
i = 0
self.reultados.append("Áreas por clase:")
for clase in Clasesunicas:
expr = QgsExpression("\"{}\"='{}'".format(
clases, clase))
clasesArea = layerArea.getFeatures(
QgsFeatureRequest(expr))
suma = 0
for areas in clasesArea:
ha = areas.geometry().area() / 10000
suma = suma + ha
matrizAreaClass[i][0] = str(clase)
matrizAreaClass[i][1] = str(round(float(str(suma)), 0))
self.reultados.append(
str(matrizAreaClass[i][0]) + ": \t" +
str(matrizAreaClass[i][1]))
i += 1
else:
QMessageBox.information(
self, "Error",
"Hay clases que no coinciden en vector de muestras y mapa temático",
QMessageBox.Ok)
error = False
matrizAreaClass = 0
else:
QMessageBox.information(
self, "Error",
"Las clases del vector de muestras no tiene la misma cantidad de clases que el mapa temático",
QMessageBox.Ok)
error = False
matrizAreaClass = 0
else:
pathRaster = layerArea.dataProvider().dataSourceUri()
data = gdal.Open(pathRaster, gdal.GA_ReadOnly)
getProjec = data.GetProjection()
encontrarMetros = getProjec.find("metre")
if encontrarMetros != -1:
clasesNew = []
geotr = data.GetGeoTransform()
pixel_area = abs(geotr[1] * geotr[5])
band = data.GetRasterBand(1).ReadAsArray().astype(int)
unicos = np.unique(band)
clasesMuestras = clasesMuestras.astype(str)
unicos = unicos.astype(str)
for clase in clasesMuestras:
if clase in unicos:
clasesNew.append(clase)
if (len(clasesMuestras) == len(clasesNew)):
classMapa = sorted(np.array(clasesNew).astype(str))
classMuestras = sorted(
np.array(clasesMuestras).astype(str))
if (np.array_equal(classMuestras, classMapa)):
i = 0
self.reultados.append("Áreas por clase:")
matrizAreaClass = np.empty(
(len(classMapa), 2)).astype(str)
for clase in clasesNew:
condition = np.bitwise_not(band != int(clase))
totalClase = np.extract(condition, band)
total = len(totalClase)
matrizAreaClass[i][0] = str(clase)
matrizAreaClass[i][1] = str(
round(float(str((total * pixel_area) / 10000)),
0))
self.reultados.append(
str(matrizAreaClass[i][0]) + ": \t" +
str(matrizAreaClass[i][1]))
i += 1
else:
QMessageBox.information(
self, "Error",
"Hay clases que no coinciden en vector de muestras y mapa temático",
QMessageBox.Ok)
error = False
matrizAreaClass = 0
else:
QMessageBox.information(
self, "Error",
"Las clases del vector de muestras no tiene la misma cantidad de clases que el mapa temático",
QMessageBox.Ok)
error = False
matrizAreaClass = 0
else:
QMessageBox.information(
self, "Error",
"El Raster debe de estar en unidades Metricas",
QMessageBox.Ok)
error = False
matrizAreaClass = 0
return matrizAreaClass, error
def optimize(data, c_amp=1, c_pha=2, show_complex_plot=False):
'''
input:
- data: data array of the form the dat_reader returns it:
data = [[f1,f2,...,fn],...,[a1,a2,...,an],[ph1,ph2,...,phn],...]
columns are to be two-dimensional raw data entries in order to enable error extraction,
average data is accepted
- c_amp, c_pha: columnmn identifiers for amplitude and phase data in the data array
- show_complex_plot: (bool) (optional, default: False) plots the data points in the complex plane
if True, only in case the data columns in data are 1D
output:
- numpy data array with x values, data values and entries
of the form [[x1,x2,...,xn],[v1,v2,...,vn],[e1,e2,...,en]] if raw data was given
'''
#generate complex data array
try:
c_raw = np.array(data[c_amp]) * np.exp(1j * np.array(data[c_pha]))
except IndexError:
print 'Bad column identifier...aborting.'
except ValueError:
print 'Faulty data input, dimension mismatch...aborting.'
if len(c_raw.shape) > 1:
#calculate mean of complex data
c = np.mean(c_raw, axis=0)
else: #given data is already averaged
c = c_raw
#point in complex plane with maximum sumed mutual distances
s = np.zeros_like(np.abs(c))
for i in range(len(c)):
for p in c:
s[i] += np.abs(p - c[i])
cmax = np.extract(s == np.max(s), c)
#calculate distances
data_opt = np.abs(c - cmax)
if len(c_raw.shape) > 1:
#calculate errors in line direction
#find maximum complex point in maximum distance
d = 0
for p in c:
if np.abs(p - cmax) > d:
d = np.abs(p - cmax)
cdist = p
#find unit vector in line direction
vunit = (cdist - cmax) / np.abs(cdist - cmax)
#calculate projected distances via dot product, projecting along the data direction
#errors via std
dist_proj = [0] * len(c)
errs = np.zeros_like(c)
for i, ci in enumerate(c_raw.T): #for each iteration
dist_proj[i] = [
np.abs(
np.vdot([np.real(vunit), np.imag(vunit)], [
np.real(cr) - np.real(c[i]),
np.imag(cr) - np.imag(c[i])
])) for cr in ci
]
errs[i] = np.std(dist_proj[i]) / np.sqrt(len(dist_proj[i]))
#normalize optimized data
data_opt -= np.min(data_opt)
maxv = np.max(data_opt)
data_opt /= maxv
if len(c_raw.shape) > 1:
errs /= maxv
#gauss plane plot
if show_complex_plot:
if len(c_raw.shape) > 1:
plt.figure(figsize=(10, 13))
ax1 = plt.subplot2grid((4, 1), (0, 0))
ax2 = plt.subplot2grid((4, 1), (1, 0), rowspan=3)
ax1.errorbar(data_opt,
np.zeros_like(data_opt),
xerr=errs,
color='blue',
fmt='o',
elinewidth=0.8,
capsize=5,
markersize=8,
ecolor='red')
ax1.plot([0], [0], '*', color='red', markersize=20)
prange = np.max(data_opt) - np.min(data_opt)
ax1.set_xlim(
np.min(data_opt) - 0.05 * prange,
np.max(data_opt) + 0.05 * prange)
ax2.plot(np.real(c), np.imag(c), '.')
ax2.plot(np.real(c)[:10], np.imag(c)[:10], '.',
color='r') #show first 10 data points in red
ax2.plot(np.real(cmax),
np.imag(cmax),
'*',
color='black',
markersize=15)
else:
plt.figure(figsize=(10, 10))
plt.plot(np.real(c), np.imag(c), '.')
plt.plot(np.real(c)[:10], np.imag(c)[:10], '.',
color='r') #show first 10 data points in red
plt.plot(np.real(cmax),
np.imag(cmax),
'*',
color='black',
markersize=15)
if len(c_raw.shape) > 1:
return np.array([
np.real(np.array(data[0])),
np.real(np.array(data_opt)),
np.real(errs)
])
else:
return np.array(
[np.real(np.array(data[0])),
np.real(np.array(data_opt))])
# In[3]:
start = pd.to_datetime('2019-04-01', format='%Y-%m-%d')
end = pd.to_datetime('2019-07-01', format='%Y-%m-%d')
print(f'start: {start}')
print(f'end: {end}')
# In[4]:
s_full = pd.array(df.iloc[:, 0])
t_full = pd.array(pd.DatetimeIndex(df.iloc[:, 1]).astype(
np.int64)) / 1000000000
t_full = np.extract([s_full == 2], t_full)
dt = t_full[1] - t_full[0]
print(f'data sampling is {dt:.2f} secs')
# In[5]:
t_start = pd.DatetimeIndex([start]).astype(np.int64) / 1000000000
t_end = pd.DatetimeIndex([end]).astype(np.int64) / 1000000000
t = np.extract([(t_full >= t_start[0]) & (t_full <= t_end[0])], t_full)
t = (t - t[0]) / 60 / 60 / 24
y = np.extract([(t_full >= t_start[0]) & (t_full <= t_end[0])],
df.iloc[:, 2]).astype(np.int64)
def get_mean_and_std(self, current_image):
mask = self.get_mask(current_image)
mean = np.mean(np.extract(mask, current_image))
std = np.std(np.extract(mask, current_image))
return mean, std
def extract(cond, x):
if isinstance(x, numbers.Number):
return x
else:
return np.extract(cond, x)
def lerp(a, b, t):
return a + (b - a) * t
latent_dim = 512
ckpt = torch.load(args.ckpt)
g = Generator(args.size, latent_dim, 8).to(device)
g.load_state_dict(ckpt['g_ema'])
g.eval()
percept = lpips.PerceptualLoss(model='net-lin',
net='vgg',
use_gpu=device.startswith('cuda'))
distances = []
n_batch = args.n_sample // args.batch
resid = args.n_sample - (n_batch * args.batch)
batch_sizes = [args.batch] * n_batch + [resid]
with torch.no_grad():
for batch in tqdm(batch_sizes):
noise = g.make_noise()
inputs = torch.randn([batch * 2, latent_dim], device=device)
lerp_t = torch.rand(batch, device=device)
if args.space == 'w':
latent = g.get_latent(inputs)
latent_t0, latent_t1 = latent[::2], latent[1::2]
latent_e0 = lerp(latent_t0, latent_t1, lerp_t[:, None])
latent_e1 = lerp(latent_t0, latent_t1,
lerp_t[:, None] + args.eps)
latent_e = torch.stack([latent_e0, latent_e1],
1).view(*latent.shape)
image, _ = g([latent_e], input_is_latent=True, noise=noise)
if args.crop:
c = image.shape[2] // 8
image = image[:, :, c * 3:c * 7, c * 2:c * 6]
factor = image.shape[2] // 256
if factor > 1:
image = F.interpolate(image,
size=(256, 256),
mode='bilinear',
align_corners=False)
dist = percept(image[::2], image[1::2]).view(
image.shape[0] // 2) / (args.eps**2)
distances.append(dist.to('cpu').numpy())
distances = np.concatenate(distances, 0)
lo = np.percentile(distances, 1, interpolation='lower')
hi = np.percentile(distances, 99, interpolation='higher')
filtered_dist = np.extract(
np.logical_and(lo <= distances, distances <= hi), distances)
print('ppl:', filtered_dist.mean())
def plot( self, name='phase-plot-name', time_unit='s', legend=None,
nrows=2, ncols=2, dpi=200):
num_var = len(self.__df.columns)
if num_var == 0:
return
today = datetime.datetime.today().strftime("%d%b%y %H:%M:%S")
lead_name = name
fig_num = None
# Loop over variables and assign to the dashboards
i_dash = 0
for i_var in range(num_var):
# if multiple of nrows*ncols start new dashboard
if i_var % (nrows*ncols) == 0:
if i_var != 0: # flush any current figure
fig_name = lead_name+'-'+self.name+'-phase-plot-' + \
str(i_dash).zfill(2)
fig.savefig(fig_name+'.png', dpi=dpi, fomat='png')
plt.close(fig_num)
#pickle.dump( fig, open(fig_name+'.pickle','wb') )
i_dash += 1
fig_num = str(np.random.random()) + '.' + str(i_dash)
fig = plt.figure(num=fig_num)
gs = gridspec.GridSpec(nrows, ncols)
# gs.update(left=0.08, right=0.98, wspace=0.4, hspace=0.4)
gs.update(left=0.11, right=0.98, wspace=0.4, hspace=0.5)
axlst = list()
nPlotsNeeded = num_var - i_var
count = 0
for i in range(nrows):
for j in range(ncols):
axlst.append(fig.add_subplot(gs[i, j]))
count += 1
if count == nPlotsNeeded:
break
if count == nPlotsNeeded:
break
axes = np.array(axlst)
text = today + ': Cortix.Phase.Plot'
fig.text(.5, .95, text, horizontalalignment='center', fontsize=14)
axs = axes.flat
axId = 0
# end of: if i_var % nrows*ncols == 0: # if a multiple of nrows*ncols
# start a new dashboard
ax = axs[axId]
axId += 1
col_name = self.__df.columns[i_var]
species = self.get_species(col_name)
if species:
varName = species.formula_name
else:
quant = self.get_quantity(col_name)
varName = quant.formal_name
# sanity check
if i_var <= len(self.__species):
assert self.__species[i_var].name == self.__df.columns[i_var]
else:
assert self.__quantities[i_var].name == self.__df.columns[i_var]
varUnit = 'g/L'
'''
if varUnit == 'gram':
varUnit = 'g'
if varUnit == 'gram/min':
varUnit = 'g/min'
if varUnit == 'gram/s':
varUnit = 'g/s'
if varUnit == 'gram/m3':
varUnit = 'g/m3'
if varUnit == 'gram/L':
varUnit = 'g/L'
if varUnit == 'sec':
varUnit = 's'
'''
varLegend = legend
varScale = 'linear-linear'
assert varScale == 'log' or varScale == 'linear' or varScale == 'log-linear' \
or varScale == 'linear-log' or varScale == 'linear-linear' or \
varScale == 'log-log'
time_unit = 's'
if time_unit == 'minute':
time_unit = 'min'
x = np.array( [i for i in self.__df.index] )
if (varScale == 'linear' or varScale == 'linear-linear' or \
varScale == 'linear-log') and x.max() >= 60.0:
x /= 60.0
if time_unit == 'min':
time_unit = 'h'
if time_unit == 'second' or time_unit=='s':
time_unit = 'min'
y = np.array( self.__df[col_name] ) # convert to numpy ndarray
'''
if (y.max() >= 1e3 or y.min() <= -1e3) and varScale != 'linear-log' and \
varScale != 'log-log' and varScale != 'log':
y /= 1e3
if varUnit == 'gram' or varUnit == 'g':
varUnit = 'kg'
if varUnit == 'L':
varUnit = 'kL'
if varUnit == 'cc':
varUnit = 'L'
if varUnit == 'Ci':
varUnit = 'kCi'
if varUnit == 'W':
varUnit = 'kW'
if varUnit == 'gram/min' or varUnit == 'g/min':
varUnit = 'kg/min'
if varUnit == 'gram/s' or varUnit == 'g/s':
varUnit = 'kg/s'
if varUnit == 'gram/m3' or varUnit == 'g/m3':
varUnit = 'kg/m3'
if varUnit == 'gram/L' or varUnit == 'g/L':
varUnit = 'kg/L'
if varUnit == 'W/L':
varUnit = 'kW/L'
if varUnit == 'Ci/L':
varUnit = 'kCi/L'
if varUnit == '':
varUnit = 'x1e3'
if varUnit == 'L/min':
varUnit = 'kL/min'
if varUnit == 'Pa':
varUnit = 'kPa'
if varUnit == 's':
varUnit = 'ks'
if varUnit == 'm':
varUnit = 'km'
if varUnit == 'm/s':
varUnit = 'km/s'
if (y.max() < 1e-6 and y.min() > -1e-6) and varScale != 'linear-log' and \
varScale != 'log-log' and varScale != 'log':
y *= 1e9
if varUnit == 'gram' or varUnit == 'g':
varUnit = 'ng'
if varUnit == 'cc':
varUnit = 'n-cc'
if varUnit == 'L':
varUnit = 'nL'
if varUnit == 'W':
varUnit = 'nW'
if varUnit == 'Ci':
varUnit = 'nCi'
if varUnit == 'gram/min' or varUnit == 'g/min':
varUnit = 'ng/min'
if varUnit == 'gram/s' or varUnit == 'g/s':
varUnit = 'ng/s'
if varUnit == 'gram/m3' or varUnit == 'g/m3':
varUnit = 'ng/m3'
if varUnit == 'gram/L' or varUnit == 'g/L':
varUnit = 'ng/L'
if varUnit == 'W/L':
varUnit = 'nW/L'
if varUnit == 'Ci/L':
varUnit = 'nCi/L'
if varUnit == 'L/min':
varUnit = 'nL/min'
if varUnit == 'Pa':
varUnit = 'nPa'
if varUnit == 's':
varUnit = 'ns'
if varUnit == 'm/s':
varUnit = 'nm/s'
if (y.max() >= 1e-6 and y.max() < 1e-3) or \
(y.min() > -1e-3 and y.min() <= -1e-6) and varScale != 'linear-log' and \
varScale != 'log-log' and varScale != 'log':
y *= 1e6
if varUnit == 'gram' or varUnit == 'g':
varUnit = 'ug'
if varUnit == 'cc':
varUnit = 'u-cc'
if varUnit == 'L':
varUnit = 'uL'
if varUnit == 'W':
varUnit = 'uW'
if varUnit == 'Ci':
varUnit = 'uCi'
if varUnit == 'gram/min' or varUnit == 'g/min':
varUnit = 'ug/min'
if varUnit == 'gram/s' or varUnit == 'g/s':
varUnit = 'ug/s'
if varUnit == 'gram/m3' or varUnit == 'g/m3':
varUnit = 'ug/m3'
if varUnit == 'gram/L' or varUnit == 'g/L':
varUnit = 'ug/L'
if varUnit == 'W/L':
varUnit = 'uW/L'
if varUnit == 'Ci/L':
varUnit = 'uCi/L'
if varUnit == 'L/min':
varUnit = 'uL/min'
if varUnit == 'Pa':
varUnit = 'uPa'
if varUnit == 's':
varUnit = 'us'
if varUnit == 'm/s':
varUnit = 'um/s'
if (y.max() >= 1e-3 and y.max() < 1e-1) or \
(y.min() <= -1e-3 and y.min() > -1e-1) and varScale != 'linear-log' and \
varScale != 'log-log' and varScale != 'log':
y *= 1e3
if varUnit == 'gram' or varUnit == 'g':
varUnit = 'mg'
if varUnit == 'cc':
varUnit = 'm-cc'
if varUnit == 'L':
varUnit = 'mL'
if varUnit == 'W':
varUnit = 'mW'
if varUnit == 'Ci':
varUnit = 'mCi'
if varUnit == 'gram/min' or varUnit == 'g/min':
varUnit = 'mg/min'
if varUnit == 'gram/s' or varUnit == 'g/s':
varUnit = 'mg/s'
if varUnit == 'gram/m3' or varUnit == 'g/m3':
varUnit = 'mg/m3'
if varUnit == 'gram/L' or varUnit == 'g/L':
varUnit = 'mg/L'
if varUnit == 'W/L':
varUnit = 'mW/L'
if varUnit == 'Ci/L':
varUnit = 'mCi/L'
if varUnit == 'L/min':
varUnit = 'mL/min'
if varUnit == 'Pa':
varUnit = 'mPa'
if varUnit == 's':
varUnit = 'ms'
if varUnit == 'm/s':
varUnit = 'mm/s'
'''
ax.set_xlabel('Time [' + time_unit + ']', fontsize=9)
ax.set_ylabel(varName + ' [' + varUnit + ']', fontsize=9)
'''
ymax = y.max()
dy = ymax * .1
ymax += dy
ymin = y.min()
ymin -= dy
if abs(ymin - ymax) <= 1.e-4:
ymin = -1.0
ymax = 1.0
ax.set_ylim(ymin, ymax)
'''
if ncols >= 4:
for l in ax.get_xticklabels():
l.set_fontsize(8)
else:
for l in ax.get_xticklabels():
l.set_fontsize(10)
for l in ax.get_yticklabels():
l.set_fontsize(10)
if time_unit == 'h' and x.max() - x.min() <= 5.0:
majorLocator = MultipleLocator(1.0)
minorLocator = MultipleLocator(0.5)
ax.xaxis.set_major_locator(majorLocator)
ax.xaxis.set_minor_locator(minorLocator)
if varScale == 'log' or varScale == 'log-log':
ax.set_xscale('log')
ax.set_yscale('log')
positiveX = x > 0.0
x = np.extract(positiveX, x)
y = np.extract(positiveX, y)
positiveY = y > 0.0
x = np.extract(positiveY, x)
y = np.extract(positiveY, y)
if y.size > 0:
if y.min() > 0.0 and y.max() > y.min():
ymax = y.max()
dy = ymax * .1
ymax += dy
ymin = y.min()
ymin -= dy
if ymin < 0.0 or ymin > ymax / 1000.0:
ymin = ymax / 1000.0
ax.set_ylim(ymin, ymax)
else:
ax.set_ylim(1.0, 10.0)
else:
ax.set_ylim(1.0, 10.0)
if varScale == 'log-linear':
ax.set_xscale('log')
positiveX = x > 0.0 # True if > 0.0
x = np.extract(positiveX, x)
y = np.extract(positiveX, y)
if varScale == 'linear-log':
ax.set_yscale('log')
positiveY = y > 0.0 # True if > 0.0
x = np.extract(positiveY, x)
y = np.extract(positiveY, y)
#assert x.size == y.size, 'size error; stop.'
if y.size > 0:
if y.min() > 0.0 and y.max() > y.min():
ymax = y.max()
dy = ymax * .1
ymax += dy
ymin = y.min()
ymin -= dy
if ymin < 0.0 or ymin > ymax / 1000.0:
ymin = ymax / 1000.0
ax.set_ylim(y.min(), ymax)
else:
ax.set_ylim(1.0, 10.0)
else:
ax.set_ylim(1.0, 10.0)
# ...................
# make the plot here
yformatter = ScalarFormatter(useMathText=True,useOffset=True)
yformatter.set_powerlimits((15, 5))
ax.yaxis.set_major_formatter(yformatter)
if varLegend:
ax.plot(x, y, 's-', color='black', linewidth=0.5, markersize=2,
markeredgecolor='black', label=varLegend)
else:
ax.plot(x, y, 's-', color='black', linewidth=0.5, markersize=2,
markeredgecolor='black')
# ...................
if species.info:
ax.set_title(species.info,fontsize=8)
if varLegend:
ax.legend(loc='best', prop={'size': 7})
ax.grid()
# end of: for i_var in range(num_var):
fig_name = name+'-'+self.name+'-phase-plot-' + str(i_dash).zfill(2)
fig.savefig(fig_name+'.png', dpi=dpi, fomat='png')
plt.close(fig_num)
#pickle.dump( fig, open(fig_name+'.pickle','wb') )
return
gdal.RasterizeLayer(maskRaster, [1], vectorLayer, burn_values=[1])
maskPX = gdarr.DatasetReadAsArray(maskRaster, 0, 0, x, y)
tempPx13 = gdarr.DatasetReadAsArray(tempDataset13, 0, 0, x, y)
tempPx17 = gdarr.DatasetReadAsArray(tempDataset17, 0, 0, x, y)
# do the multiplication:
# membership value* mask = either store as 0 or store the membership value.
# output is an array
pxResult13 = numpy.multiply(maskPX, tempPx13)
pxResult17 = numpy.multiply(maskPX, tempPx17)
print("hey!")
# remove zeros from the array. convert array to a one dimentional list
pxResult13 = numpy.extract(pxResult13 > 0, pxResult13)
pxResult17 = numpy.extract(pxResult17 > 0, pxResult17)
# step6. plot and add labels and legends
plt.xlabel("pixel number")
plt.ylabel("membership value")
plt.grid()
plt.plot(pxResult13, ":", color='red', label="2013")
plt.plot(pxResult17, ":", color='green', label="2017")
plt.title("Plot Of Membership Value Along The Transect Line In 2013 And 2017")
plt.legend(loc="lower left")
plt.show()
# step7. flush the cache and clean memory
layer = None
maskBand.FlushCache()
def _get_mean_and_std(self, name, currency, league, sale_time):
"""
For a given currency sale, get the weighted mean and standard deviation.
Full returned value list is:
* mean
* standard deviation
* total of all weights used
* count of considered rows
This used to be done in the DB, but doing math in the database is
a pain, and not very portable. Numpy lets us be pretty efficient,
so we're not losing all that much.
"""
def calc_mean_std(values, weights):
mean = numpy.average(values, weights=weights)
variance = numpy.average((values - mean)**2, weights=weights)
stddev = math.sqrt(variance)
return (mean, stddev)
now = int(time.time())
# This may be DB-specific. Eventually getting it into a
# pure-SQLAlchemy form would be good...
query = self.db.session.query(poefixer.Sale)
query = query.join(poefixer.Item,
poefixer.Sale.item_id == poefixer.Item.id)
query = query.filter(poefixer.Sale.name == name)
query = query.filter(poefixer.Item.league == league)
query = query.filter(poefixer.Sale.sale_currency == currency)
# Items older than a month are really not worth anything in terms
# establishing the behavior of the economy. Even rare items like
# mirrors move fast enough for a month to be sufficient.
query = query.filter(
poefixer.Sale.item_updated_at > (now - self.relevant))
query = query.add_columns(poefixer.Sale.sale_amount,
poefixer.Sale.item_updated_at)
values = numpy.array([
(row.sale_amount,
self.weight_increment / max(1, sale_time - row.item_updated_at))
for row in query.all()
])
if len(values) == 0:
return (None, None, None, None)
prices = values[:, 0]
weights = values[:, 1]
mean, stddev = calc_mean_std(prices, weights)
count = len(prices)
total_weight = weights.sum()
if count > 3 and stddev > mean / 2:
self.logger.debug(
"%s->%s: Large stddev=%s vs mean=%s, recalibrating", name,
currency, stddev, mean)
# Throw out values outside of 2 stddev and try again
prices_ok = numpy.absolute(prices - mean) <= stddev * 2
prices = numpy.extract(prices_ok, prices)
weights = numpy.extract(prices_ok, weights)
mean, stddev = calc_mean_std(prices, weights)
count2 = len(prices)
total_weight = weights.sum()
self.logger.debug(
"Recalibration ignored %s rows, final stddev=%s, mean=%s",
count - count2, stddev, mean)
count = count2
return (float(mean), float(stddev), float(total_weight), count)
def cluster(path):
timer7 = []
start7 = timer()
#fileSize = os.path.getsize(path)/1024000
df = DF(path) # Creates dataframe from the .dat file
X_db = np.array(
df.drop(columns=['Width', 'S/N'])
) # Drops width and S/N data for DBScan to run on the DM - Time space
X = np.array(df)
# Sorts the data points by DM
points_db = X_db[X_db[:, 0].argsort()]
points = X[X[:, 0].argsort()]
# Lower DM limit below which the DBScan is now run as it is most likely not extragalactic
# Speeds up the DBScan runtime
dm_lim = 0.03 * max(points_db[:, 0])
points_new = np.array(points_db[points_db[:, 0] > dm_lim])
X_scaled = scale(1077.4, 50.32576).transform(
points_new
) # Rescales the data so that the x- and y-axes get ratio 1:1
X_scaled[:, 1] = 3 * X_scaled[:, 1]
clusters = DBSCAN(eps=xeps, min_samples=xmin).fit_predict(
X_scaled
) # Clustering algorithm, returns array with cluster labels for each point
# Re-inserts bottom points with labels -1 for RFI
length = len(points) - len(clusters)
clusters = np.insert(clusters, 0, np.full(length, -1))
# Adds column to the points arrays for cluster label
newArr = np.column_stack((points, clusters[np.newaxis].T))
# Re-order
newArr[:, -1] = clusterOrder(newArr[:, -1])
# Noise condition for Nevents<Nmin => noise
N_min = 20 # Tuneable number (set to match Karako)
labels = np.unique(newArr[:, -1])
for q in range(1, len(labels)):
label = labels[q]
labSlice = np.extract(newArr[:, -1] == label, newArr[:, -1])
if (
len(labSlice) < N_min
): # Gives points of clusters with less than N_min events the RFI lable of -1
newArr[:, -1] = np.where(newArr[:, -1] == label, -1, newArr[:, -1])
# Re-order
newArr[:, -1] = clusterOrder(newArr[:, -1])
# Break
dm_lim = 40 # Increased DM-limit to check for clusters with 'fraction' of their points below this limit
fraction = 0.05 # Size of the fraction of points allowed in below 'dm_lim'
# Condition that sets all clusters with 'fraction' of its points below dm_lim to also be classified as RFI
labels = np.unique(newArr[:, -1])
for q in range(1, len(labels)):
label = labels[q]
labSlice = np.extract(newArr[:, -1] == label, newArr[:, -1])
num_temp = int(
round(fraction * len(labSlice), 0)
) # Calculates how many points of a label a certain fraction corresponds to and rounds to nearest integer
temp = sort(
newArr[newArr[:, -1] == label][:, 0],
num_temp) # Returns the 'num_temp' lowest dms in labels_arr[q]
if (
(len(temp) > 0) and (max(temp) < dm_lim)
): # If the highest number in temp is below dm_lim then so is the rest in 'temp'
newArr[:, -1] = np.where(newArr[:, -1] == label, -1, newArr[:, -1])
# Condition that sets all points with dm below dm_lim to be classified as RFI
newArr[:, -1][newArr[:, 0] < dm_lim] = -1
# Re-order
newArr[:, -1] = clusterOrder(newArr[:, -1])
# Burst duration condition
labels = np.unique(newArr[:, -1])
for q in range(1, len(labels)):
label = labels[q]
upper = np.quantile(newArr[newArr[:, -1] == label][:, 1], 0.8)
lower = np.quantile(newArr[newArr[:, -1] == label][:, 1], 0.2)
if (
upper - lower
) >= 1: # If the time between the quantiles is longer than 1s set cluster to RFI
newArr[:, -1] = np.where(newArr[:, -1] == label, -1, newArr[:, -1])
# Re-order
newArr[:, -1] = clusterOrder(newArr[:, -1])
# Loops through all remaining clusters to exclude further clusters, calculate feature values, and classify them using Random Forest
labels = np.unique(newArr[:, -1])
end7 = timer()
timer7.append(end7 - start7)
#print("Clustering in module: ", np.mean(timer7))
return newArr, labels
def count_R_Fb(mag):
r, dens, densms, densps = np.loadtxt('density_{0}_{1}_{2}.txt'.format(name, mag, h), unpack=True, usecols = (0, 1, 3, 4))
rmax = np.amax(r)
densmax = int(np.amax(densps))+1
density_spline = spline(r, dens)
density_msigma_spline = spline(r, densms)
density_psigma_spline = spline(r, densps)
radius = np.arange(0, rmax+PRECISION, PRECISION)
density = density_spline(radius)
density_low = density_msigma_spline(radius)
density_up = density_psigma_spline(radius)
rect_deviation = np.zeros_like(radius)
for i in range(len(radius)):
rect = density[i]*(rmax-radius[i])
integral = simps(density[i:], radius[i:])
rect_deviation[i] = abs(rect-integral)
plt.plot(radius, density, 'k', linewidth=1)
plt.plot(radius, density_low, 'k', linestyle='dashed', linewidth=1)
plt.plot(radius, density_up, 'k', linestyle='dashed', linewidth=1)
for i in range(len(radius)):
if rect_deviation[i] < 0.5:
R, Fb = radius[i], density[i]
dFb = max(density_up[i]-Fb, Fb-density_low[i])
ibound = i
break
dR1, dR2, dR3, dR4 = 0, 0, 0, 0
roots = (spline(radius, density-Fb, extrapolate=False)).roots()
roots_l = (spline(radius, density_low-Fb, extrapolate=False)).roots()
roots_u = (spline(radius, density_up-Fb, extrapolate=False)).roots()
rl_left = np.extract(roots_l < R, roots_l)
ru_left = np.extract(roots_u < R, roots_u)
rl_right = np.extract(roots_l > R, roots_l)
ru_right = np.extract(roots_u > R, roots_u)
ind_r = (np.argwhere(roots == R))[0][0]
if ind_r == roots.size-1:
right_r = rmax - R
else:
right_r = roots[ind_r+1]-R
if ind_r == 0:
left_r = R
else:
left_r = R - roots[ind_r-1]
if rl_left.size != 0:
if np.amin(R-rl_left) < left_r:
dR1 = np.amin(R-rl_left)
if ru_left.size != 0:
if np.amin(R-ru_left) < left_r:
dR2 = np.amin(R-ru_left)
if rl_right.size != 0:
if np.amin(rl_right-R) < right_r:
dR3 = np.amin(rl_right-R)
if ru_right.size != 0:
if np.amin(ru_right-R) < right_r:
dR4 = np.amin(ru_right-R)
dR = max(dR1, dR2, dR3, dR4)
if dR > R:
dR = R
if dFb > Fb:
dFb = Fb
ax = plt.gca()
for tick in ax.xaxis.get_major_ticks():
tick.label.set_fontsize(12)
for tick in ax.yaxis.get_major_ticks():
tick.label.set_fontsize(12)
ax.axhline(Fb, c='b', alpha=0.8, lw=0.5, label = "Fb = {0:5.2f} ± {1:5.2f}".format(Fb, dFb))
ax.axvline(R, c='r', alpha=0.8, lw=0.5, label = "R = {0:5.2f} ± {1:5.2f}".format(R, dR))
ax.tick_params(width=2)
ax.grid(alpha=0.2, linestyle='dashed', linewidth=0.5)
ax.set_xlim(0, rmax)
ax.set_ylim(0, densmax)
ax.legend()
plt.title("Радиальный профиль плотности {0},\nh={1}', maglim={2}m".format(name, h, mag), fontsize=18)
ax.set_xlabel(r'Угловое расстояние от центра, $arcmin$', fontsize=16)
ax.set_ylabel(r'Радиальная плотность, $arcmin^{-2}$', fontsize=16)
plt.savefig('density_{0}_{1}_{2}_RFb.png'.format(name, mag, h), dpi=300)
plt.close()
return (R, dR, Fb, dFb)
def glrm_orthonnmf():
m = 1000
n = 100
k = 10
print "Uploading random uniform matrix with rows = " + str(
m) + " and cols = " + str(n)
Y = np.random.rand(k, n)
X = np.random.rand(m, k)
train = np.dot(X, Y)
train_h2o = h2o.H2OFrame(train.tolist())
print "Run GLRM with orthogonal non-negative regularization on X, non-negative regularization on Y"
initial_y = np.random.rand(k, n)
initial_y_h2o = h2o.H2OFrame(initial_y.tolist())
glrm_h2o = h2o.glrm(x=train_h2o,
k=k,
init="User",
user_y=initial_y_h2o,
loss="Quadratic",
regularization_x="OneSparse",
regularization_y="NonNegative",
gamma_x=1,
gamma_y=1)
glrm_h2o.show()
print "Check that X and Y matrices are non-negative"
fit_y = glrm_h2o._model_json['output']['archetypes'].cell_values
fit_y_np = [[float(s) for s in list(row)[1:]] for row in fit_y]
fit_y_np = np.array(fit_y_np)
fit_x = h2o.get_frame(
glrm_h2o._model_json['output']['loading_key']['name'])
fit_x_np = np.array(h2o.as_list(fit_x))
assert np.all(fit_y_np >= 0), "Y must contain only non-negative elements"
assert np.all(fit_x_np >= 0), "X must contain only non-negative elements"
print "Check that columns of X are orthogonal"
xtx = np.dot(np.transpose(fit_x_np), fit_x_np)
offdiag = np.extract(1 - np.eye(k), xtx)
assert np.all(
offdiag == 0), "All off diagonal elements of X'X must equal zero"
print "Check final objective function value"
fit_xy = np.dot(fit_x_np, fit_y_np)
glrm_obj = glrm_h2o._model_json['output']['objective']
sse = np.sum(np.square(train.__sub__(fit_xy)))
assert abs(glrm_obj - sse) < 1e-6, "Final objective was " + str(
glrm_obj) + " but should equal " + str(sse)
print "Impute XY and check error metrics"
pred_h2o = glrm_h2o.predict(train_h2o)
pred_np = np.array(h2o.as_list(pred_h2o))
assert np.allclose(
pred_np, fit_xy
), "Imputation for numerics with quadratic loss should equal XY product"
glrm_numerr = glrm_h2o._model_json['output'][
'training_metrics']._metric_json['numerr']
glrm_caterr = glrm_h2o._model_json['output'][
'training_metrics']._metric_json['caterr']
assert abs(glrm_numerr - glrm_obj) < 1e-3, "Numeric error was " + str(
glrm_numerr) + " but should equal final objective " + str(glrm_obj)
assert glrm_caterr == 0, "Categorical error was " + str(
glrm_caterr) + " but should be zero"
print "Run GLRM with orthogonal non-negative regularization on both X and Y"
initial_y = np.random.rand(k, n)
initial_y_h2o = h2o.H2OFrame(initial_y.tolist())
glrm_h2o = h2o.glrm(x=train_h2o,
k=k,
init="User",
user_y=initial_y_h2o,
loss="Quadratic",
regularization_x="OneSparse",
regularization_y="OneSparse",
gamma_x=1,
gamma_y=1)
glrm_h2o.show()
print "Check that X and Y matrices are non-negative"
fit_y = glrm_h2o._model_json['output']['archetypes'].cell_values
fit_y_np = [[float(s) for s in list(row)[1:]] for row in fit_y]
fit_y_np = np.array(fit_y_np)
fit_x = h2o.get_frame(
glrm_h2o._model_json['output']['loading_key']['name'])
fit_x_np = np.array(h2o.as_list(fit_x))
assert np.all(fit_y_np >= 0), "Y must contain only non-negative elements"
assert np.all(fit_x_np >= 0), "X must contain only non-negative elements"
print "Check that columns of X are orthogonal"
xtx = np.dot(np.transpose(fit_x_np), fit_x_np)
offdiag_x = np.extract(1 - np.eye(k), xtx)
assert np.all(
offdiag_x == 0), "All off diagonal elements of X'X must equal zero"
print "Check that rows of Y are orthogonal"
yyt = np.dot(fit_y_np, np.transpose(fit_y_np))
offdiag_y = np.extract(1 - np.eye(k), yyt)
assert np.all(
offdiag_y == 0), "All off diagonal elements of YY' must equal zero"
print "Check final objective function value"
fit_xy = np.dot(fit_x_np, fit_y_np)
glrm_obj = glrm_h2o._model_json['output']['objective']
sse = np.sum(np.square(train.__sub__(fit_xy)))
assert abs(glrm_obj - sse) < 1e-6, "Final objective was " + str(
glrm_obj) + " but should equal " + str(sse)
print "Impute XY and check error metrics"
pred_h2o = glrm_h2o.predict(train_h2o)
pred_np = np.array(h2o.as_list(pred_h2o))
assert np.allclose(
pred_np, fit_xy
), "Imputation for numerics with quadratic loss should equal XY product"
glrm_numerr = glrm_h2o._model_json['output'][
'training_metrics']._metric_json['numerr']
glrm_caterr = glrm_h2o._model_json['output'][
'training_metrics']._metric_json['caterr']
assert abs(glrm_numerr - glrm_obj) < 1e-3, "Numeric error was " + str(
glrm_numerr) + " but should equal final objective " + str(glrm_obj)
assert glrm_caterr == 0, "Categorical error was " + str(
glrm_caterr) + " but should be zero"
def WENO(thing):
"""
I have to split this up. It has to return a value depending on if
the original value was positive or negative. It is a small waste
but I can just calculate the entire thing both the positive way and
the negative way, then select out the values that were originally
positive or negative and then put them into the final array
"""
# determine the indices where positive
positive_indices = []
for i in range(len(thing)):
if thing[i] >= 0:
positive_indices.append(i)
# determine the indices where negative
negative_indices = []
for i in range(len(thing)):
if thing[i] < 0:
negative_indices.append(i)
# calculate positive
um2 = roll(thing, 2)
um1 = roll(thing, 1)
up1 = roll(thing, -1)
up2 = roll(thing, -2)
u0 = (1. / 3.) * um2 - (7. / 6.) * um1 + (11. / 6.) * thing
u1 = -(1. / 6.) * um1 + (5. / 6.) * thing + (1. / 3.) * up1
u2 = (1. / 3.) * thing + (5. / 6.) * up1 - (1. / 6.) * up2
# nonlinear weights
epsilon = 10**-6
beta0 = (13. / 12.) * (um2 - 2 * um1 + thing)**2 + .25 * (um2 - 4 * um1 +
3 * thing)**2
beta1 = (13. / 12.) * (um1 - 2 * thing + up1)**2 + .25 * (um1 - up1)**2
beta2 = (13. / 12.) * (thing - 2 * up1 + up2)**2 + .25 * (3 * thing -
4 * up1 + up2)**2
beta = [beta0, beta1, beta2]
gamma = [.1, .6, .3]
omegatilde = [0, 0, 0]
for i in range(3):
omegatilde[i] = gamma[i] / (epsilon + beta[i])**2
omega = [0, 0, 0]
for i in range(3):
omega[i] = omegatilde[i] / sum(omegatilde)
u_hlf = omega[0] * u0 + omega[1] * u1 + omega[2] * u2
flux = (u_hlf)**2
duudx = -(flux - roll(flux, 1)) / dx
# calculate negative
um1 = np.roll(thing, 1)
up1 = np.roll(thing, -1)
up2 = np.roll(thing, -2)
up3 = np.roll(thing, -3)
u0 = (1. / 3.) * up3 - (7. / 6.) * up2 + (11. / 6.) * up1
u1 = -(1. / 6.) * up2 + (5. / 6.) * up1 + (1. / 3.) * thing
u2 = (1. / 3.) * up1 + (5. / 6.) * thing - (1. / 6.) * um1
# nonlinear weights
epsilon = 10**-6
beta0 = (13. / 12.) * (up3 - 2 * up2 + up1)**2 + .25 * (up3 - 4 * up2 +
3 * up1)**2
beta1 = (13. / 12.) * (up2 - 2 * up1 + thing)**2 + .25 * (up2 - thing)**2
beta2 = (13. / 12.) * (up1 - 2 * thing +
um1)**2 + .25 * (3 * up1 - 4 * thing + um1)**2
beta = [beta0, beta1, beta2]
gamma = [.1, .6, .3]
omegatilde = [0, 0, 0]
for i in range(3):
omegatilde[i] = gamma[i] / (epsilon + beta[i])**2
omega = [0, 0, 0]
for i in range(3):
omega[i] = omegatilde[i] / sum(omegatilde)
u_hlfneg = omega[0] * u0 + omega[1] * u1 + omega[2] * u2
fluxneg = (u_hlfneg)**2
duudxneg = -(fluxneg - roll(fluxneg, 1)) / dx
# extract values where was positive
positive_values = np.extract(thing >= 0, duudx)
# negative
negative_values = np.extract(thing < 0, duudxneg)
# combine into new list
positive = np.zeros(nx)
np.put(positive, positive_indices, positive_values)
negative = np.zeros(nx)
np.put(negative, negative_indices, negative_values)
dudt = np.zeros(nx)
np.put(dudt, positive_indices, positive_values)
np.put(dudt, negative_indices, negative_values)
return dudt
class Bfloat16Test(parameterized.TestCase):
"""Tests the non-numpy Python methods of the bfloat16 type."""
def testRoundTripToFloat(self):
for v in FLOAT_VALUES:
np.testing.assert_equal(v, float(bfloat16(v)))
def testRoundTripNumpyTypes(self):
for dtype in [np.float16, np.float32, np.float64, np.longdouble]:
np.testing.assert_equal(-3.75, dtype(bfloat16(dtype(-3.75))))
np.testing.assert_equal(1.5, float(bfloat16(dtype(1.5))))
np.testing.assert_equal(4.5, dtype(bfloat16(np.array(4.5, dtype))))
np.testing.assert_equal(np.array([2, 5, -1], bfloat16),
bfloat16(np.array([2, 5, -1], dtype)))
def testRoundTripToInt(self):
for v in [
-256, -255, -34, -2, -1, 0, 1, 2, 10, 47, 128, 255, 256, 512
]:
self.assertEqual(v, int(bfloat16(v)))
# pylint: disable=g-complex-comprehension
@parameterized.named_parameters(
({
"testcase_name": "_" + dtype.__name__,
"dtype": dtype
} for dtype in
[bfloat16, np.float16, np.float32, np.float64, np.longdouble]))
def testRoundTripToNumpy(self, dtype):
for v in FLOAT_VALUES:
np.testing.assert_equal(v, bfloat16(dtype(v)))
np.testing.assert_equal(v, dtype(bfloat16(dtype(v))))
np.testing.assert_equal(v, dtype(bfloat16(np.array(v, dtype))))
if dtype != bfloat16:
np.testing.assert_equal(
np.array(FLOAT_VALUES, dtype),
bfloat16(np.array(FLOAT_VALUES, dtype)).astype(dtype))
def testStr(self):
self.assertEqual("0", str(bfloat16(0.0)))
self.assertEqual("1", str(bfloat16(1.0)))
self.assertEqual("-3.5", str(bfloat16(-3.5)))
self.assertEqual("0.0078125", str(bfloat16(float.fromhex("1.0p-7"))))
self.assertEqual("inf", str(bfloat16(float("inf"))))
self.assertEqual("-inf", str(bfloat16(float("-inf"))))
self.assertEqual("nan", str(bfloat16(float("nan"))))
def testRepr(self):
self.assertEqual("0", repr(bfloat16(0)))
self.assertEqual("1", repr(bfloat16(1)))
self.assertEqual("-3.5", repr(bfloat16(-3.5)))
self.assertEqual("0.0078125", repr(bfloat16(float.fromhex("1.0p-7"))))
self.assertEqual("inf", repr(bfloat16(float("inf"))))
self.assertEqual("-inf", repr(bfloat16(float("-inf"))))
self.assertEqual("nan", repr(bfloat16(float("nan"))))
def testHashZero(self):
"""Tests that negative zero and zero hash to the same value."""
self.assertEqual(hash(bfloat16(-0.0)), hash(bfloat16(0.0)))
@parameterized.parameters(
np.extract(np.isfinite(FLOAT_VALUES), FLOAT_VALUES))
def testHashNumbers(self, value):
self.assertEqual(hash(value), hash(bfloat16(value)), str(value))
@parameterized.named_parameters(("PositiveNan", bfloat16(float("nan"))),
("NegativeNan", bfloat16(float("-nan"))))
def testHashNan(self, nan):
nan_hash = hash(nan)
nan_object_hash = object.__hash__(nan)
# The hash of a NaN is either 0 or a hash of the object pointer.
self.assertIn(nan_hash, (sys.hash_info.nan, nan_object_hash), str(nan))
def testHashInf(self):
self.assertEqual(sys.hash_info.inf, hash(bfloat16(float("inf"))),
"inf")
self.assertEqual(-sys.hash_info.inf, hash(bfloat16(float("-inf"))),
"-inf")
# Tests for Python operations
def testNegate(self):
for v in FLOAT_VALUES:
np.testing.assert_equal(-v, float(-bfloat16(v)))
def testAdd(self):
np.testing.assert_equal(0, float(bfloat16(0) + bfloat16(0)))
np.testing.assert_equal(1, float(bfloat16(1) + bfloat16(0)))
np.testing.assert_equal(0, float(bfloat16(1) + bfloat16(-1)))
np.testing.assert_equal(5.5, float(bfloat16(2) + bfloat16(3.5)))
np.testing.assert_equal(1.25, float(bfloat16(3.5) + bfloat16(-2.25)))
np.testing.assert_equal(
float("inf"), float(bfloat16(float("inf")) + bfloat16(-2.25)))
np.testing.assert_equal(
float("-inf"), float(bfloat16(float("-inf")) + bfloat16(-2.25)))
self.assertTrue(
math.isnan(float(bfloat16(3.5) + bfloat16(float("nan")))))
def testAddScalarTypePromotion(self):
"""Tests type promotion against Numpy scalar values."""
types = [bfloat16, np.float16, np.float32, np.float64, np.longdouble]
for lhs_type in types:
for rhs_type in types:
expected_type = numpy_promote_types(lhs_type, rhs_type)
actual_type = type(lhs_type(3.5) + rhs_type(2.25))
self.assertEqual(expected_type, actual_type)
def testAddArrayTypePromotion(self):
self.assertEqual(np.float32,
type(bfloat16(3.5) + np.array(2.25, np.float32)))
self.assertEqual(np.float32,
type(np.array(3.5, np.float32) + bfloat16(2.25)))
def testSub(self):
np.testing.assert_equal(0, float(bfloat16(0) - bfloat16(0)))
np.testing.assert_equal(1, float(bfloat16(1) - bfloat16(0)))
np.testing.assert_equal(2, float(bfloat16(1) - bfloat16(-1)))
np.testing.assert_equal(-1.5, float(bfloat16(2) - bfloat16(3.5)))
np.testing.assert_equal(5.75, float(bfloat16(3.5) - bfloat16(-2.25)))
np.testing.assert_equal(
float("-inf"), float(bfloat16(-2.25) - bfloat16(float("inf"))))
np.testing.assert_equal(
float("inf"), float(bfloat16(-2.25) - bfloat16(float("-inf"))))
self.assertTrue(
math.isnan(float(bfloat16(3.5) - bfloat16(float("nan")))))
def testMul(self):
np.testing.assert_equal(0, float(bfloat16(0) * bfloat16(0)))
np.testing.assert_equal(0, float(bfloat16(1) * bfloat16(0)))
np.testing.assert_equal(-1, float(bfloat16(1) * bfloat16(-1)))
np.testing.assert_equal(-7.875, float(bfloat16(3.5) * bfloat16(-2.25)))
np.testing.assert_equal(
float("-inf"), float(bfloat16(float("inf")) * bfloat16(-2.25)))
np.testing.assert_equal(
float("inf"), float(bfloat16(float("-inf")) * bfloat16(-2.25)))
self.assertTrue(
math.isnan(float(bfloat16(3.5) * bfloat16(float("nan")))))
def testDiv(self):
self.assertTrue(math.isnan(float(bfloat16(0) / bfloat16(0))))
np.testing.assert_equal(float("inf"), float(bfloat16(1) / bfloat16(0)))
np.testing.assert_equal(-1, float(bfloat16(1) / bfloat16(-1)))
np.testing.assert_equal(-1.75, float(bfloat16(3.5) / bfloat16(-2)))
np.testing.assert_equal(
float("-inf"), float(bfloat16(float("inf")) / bfloat16(-2.25)))
np.testing.assert_equal(
float("inf"), float(bfloat16(float("-inf")) / bfloat16(-2.25)))
self.assertTrue(
math.isnan(float(bfloat16(3.5) / bfloat16(float("nan")))))
def testLess(self):
for v in FLOAT_VALUES:
for w in FLOAT_VALUES:
self.assertEqual(v < w, bfloat16(v) < bfloat16(w))
def testLessEqual(self):
for v in FLOAT_VALUES:
for w in FLOAT_VALUES:
self.assertEqual(v <= w, bfloat16(v) <= bfloat16(w))
def testGreater(self):
for v in FLOAT_VALUES:
for w in FLOAT_VALUES:
self.assertEqual(v > w, bfloat16(v) > bfloat16(w))
def testGreaterEqual(self):
for v in FLOAT_VALUES:
for w in FLOAT_VALUES:
self.assertEqual(v >= w, bfloat16(v) >= bfloat16(w))
def testEqual(self):
for v in FLOAT_VALUES:
for w in FLOAT_VALUES:
self.assertEqual(v == w, bfloat16(v) == bfloat16(w))
def testNotEqual(self):
for v in FLOAT_VALUES:
for w in FLOAT_VALUES:
self.assertEqual(v != w, bfloat16(v) != bfloat16(w))
def testNan(self):
a = np.isnan(bfloat16(float("nan")))
self.assertTrue(a)
numpy_assert_allclose(np.array([1.0, a]), np.array([1.0, a]))
a = np.array(
[bfloat16(1.34375),
bfloat16(1.4375),
bfloat16(float("nan"))],
dtype=bfloat16)
b = np.array(
[bfloat16(1.3359375),
bfloat16(1.4375),
bfloat16(float("nan"))],
dtype=bfloat16)
numpy_assert_allclose(a,
b,
rtol=0.1,
atol=0.1,
equal_nan=True,
err_msg="",
verbose=True)
def testSort(self):
values_to_sort = np.float32(FLOAT_VALUES)
sorted_f32 = np.sort(values_to_sort)
sorted_bf16 = np.sort(values_to_sort.astype(bfloat16)) # pylint: disable=too-many-function-args
np.testing.assert_equal(sorted_f32, np.float32(sorted_bf16))
def testArgmax(self):
values_to_sort = np.float32(bfloat16(np.float32(FLOAT_VALUES)))
argmax_f32 = np.argmax(values_to_sort)
argmax_bf16 = np.argmax(values_to_sort.astype(bfloat16)) # pylint: disable=too-many-function-args
np.testing.assert_equal(argmax_f32, argmax_bf16)
def testArgmaxOnNan(self):
"""Ensures we return the right thing for multiple NaNs."""
one_with_nans = np.array(
[1.0, float("nan"), float("nan")], dtype=np.float32)
np.testing.assert_equal(np.argmax(one_with_nans.astype(bfloat16)),
np.argmax(one_with_nans))
def testArgmaxOnNegativeInfinity(self):
"""Ensures we return the right thing for negative infinities."""
inf = np.array([float("-inf")], dtype=np.float32)
np.testing.assert_equal(np.argmax(inf.astype(bfloat16)),
np.argmax(inf))
def testArgmin(self):
values_to_sort = np.float32(bfloat16(np.float32(FLOAT_VALUES)))
argmin_f32 = np.argmin(values_to_sort)
argmin_bf16 = np.argmin(values_to_sort.astype(bfloat16)) # pylint: disable=too-many-function-args
np.testing.assert_equal(argmin_f32, argmin_bf16)
def testArgminOnNan(self):
"""Ensures we return the right thing for multiple NaNs."""
one_with_nans = np.array(
[1.0, float("nan"), float("nan")], dtype=np.float32)
np.testing.assert_equal(np.argmin(one_with_nans.astype(bfloat16)),
np.argmin(one_with_nans))
def testArgminOnPositiveInfinity(self):
"""Ensures we return the right thing for positive infinities."""
inf = np.array([float("inf")], dtype=np.float32)
np.testing.assert_equal(np.argmin(inf.astype(bfloat16)),
np.argmin(inf))
def testDtypeFromString(self):
assert np.dtype("bfloat16") == np.dtype(bfloat16)
def interpolate(self):
i0_copy = np.copy(self.i0)
iflu_copy = np.copy(self.iflu)
ir_copy = np.copy(self.ir)
energy_copy = np.copy(self.energy)
i0_interp = []
iflu_interp = []
ir_interp = []
energy_interp = []
trigger_interp = []
timestamps = []
for i in range(len(self.trigger) - 1):
condition1 = (self.trigger[i, 0] <= i0_copy[:, 0]) == (
self.trigger[i + 1, 0] > i0_copy[:, 0])
interval1 = np.extract(condition1, i0_copy[:, 0])
condition2 = (self.trigger[i, 0] <= iflu_copy[:, 0]) == (
self.trigger[i + 1, 0] > iflu_copy[:, 0])
interval2 = np.extract(condition2, iflu_copy[:, 0])
condition3 = (self.trigger[i, 0] <= energy_copy[:, 0]) == (
self.trigger[i + 1, 0] > energy_copy[:, 0])
interval3 = np.extract(condition3, energy_copy[:, 0])
condition4 = (self.trigger[i, 0] <= ir_copy[:, 0]) == (
self.trigger[i + 1, 0] > ir_copy[:, 0])
interval4 = np.extract(condition4, ir_copy[:, 0])
if len(interval1) and len(interval2) and len(interval3):
interval_mean_i0 = np.mean(
np.extract(condition1, i0_copy[:, 1]))
i0_interp.append(interval_mean_i0)
i0_pos_high = np.where(i0_copy[:,
0] == interval1[len(interval1) -
1])[0][0]
i0_copy = i0_copy[i0_pos_high + 1:len(i0_copy)]
interval_mean_iflu = np.mean(
np.extract(condition2, iflu_copy[:, 1]))
iflu_interp.append(interval_mean_iflu)
iflu_pos_high = np.where(
iflu_copy[:, 0] == interval2[len(interval2) - 1])[0][0]
iflu_copy = iflu_copy[iflu_pos_high + 1:len(iflu_copy)]
interval_mean_energy = np.mean(
np.extract(condition3, energy_copy[:, 1]))
energy_interp.append(interval_mean_energy)
energy_pos_high = np.where(
energy_copy[:, 0] == interval3[len(interval3) - 1])[0][0]
energy_copy = energy_copy[energy_pos_high + 1:len(energy_copy)]
interval_mean_ir = np.mean(
np.extract(condition4, ir_copy[:, 1]))
ir_interp.append(interval_mean_ir)
ir_pos_high = np.where(ir_copy[:,
0] == interval4[len(interval4) -
1])[0][0]
ir_copy = ir_copy[ir_pos_high + 1:len(ir_copy)]
timestamps.append(
(self.trigger[i, 0] + self.trigger[i + 1, 0]) / 2)
trigger_interp.append(self.trigger[i, 1])
self.i0_interp = np.array([timestamps, i0_interp]).transpose()
self.iflu_interp = np.array([timestamps, iflu_interp]).transpose()
self.ir_interp = np.array([timestamps, ir_interp]).transpose()
#self.it_interp = np.copy(self.iflu_interp)
self.energy_interp = np.array([timestamps, energy_interp]).transpose()
self.trigger_interp = np.array([timestamps,
trigger_interp]).transpose()
def compute_transport(timeavg, mesh, mask, name='Drake Passage',output='transport.nc'):
mesh = xr.open_dataset(mesh)
mask = get_mask_short_names(xr.open_dataset(mask))
if name.lower() == 'all':
transectList = mask.shortNames[:].values
condition = transectList != "Atlantic Transec"
transectList = np.extract(condition, transectList)
else:
transectList = name.split(',')
if platform.release()[0] == '3':
for i in range(len(transectList)):
transectList[i] = "b'" + transectList[i]
print('Computing Transport for the following transects ',transectList)
nTransects = len(transectList)
maxEdges = mask.dims['maxEdgesInTransect']
# create empty t list for time
t = []
# Compute refLayerThickness to avoid need for hist file
refBottom = mesh.refBottomDepth.values
nz = mesh.dims['nVertLevels']
h = np.zeros(nz)
h[0] = refBottom[0]
for i in range(1,nz):
h[i] = refBottom[i] - refBottom[i-1]
# Get a list of edges and total edges in each transect
nEdgesInTransect = np.zeros(nTransects)
edgeVals = np.zeros((nTransects,maxEdges))
for i in range(nTransects):
amask = mask.sel(shortNames=transectList[i]).squeeze()
transectEdges = amask.transectEdgeGlobalIDs.values
inds = np.where(transectEdges > 0)[0]
nEdgesInTransect[i] = len(inds)
transectEdges = transectEdges[inds]
edgeVals[i,:len(inds)] = np.asarray(transectEdges-1, dtype='i')
nEdgesInTransect = np.asarray(nEdgesInTransect, dtype='i')
# Create a list with the start and stop for transect bounds
nTransectStartStop = np.zeros(nTransects+1)
for j in range(1,nTransects+1):
nTransectStartStop[j] = nTransectStartStop[j-1] + nEdgesInTransect[j-1]
edgesToRead = edgeVals[0,:nEdgesInTransect[0]]
for i in range(1,nTransects):
edgesToRead = np.hstack([edgesToRead,edgeVals[i,:nEdgesInTransect[i]]])
edgesToRead = np.asarray(edgesToRead, dtype='i')
dvEdge = mesh.dvEdge.sel(nEdges=edgesToRead).values
edgeSigns = np.zeros((nTransects,len(edgesToRead)))
for i in range(nTransects):
edgeSigns[i,:] = mask.sel(nEdges=edgesToRead, shortNames=transectList[i]).squeeze().transectEdgeMaskSigns.values
# Read time average files one at a time and slice
fileList = sorted(glob.glob(timeavg))
transport = np.zeros((len(fileList),nTransects))
t = np.zeros(len(fileList))
for i,fname in enumerate(fileList):
ncid = Dataset(fname,'r')
if 'timeMonthly_avg_normalTransportVelocity' in ncid.variables.keys():
vel = ncid.variables['timeMonthly_avg_normalTransportVelocity'][0,edgesToRead,:]
elif 'timeMonthly_avg_normalVelocity' in ncid.variables.keys():
vel = ncid.variables['timeMonthly_avg_normalVelocity'][0,edgesToRead,:]
if 'timeMonthly_avg_normalGMBolusVelocity' in ncid.variables.keys():
vel += ncid.variables['timeMonthly_avg_normalGMBolusVelocity'][0,edgesToRead,:]
else:
raise KeyError('no appropriate normalVelocity variable found')
t[i] = ncid.variables['timeMonthly_avg_daysSinceStartOfSim'][:] / 365.
ncid.close()
# Compute transport for each transect
for j in range(nTransects):
start = int(nTransectStartStop[j])
stop = int(nTransectStartStop[j+1])
transport[i,j] = (dvEdge[start:stop,np.newaxis]*h[np.newaxis,:]*vel[start:stop,:] \
*edgeSigns[j,start:stop,np.newaxis]).sum()*m3ps_to_Sv
# Define some dictionaries for transect plotting
obsDict = {'Drake Passage':[120,175],'Tasmania-Ant':[147,167],'Africa-Ant':None,'Antilles Inflow':[-23.1,-13.7], \
'Mona Passage':[-3.8,-1.4],'Windward Passage':[-7.2,-6.8],'Florida-Cuba':[30,33],'Florida-Bahamas':[30,33], \
'Indonesian Throughflow':[-21,-11],'Agulhas':[-90,-50],'Mozambique Channel':[-20,-8], \
'Bering Strait':[0.17,1.49],'Lancaster Sound':None,'Fram Strait':None,'Robeson Channel':None,'Nares Strait':None}
labelDict = {'Drake Passage':'drake','Tasmania-Ant':'tasmania','Africa-Ant':'africaAnt','Antilles Inflow':'Antilles', \
'Mona Passage':'monaPassage','Windward Passage':'windwardPassage','Florida-Cuba':'floridaCuba',\
'Florida-Bahamas':'floridaBahamas', \
'Indonesian Throughflow':'indonesia','Agulhas':'agulhas','Mozambique Channel':'mozambique', \
'Bering Strait':'beringstrait','Lancaster Sound':'lancaster','Fram Strait':'fram','Robeson Channel':'robeson','Nares Strait':'nares'}
for i in range(nTransects):
plt.figure()
if platform.release()[0]=='3':
searchString = transectList[i][2:]
else:
searchString = transectList[i]
bounds = obsDict[searchString]
title = 'Transport for '+searchString
plt.plot(t,transport[:,i],'k',linewidth=2)
if bounds is not None:
plt.gca().fill_between(t, bounds[0]*np.ones_like(t), bounds[1]*np.ones_like(t), alpha=0.3, label='observations')
plt.ylabel('Transport (Sv)',fontsize=32)
plt.xlabel('Time (Years)',fontsize=32)
plt.title(title,fontsize=32)
plt.savefig('transport_'+labelDict[searchString]+'.png')
# Add calls to save transport and then can build up
ncid=Dataset(output,mode='w',clobber=True, format='NETCDF3_CLASSIC')
ncid.createDimension('Time',None)
ncid.createDimension('nTransects',nTransects)
ncid.createDimension('StrLen',64)
transectNames=ncid.createVariable('TransectNames','c',('nTransects','StrLen'))
times=ncid.createVariable('Time','f8','Time')
transportOut=ncid.createVariable('Transport','f8',('Time','nTransects'))
times[:] = t
transportOut[:,:] = transport
for i in range(nTransects):
nLetters = len(transectList[i])
transectNames[i,:nLetters] = transectList[i]
ncid.close()
def show_recall(y, y_hat):
# print y_hat[y == 1]
print(
'召回率：%.2f%%' %
(100 * float(np.sum(y_hat[y == 1] == 1)) / np.extract(y == 1, y).size))
def rotate_IQ_plane(self, entries=None):
'''
The data optimizer is fed with microwave data of both quadratures, typically amplitude and phase.
It effectively performs a principle axis transformation in the complex plane of all data points
that are located along a line for a typical projective qubit state measurement without single-shot
fidelity and using a ADC acquisition card. Successively, data optimizer returns a projection along
the in-line direction quadrature containing maximum information. Indeally, no information is lost
as no data is left in the orthogonal quadrature.
The algorithm locates one of the edge data points on either end of the line-shaped data point distribution
by summing and maximizing the mutual distances between data points. It then calculates the distance
of each data point with respect to this distinct extremal data point.
We assume that the possible outcomes of a strong and projective quantum measurement, i.e. the qubit
states |0>, |1>, result in two distinct locations in the complex plane, corresponding to signatures
of the dispersive readout resonator. In the absence of single-shot readout fidelity, the pre-averaging
that is perfomed by the ADC card in the complex plane leads to counts in between the piles denoting |0>
and |1>. The points laong this line are distributed due to statistical measurement noise and ultimately
due to the quantum uncertainty. As the averaging takes place in the complex plane, all points recorded
in such a measurement should be located on a line. In case of a phase sensitive reflection measurement
at a high internal quality cavity, points may span a segment of a circle that can be caused by noise
caused measurements of the resonator position away from the two quantumn positions.
The line shape of the data distribution generally justifies the projection of measurement data to an
arbitrary axis without altering the internal data shape.
Errors are calculated projected on the axis complex data points are aligned along. Data is normalized
prior to returning.
Requires coordinate, amplitude, and phase data to be specified via entries keyword.
self.amplitude and self.phase can be lists of vectors or a single (averaged) data set.
'''
if self.file_name[-2:] == 'h5':
self.discover_hdf_data(entries=entries)
if len(self.urls) != 3:
logging.error('Invalid entry specification. Aborting.')
return
self.coordinate, self.amplitude, self.phase = self.read_hdf_data(
return_data=True)[0]
else:
logging.warning('Reading out amplitude and phase attributes.')
#generate complex data array
self.c_raw = np.array(self.amplitude) * np.exp(
1j * np.array(self.phase))
if len(self.c_raw.shape) > 1:
#calculate mean of complex data
c = np.mean(self.c_raw, axis=0)
else: #given data is already averaged
c = self.c_raw
#point in complex plane with maximum sumed mutual distances
s = np.zeros_like(np.abs(c))
for i in range(len(c)):
for p in c:
s[i] += np.abs(p - c[i])
cmax = np.extract(s == np.max(s), c)
#calculate distances
data_opt = np.abs(c - cmax)
if len(self.c_raw.shape) > 1: #calculate errors in line direction
#find maximum complex point in maximum distance
d = 0
for p in c:
if np.abs(p - cmax) > d:
d = np.abs(p - cmax)
cdist = p
#find unit vector in line direction
vunit = (cdist - cmax) / np.abs(cdist - cmax)
#calculate projected distances via dot product, projecting along the data direction
#errors via std
dist_proj = [0] * len(c)
errs = np.zeros_like(c)
for i, ci in enumerate(self.c_raw.T): #for each iteration
dist_proj[i] = [
np.abs(
np.vdot(
[np.real(vunit), np.imag(vunit)], [
np.real(cr) - np.real(c[i]),
np.imag(cr) - np.imag(c[i])
])) for cr in ci
]
errs[i] = np.std(dist_proj[i]) / np.sqrt(len(dist_proj[i]))
#normalize optimized data
data_opt -= np.min(data_opt)
maxv = np.max(data_opt)
data_opt /= maxv
if len(self.c_raw.shape) > 1:
errs /= maxv
#gauss plane plot
if self.cfg['show_complex_plot'] and self.cfg['matplotlib']:
if len(self.c_raw.shape) > 1:
plt.figure(figsize=(10, 13))
ax1 = plt.subplot2grid((4, 1), (0, 0))
ax2 = plt.subplot2grid((4, 1), (1, 0), rowspan=3)
ax1.errorbar(data_opt,
np.zeros_like(data_opt),
xerr=np.real(errs),
color='blue',
fmt='o',
elinewidth=0.8,
capsize=5,
markersize=8,
ecolor='red')
ax1.plot([0], [0], '*', color='red', markersize=20)
prange = np.max(data_opt) - np.min(data_opt)
ax1.set_xlim(
np.min(data_opt) - 0.05 * prange,
np.max(data_opt) + 0.05 * prange)
ax2.plot(np.real(c), np.imag(c), '.')
ax2.plot(np.real(c)[:10], np.imag(c)[:10], '.',
color='r') #show first 10 data points in red
ax2.plot(np.real(cmax),
np.imag(cmax),
'*',
color='black',
markersize=15)
self.errors = np.real(np.array(errs))
else:
plt.figure(figsize=(10, 10))
plt.plot(np.real(c), np.imag(c), '.')
plt.plot(np.real(c)[:10], np.imag(c)[:10], '.',
color='r') #show first 10 data points in red
plt.plot(np.real(cmax),
np.imag(cmax),
'*',
color='black',
markersize=15)
self.errors = None
self.data = np.real(np.array(data_opt))
self._save_opt_data()
def BTEblkEstimate(blkIm, airlight, lamb, fTrans):
global ld
le = []
Trans = 0.0
nTrans = np.floor(1.0 / fTrans * 128)
fMinCost = sys.maxsize
# or 9223372036854775807
numberOfPixels = blkIm.shape[0] * blkIm.shape[1] * blkIm.shape[2]
nCounter = 0
bgr = cv2.split(blkIm)
while (nCounter < (1 - fTrans) * 10):
color = copy.deepcopy(bgr[0])
bChannel = BTEpreDehaze(color, airlight[0], nTrans)
#print("bChannel "+str(bChannel[0,0]))
if ld:
ld = False
le = bChannel
guess = False
color = copy.deepcopy(bgr[1])
gChannel = BTEpreDehaze(color, airlight[1], nTrans)
color = copy.deepcopy(bgr[2])
rChannel = BTEpreDehaze(color, airlight[2], nTrans)
nSumOfLoss = 0
condition = bChannel > 255
condExtract = np.array(np.extract(condition, bChannel))
condExtract = np.sum((condExtract - 255)**2)
nSumOfLoss += condExtract
condition = bChannel < 0
condExtract = np.array(np.extract(condition, bChannel))
nSumOfLoss += np.sum(condExtract**2)
condition = gChannel > 255
condExtract = np.array(np.extract(condition, gChannel))
condExtract = np.sum((condExtract - 255)**2)
nSumOfLoss += condExtract
condition = gChannel < 0
condExtract = np.array(np.extract(condition, gChannel))
nSumOfLoss += np.sum(condExtract**2)
condition = rChannel > 255
condExtract = np.array(np.extract(condition, rChannel))
condExtract = np.sum((condExtract - 255)**2)
condition = rChannel < 0
condExtract = np.array(np.extract(condition, rChannel))
nSumOfLoss += np.sum(condExtract**2)
nSumOfSquareOuts = np.sum(np.multiply(bChannel, bChannel)) + np.sum(
np.multiply(gChannel, gChannel)) + np.sum(
np.multiply(rChannel, rChannel))
nSumOfOuts = np.sum(bChannel) + np.sum(gChannel) + np.sum(rChannel)
fMean = nSumOfOuts / numberOfPixels
fCost = lamb * nSumOfLoss / numberOfPixels - \
(nSumOfSquareOuts/numberOfPixels - fMean**2)
if nCounter == 0 or fMinCost > fCost:
fMinCost = fCost
Trans = fTrans
fTrans = fTrans + 0.1
nTrans = 1.0 / fTrans * 128.0
nCounter = nCounter + 1
return Trans
import numpy as np
import pandas as pd
from scipy import optimize
import mini1.ddm_sampler as ddm
data_file = './data/dots_psychophysics.txt'
tdata = pd.read_csv(data_file, delimiter=' ', skipinitialspace=True, index_col=None, header=None, names=['coherence', 'direction', 'choice', 'rewarded', 'rt'])
tdata = tdata.values
scalar_rt = tdata[:,4]/10
tdata[:,4] = scalar_rt.astype(np.int)
coherence_values = np.unique(tdata[:,0])
plt.hist(np.extract(tdata[:,0]==0.724, tdata[:, 4]), bins=100);plt.show()
#res = optimize.minimize(ddm.ddm_pdf, np.array([0.0015, 0.03]), args=(tdata[:,4].astype(np.int),))
#print(res.x)
#print(res.success)
#print(res.message)
res = optimize.minimize(ddm.ddm_pdf, np.array([0.0015, 0.03]), args=(fake_data), method = 'Nelder-Mead', tol=0.05, options={'maxiter':200}); print(res)
# final_simplex: (array([[0.00177656, 0.05034375],
# [0.00168281, 0.05371875],
# [0.00159844, 0.04715625]]), array([5.27673636, 5.28378169, 5.29123604]))
# fun: 5.276736355372437
# message: 'Optimization terminated successfully.'
# nfev: 15
# nit: 7
# status: 0
# success: True
def get_indices(valid_mask: np.ndarray) -> np.ndarray:
return np.extract(valid_mask, np.array(range(valid_mask.size)))
def findOptRoute(possible_locations, D, start, elevation_change_indicator):
possible_locations, distances_to_home = removeDistantSites(
possible_locations, start, D)
if len(possible_locations) > 24:
# distances_to_start_direct=getMatrixOneDirect(possible_locations,start)
# direct_to_walk_frac=np.min(np.true_divide(distances_to_start_direct,distances_to_home[1:]))
# print direct_to_walk_frac
# print np.mean(np.true_divide(distances_to_start_direct,distances_to_home[1:]))
matrix = getMatrixDirect(np.concatenate((start, possible_locations)))
else:
matrix = getMatrixMQ(np.concatenate((start, possible_locations)))
numbers_seen, paths, path_distances = getAllRoutes(matrix, D)
# Pick best route and rank the rest
try:
opt_routes_ind = np.argwhere(
numbers_seen == np.amax(numbers_seen)).flatten().tolist()
except:
a = 0
return a, a, a, a, a, a, a, a, a, a, a, a, a
opt_route_max_dist_ind = opt_routes_ind[np.argmax(
np.array(path_distances)[opt_routes_ind])]
opt_path = paths[opt_route_max_dist_ind]
paths = np.extract(numbers_seen > 0, paths)
path_distances = np.extract(numbers_seen > 0, path_distances)
numbers_seen = np.extract(numbers_seen > 0, numbers_seen)
paths_sorted_ind = np.argsort(numbers_seen)[::-1]
numbers_seen_sorted = numbers_seen[paths_sorted_ind]
distances_sorted = path_distances[paths_sorted_ind]
paths_sorted = paths[paths_sorted_ind]
percent_diff_distance = np.true_divide(np.abs(distances_sorted - D), D)
#Adjust path ranking if flat elevation desired
if elevation_change_indicator == 1:
#Don't want to add a point too far because will change elevation when adding a point
numbers_seen_sorted = np.extract(percent_diff_distance < 0.15,
numbers_seen_sorted)
paths_sorted = np.extract(percent_diff_distance < 0.15, paths_sorted)
distances_sorted = np.extract(percent_diff_distance < 0.15,
distances_sorted)
elevation_rank, mean_elevation_change, max_elevation_change = rankElevations(
possible_locations, paths_sorted, start)
paths_sorted = paths_sorted[elevation_rank]
opt_path = paths_sorted[0]
# Get the locations along the route
opt_route_locations = np.array(start)
for j in opt_path:
x = np.reshape(
possible_locations[j - 1],
(1,
2)) #Because j was calculated from array with starting point at 0
opt_route_locations = np.concatenate((opt_route_locations, x))
opt_route_locations = np.concatenate((opt_route_locations, start))
direction_coordinates, narratives, distance = getFinalRoute(
opt_route_locations)
if distance < 0.9 * D:
opt_route_locations, direction_coordinates, narratives, distance, added_indicator = addPoint(
opt_route_locations, D, distance)
else:
added_indicator = 0
elevations, distances_el = getElevationMQ(direction_coordinates[0])
mean_elevation_change = np.mean(
np.true_divide(np.abs(np.diff(elevations)), distances_el[1:]))
max_elevation_change = np.max(
np.true_divide(np.abs(np.diff(elevations)), distances_el[1:]))
return opt_route_locations, distance, direction_coordinates, narratives, added_indicator, paths_sorted, distances_sorted, numbers_seen_sorted, possible_locations, mean_elevation_change, max_elevation_change, elevations, distances_el
def main(lata, longa, latb, longb):
lata = conv.convert_gps(float(lata))
longa = conv.convert_gps(float(longa))
latb = conv.convert_gps(float(latb))
longb = conv.convert_gps(float(longb))
try:
from pylibfreenect2 import OpenGLPacketPipeline
pipeline = OpenGLPacketPipeline()
except:
try:
from pylibfreenect2 import OpenCLPacketPipeline
pipeline = OpenCLPacketPipeline()
except:
from pylibfreenect2 import CpuPacketPipeline
pipeline = CpuPacketPipeline()
print("Packet pipeline:", type(pipeline).__name__)
#create list for gps location of obstacles
###############################################################
# Create and set logger
logger = createConsoleLogger(LoggerLevel.Debug)
setGlobalLogger(logger)
fn = Freenect2()
num_devices = fn.enumerateDevices()
if num_devices == 0:
print("No device connected!")
sys.exit(1)
serial = fn.getDeviceSerialNumber(0)
device = fn.openDevice(serial, pipeline=pipeline)
listener = SyncMultiFrameListener(FrameType.Color | FrameType.Ir
| FrameType.Depth)
# Register listeners
device.setColorFrameListener(listener)
device.setIrAndDepthFrameListener(listener)
device.start()
# NOTE: must be called after device.start()
registration = Registration(device.getIrCameraParams(),
device.getColorCameraParams())
undistorted = Frame(512, 424, 4)
registered = Frame(512, 424, 4)
# Optinal parameters for registration
# set True if you need
need_bigdepth = False
need_color_depth_map = False
bigdepth = Frame(1920, 1082, 4) if need_bigdepth else None
color_depth_map = np.zeros((424, 512), np.int32).ravel() \
if need_color_depth_map else None
while True:
frames = listener.waitForNewFrame()
#time.sleep(5)
color = frames["color"]
ir = frames["ir"]
depth = frames["depth"]
registration.apply(color,
depth,
undistorted,
registered,
bigdepth=bigdepth,
color_depth_map=color_depth_map)
# NOTE for visualization:
# cv2.imshow without OpenGL backend seems to be quite slow to draw all
# things below. Try commenting out some imshow if you don't have a fast
# visualization backend.
#cv2.imshow("ir", ir.asarray() / 65535.)
#time.sleep(2)
cv2.imshow("depth", depth.asarray() / 4500.)
#Comment the coming lines
print('center value is:', depth.asarray(np.float32).item((212, 256)))
print('down1 value is:', depth.asarray(np.float32).item((270, 256)))
'''print('up1 value is:',depth.asarray(np.float32).item((150,256)))
print('up2 value is:',depth.asarray(np.float32).item((100,256)))
print('down2 value is:',depth.asarray(np.float32).item((350,256)))
print('right1 value is:',depth.asarray(np.float32).item((212,300)))
print('right2 value is:',depth.asarray(np.float32).item((212,350)))
print('left1 value is:',depth.asarray(np.float32).item((212,200)))
print('left2 value is:',depth.asarray(np.float32).item((212,150)))'''
x = depth.asarray(np.float32)
y = np.logical_and(np.greater(x, 1200), np.less(x, 1500))
#print(np.extract(y, x))
no_of_pixels = np.count_nonzero(np.extract(y, x))
print(no_of_pixels)
#get gps coordinate at this location########
'''
assumingthegps received is lat,long variables:
obs.append([lat,long])
this is to be retrieved in the path planning code.
'''
if no_of_pixels > 14000: #approx distance from the camera = 1.5 m
print('big Obstacle found, stop!!!')
obs.append(findPosition())
elif no_of_pixels > 8000: #approx distance from the camera = 1.5 m
print('small Obstacle found!!')
obs.append(findPosition())
#cv2.imshow("color", cv2.resize(color.asarray(),(int(1920 / 3), int(1080 / 3))))
#cv2.imshow("registered", registered.asarray(np.uint8))
if need_bigdepth:
cv2.imshow(
"bigdepth",
cv2.resize(bigdepth.asarray(np.float32),
(int(1920 / 3), int(1082 / 3))))
if need_color_depth_map:
cv2.imshow("color_depth_map", color_depth_map.reshape(424, 512))
listener.release(frames)
#time.sleep(20)
key = cv2.waitKey(delay=1) & 0xFF
if key == ord('q'):
break
#######################################################
target(latb, longb, lata, longa)
#stage3.py main fnc
#######################################################
device.stop()
device.close()
def segment(self, frame):
blur = cv2.medianBlur(frame, 51)
# Get red channel
r = blur.copy()
r[:, :, 0] = 0
r[:, :, 1] = 0
# Convert to greyscale
grey = self.bgr_2_grey(r)
# Down sample 1/2 x 1/2
scale = 2
w = int(grey.shape[1] / scale)
h = int(grey.shape[0] / scale)
half = cv2.pyrDown(grey, dstsize=(w, h))
img = half.copy()
# Largest grey value
L = img.max()
# mean grey value
muT = np.mean(img)
# Number of pixels
N = img.size
Gv = cv2.Sobel(img, cv2.CV_32F, 1, 0, ksize=5)
Gh = cv2.Sobel(img, cv2.CV_32F, 0, 1, ksize=5)
g = np.sqrt((Gh**2) + (Gv**2))
gmean = np.mean(g)
if gmean == 0:
return frame
h = np.zeros(L)
h_prime = np.zeros(L)
w = np.zeros(L)
sigma_T = 0
maxKt = 0
max_eta = 0
optimal_thresh = 0
final = img.copy()
for i in xrange(0, L):
# Number of pixels at threshold i
h[i] = len(np.extract(img == i, img))
if h[i] == 0:
h[i] == 1
w[i] = (L / 1) * (1 / gmean)
h_prime[i] = w[i] * h[i]
sigma_T = sigma_T + ((i - muT)**2) * (h_prime[i] / N)
for t in xrange(0, L):
lambda_t = (-t + L) * (1 / gmean)
# Classes
C0 = np.extract(img <= t, img)
C1 = np.extract(img > t, img)
if C0.any() and C1.any():
# mean of classes
mu_0 = np.mean(C0)
mu_1 = np.mean(C1)
omega_0 = float(len(C0)) / float(N)
omega_1 = float(len(C1)) / float(N)
sigma_B = (omega_0 * omega_1) * ((mu_1 - mu_0)**2)
eta_t = sigma_B / sigma_T
kt = lambda_t * eta_t
if kt > maxKt:
maxKt = kt
optimal_thresh = t
# Threshold using calculated threshold value (otimal threshold)
temp_img = img.copy()
w = int(temp_img.shape[1] * scale)
h = int(temp_img.shape[0] * scale)
# Scale up to original size
temp_img = cv2.pyrUp(temp_img, dstsize=(w, h))
ret, mask = cv2.threshold(temp_img, optimal_thresh, 255,
cv2.THRESH_BINARY)
# Invert mask to the region we want (the lumen)
mask_inv = cv2.bitwise_not(mask)
# Get masked images
# colour_masked = cv2.bitwise_and(crop,crop,mask = mask_inv)
grey_masked = cv2.bitwise_and(temp_img, temp_img, mask=mask_inv)
# Find contours in masked image and get largest region
contours = cv2.findContours(grey_masked.copy(), cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)[-2]
mask = np.ones(frame.shape[:2], dtype="uint8") * 255
c = max(contours, key=cv2.contourArea)
# Mask to leave only the largest thresholded region
mask_cnt = cv2.drawContours(mask, [c], -1, 0, -1)
mask_cnt_inv = cv2.bitwise_not(mask_cnt)
LRC = cv2.bitwise_and(frame, frame, mask=mask_cnt_inv)
## Zabulis et al.
smax = 0
for i in contours:
A = cv2.contourArea(i)
if A != 0 and cv2.arcLength(i, True) != 0:
C = A / cv2.arcLength(i, True)
mask = np.ones(grey.shape[:2], dtype="uint8") * 255
mask_cnt = cv2.drawContours(mask, [i], -1, 0, -1)
mask_inv = cv2.bitwise_not(mask_cnt)
colour_masked = cv2.bitwise_and(frame, frame, mask=mask_inv)
mI = cv2.mean(grey, mask=mask_inv)
I = 1 + (1 - mI[0])
S = (I**2) * C * A
if S > smax:
smax = S
LRC = colour_masked.copy()
LRC[mask == 255] = (255, 255, 255)
return LRC
img1 = io.imread("Supervised/food_pics/food_pics/" + folder + "/" +
files[0])
img2 = io.imread("Supervised/food_pics/food_pics/" + folder + "/" +
files[1])
grey1, grey2 = color.rgb2grey(img1), color.rgb2grey(img2)
cond1, cond2 = grey1 != 0, grey2 != 0
img1_src = io.imread("Supervised/food_pics/food_pics_source/" + folder +
"/" + files_src[0])
img2_src = io.imread("Supervised/food_pics/food_pics_source/" + folder +
"/" + files_src[1])
if img1.shape != img1_src.shape:
img_copy = img2_src.copy()
img2_src = img1_src
img1_src = img_copy
print(folder)
img1_r1, img1_r2 = np.extract(cond1, img1_src), np.extract(
np.invert(cond1), img1_src)
img2_r1, img2_r2 = np.extract(cond2, img2_src), np.extract(
np.invert(cond2), img2_src)
greyimg1, greyimg2 = color.rgb2grey(img1_src), color.rgb2grey(img2_src)
greyimg1_r1, greyimg1_r2 = np.extract(cond1, greyimg1), np.extract(
np.invert(cond1), greyimg1)
greyimg2_r1, greyimg2_r2 = np.extract(cond2, greyimg2), np.extract(
np.invert(cond2), greyimg2)
hsvimg1, hsvimg2 = color.rgb2hsv(img1_src), color.rgb2hsv(img2_src)
hsvimg1_r1, hsvimg1_r2 = np.extract(cond1, hsvimg1), np.extract(
np.invert(cond1), hsvimg1)
hsvimg2_r1, hsvimg2_r2 = np.extract(cond2, hsvimg2), np.extract(
np.invert(cond2), hsvimg2)
traindata[i, :3], traindata[-i - 1, :3] = np.mean(
def categorical_crossentropy(Y_hat, Y):
return np.mean(-np.log(np.extract(Y, Y_hat)))
