def ellipse(width, height):
horpow = 2
verpow = horpow
# to produce a Ellipse with horizontal axis == ceil(2*hor semi axis)
width = width + 0.5
#to produce a Ellipse with vertical axis == ceil(2*vert semi axis)
height = height + 0.5
[x, y] = np.meshgrid(np.arange(-width, width + 1),
np.arange(-height, height + 1))
#print width, height, x.shape, y.shape
bll = (abs(x / width) ** horpow + abs(y / height) ** verpow) < 1
xs = plt.find(bll.sum(0) == 0)
ys = plt.find(bll.sum(1) == 0)
bll = bll[ys[0] + 1:ys[1], :]
bll = bll[:, xs[0] + 1:xs[1]]
bll = bll.T
mask_inds = np.where(bll > 0)
mask_inds = np.array(mask_inds).T
inv_mask_inds = np.where(bll == 0)
inv_mask_inds = np.array(inv_mask_inds).T
box_inds = np.array(np.where(bll != 5555)).T
box_edges = np.array(
[[box_inds[:, 0].min(), box_inds[:, 1].min()],
[box_inds[:, 0].max(), box_inds[:, 1].max()]])
return mask_inds, inv_mask_inds, box_edges
def vl_test_hikmeans():
""" VL_TEST_HIKMEANS Test VL_HIKMEANS function
"""
K = 3
nleaves = 100
data = numpy.array(numpy.random.rand(2, 1000) * 255, 'uint8')
datat = numpy.array(numpy.random.rand(2, 10000) * 255, 'uint8')
[tree, A] = vlfeat.vl_hikmeans(data, K, nleaves, verb=1)
AT = vlfeat.vl_hikmeanspush(tree, datat)
pylab.figure()
plottree(tree)
pylab.xlim(0, 255)
pylab.ylim(0, 255)
print('hikmeans-tree')
pylab.figure()
gen = color_gen()
for k in range(K * K):
color = next(gen)
sel = pylab.find(A[-1, :] == k)
pylab.plot(data[0, sel], data[1, sel], '.', color=color)
sel = pylab.find(AT[-1, :] == k)
pylab.plot(datat[0, sel], datat[1, sel], '+', color=color)
plottree(tree, linewidth=4)
pylab.xlim(0, 255)
pylab.ylim(0, 255)
print('hikmeans-clusters')
def lognormal_cdf(x, mu, sigma):
if sigma > 0:
small = 0.5 + 0.5 * special.erf(
(np.log(1e-6) - mu) / (np.sqrt(2.0) * sigma))
if x.__class__ == np.ndarray:
lp = np.zeros(len(x))
I = pp.find(x > 1e-6)
J = pp.find(x <= 1e-6)
lp[I] = 0.5 + 0.5 * special.erf(
(np.log(x) - mu) / (np.sqrt(2.0) * sigma))
lp[J] = small
return lp
else:
if x > 1e-6:
return 0.5 + 0.5 * special.erf(
(np.log(x) - mu) / (np.sqrt(2.0) * sigma))
else:
return small
else:
if x.__class__ == np.ndarray:
logx = np.log(x + 1e-6)
lp = 0.5 * np.ones(len(x))
I1 = pp.find(logx < mu)
I2 = pp.find(logx > mu)
lp[I1] = 0
lp[I2] = 1
return lp
else:
if np.log(x) < mu:
return 0
elif np.log(x) > mu:
return 1
else:
return 0.5
def logpdf(x, mu, sigma):
if type(x) == np.ndarray:
if sigma > 0:
small = np.log(0.5 + 0.5 * special.erf((np.log(1e-6) - mu) /
(np.sqrt(2.0) * sigma)))
I = pp.find(x > 1e-6)
log_x = np.log(x[I])
lp = small * np.ones(x.shape)
lp[I] = -log_x - 0.5 * np.log(2.0 * np.pi) - np.log(sigma) - \
0.5 * ((log_x - mu) ** 2) / (sigma ** 2)
else:
I = pp.find(x == mu)
lp = -np.inf * np.ones(x.shape)
lp[I] = 0
else:
if sigma > 0:
if x > 1e-6:
log_x = np.log(x)
lp = - log_x - 0.5 * np.log(2.0 * np.pi) - \
np.log(sigma) - \
0.5 * ((log_x - mu) ** 2) / (sigma ** 2)
else:
lp = np.log(0.5 + 0.5 * special.erf((np.log(1e-6) - mu) /
(np.sqrt(2.0) * sigma)))
else:
if x == mu:
lp = 0
else:
lp = -np.inf
return lp
def psd(self, bin_w = 5., nmax = 4000, time_range = [], pop_id = 'all'):
self.load_spikes()
spikes = self.events['spikes']
pop_id,spikes,neuron_nr = self.get_pop_spikes(spikes,nmax,pop_id)
if len(spikes) == 0:
print 'psd: spike array is empty'
return np.nan, np.nan, np.nan, np.nan
if time_range!=[]:
idx = (spikes[:,1]>time_range[0]) & (spikes[:,1]<=time_range[1])
spikes = spikes[idx]
if time_range==[]:
total_time = self.pars['T_total']
else:
total_time = time_range[1] - time_range[0]
if len(spikes) == 0:
print 'psd: spike array is empty'
return np.nan, np.nan, np.nan, np.nan
ids = np.unique(spikes[:,0])[:nmax]
nr_neurons = len(ids)
#psd, max_value, freq,h = misc2.psd_sp(spikes[:,1],nr_bins,nr_neurons)
bins = np.arange(time_range[0],time_range[1],bin_w)
a,b = np.histogram(spikes[:,1], bins)
ff = abs(np.fft.fft(a- np.mean(a)))**2
Fs = 1./(bin_w*0.001)
freq2 = np.fft.fftfreq(len(bins))[0:len(bins/2)+1]
freq = np.linspace(0,Fs/2,len(ff)/2+1)
px = ff[0:len(ff)/2+1]
max_px = np.max(px[1:])
idx = px == max_px
corr_freq = freq[pl.find(idx)]
new_px = px
max_pow = new_px[pl.find(idx)]
return new_px,freq, freq2, corr_freq[0], max_pow
def solve(p):
rr = p.T[0]
x = p.T[1].astype('int')
N = len(x)
i1 = find(rr == 'B')
i2 = find(rr == 'O')
assert (len(rr) == (len(i2) + len(i1)))
tt = zeros(N, dtype='int')
tt[i1] = computeTravelTimes(x[i1])
tt[i2] = computeTravelTimes(x[i2])
totalTime = tt[0] + 1
prev = rr[0]
bonus = totalTime
for r, t in zip(rr[1:], tt[1:]):
if prev == r:
#print "same robot", t
bonus += (t + 1)
totalTime += (t + 1)
else:
t = max(0, t - bonus)
#print "other", t
bonus = t + 1
totalTime += (t + 1)
prev = r
return totalTime
def computeBalancedLogisticAndExponentialCosts(x, y):
x = np.array(x, float)
y = np.array(y, float)
# If specified, limit ourselves to data for which tag == 1
idxs1 = pylab.find(y == 1)
idxs0 = pylab.find(y == 0)
n1 = len(idxs1)
n0 = len(idxs0)
logisticCost = 0
exponentialCost = 0
for idx in idxs1:
p = convertLLRtoProb(x[idx], 0)
if p * (
1 - p
) == 0: # Precision error -- return "inf" as infinite cost so that, in optimization, we don't use the parameters that got us here
return (float("inf"), float("inf"))
logisticCost -= 0.5 / n1 * math.log(p)
exponentialCost += 0.5 / n1 * abs(1 - p) / math.sqrt(p * (1 - p))
for idx in idxs0:
p = convertLLRtoProb(x[idx], 0)
if p * (
1 - p
) == 0: # Precision error -- return "inf" as infinite cost so that, in optimization, we don't use the parameters that got us here
return (float("inf"), float("inf"))
logisticCost -= 0.5 / n0 * math.log(1 - p)
exponentialCost += 0.5 / n0 * abs(0 - p) / math.sqrt(p * (1 - p))
return (logisticCost, exponentialCost)
def detect_saccades(orientations, timestep):
orientations = orientations[~numpy.isnan(orientations)]
unwrappedOrientations = numpy.unwrap(orientations, 180)
SMOOTH_TIME = 1
smoothWindow = int(round(SMOOTH_TIME / timestep))
smoothedOrientations = tools.smooth(unwrappedOrientations, smoothWindow)
angSpds = abs(numpy.diff(smoothedOrientations)) / timestep
MIN_PEAK_SPD = 30
sacInds = tools.local_minima(-angSpds) & (angSpds > MIN_PEAK_SPD)
sacSpds = angSpds[sacInds]
MIN_SAC_SPD = 5
inSac = angSpds > MIN_SAC_SPD
startInds = pylab.find(numpy.diff(inSac.view(dtype=int8)) == 1)
stopInds = pylab.find(numpy.diff(inSac.view(dtype=int8)) == -1)
if stopInds[-1] < startInds[-1]:
l = stopInds.tolist()
l.append(len(inSac) - 1)
stopInds = numpy.array(l)
if startInds[0] > stopInds[0]:
l = startInds.tolist()
l.insert(0, 0)
startInds = numpy.array(l)
angDisp = numpy.zeros(len(angSpds))
for i, startInd in enumerate(startInds):
stopInd = stopInds[i]
angDisp[startInd:stopInd] = smoothedOrientations[stopInd] - smoothedOrientations[startInd]
sacAmps = angDisp[sacInds]
return sacAmps
def extract_time_series(self, filename, node_yxz):
dis = self.parent.get_package('DIS')
if (dis != None):
yy, xx, zz = dis.get_node_coordinates()
times, heads, strings = self.read_all(filename)
if self.silent == 0: print ('Please wait. Extracting time series.')
ht = np.empty((1, len(node_yxz)))
rr = 0
for h in heads:
if (rr > 0):
ht = np.vstack((ht, np.empty(len(node_yxz))))
cc = 0
for yxz in node_yxz:
y = yxz[0]
x = yxz[1]
z = yxz[2]
r = find(abs(yy - y) < abs(y) / 1e6)[0]
c = find(abs(xx - x) < abs(x) / 1e6)[0]
l = find(abs(zz[r, c] - z) < abs(z) / 1e6)[0]
ht[rr, cc] = h[r, c, l]
cc = cc + 1
rr = rr + 1
return times, ht
def __init__(self, path):
file1 = path + 'scow_taux_climatology.nc'
file2 = path + 'scow_tauy_climatology.nc'
file3 = path + 'scow_wind_stress_curl_climatology.nc'
variables = ['taux', 'tauy', 'stress_curl']
data1 = Dataset(file1, 'r')
data2 = Dataset(file2, 'r')
data3 = Dataset(file3, 'r')
# Coordinates systems - South Atlantic Coordinates
latitude = data1.variables.pop('latitude')[:].squeeze()
longitude = data1.variables.pop('longitude')[:].squeeze()
jgood = (longitude >= (-69.5 + 360.)) | (longitude <= 24.5)
jgood = py.find(jgood)
shift = len(longitude[longitude >= (-69.5 + 360.)])
jgood = np.roll(jgood, shift)
igood = (latitude >= -74.5) & (latitude <= 9.5)
igood = py.find(igood)
latitude = latitude[igood]
longitude = longitude[jgood]
longitude[longitude > 180] = longitude[longitude > 180] - 360.
igood, jgood = np.meshgrid(igood, jgood)
# Organizing the data
months = data1.variables.keys()
data = np.ones((len(variables), len(months), latitude.shape[0],
longitude.shape[0])) * np.nan
for i in xrange(len(months)):
taux = data1[months[i]][:][igood, jgood].transpose()
tauy = data2[months[i]][:][igood, jgood].transpose()
curl = data3[months[i]][:][igood, jgood].transpose()
# Replacing the missing filling number -9999.0 by nan
ibad = taux == -9999.0
taux[ibad] = np.nan
ibad = tauy == -9999.0
tauy[ibad] = np.nan
ibad = curl == -9999.0
curl[ibad] = np.nan
# Fixing units (everything should be in the SI)
curl = curl * 1.e-7
month = np.array([taux, tauy, curl])
data[:, i, :, :] = month
self.data = data
self.months = months
self.variables = variables
self.latitude = latitude
self.longitude = longitude
def get_affine_inliers_RANSAC(num_m, xy1_m, xy2_m,\
acd1_m, acd2_m, xy_thresh_sqrd, sigma_thresh_sqrd=None):
'''Computes initial inliers by iteratively computing affine transformations
between matched keypoints'''
aff_inliers = []
# Enumerate All Hypothesis (Match transformations)
for mx in xrange(num_m):
xy1 = xy1_m[:,mx].reshape(2,1) # XY Positions
xy2 = xy2_m[:,mx].reshape(2,1)
A1 = matrix(insert(acd1_m[:,mx], [1.], 0.)).reshape(2,2)
A2 = matrix(insert(acd2_m[:,mx], [1.], 0.)).reshape(2,2)
# Compute Affine Tranform
# from img1 to img2 = (E2\E1)
Aff = linalg.inv(A2).dot(A1)
#
# Transform XY-Positions
xy1_mAt = xy2 + Aff.dot( (xy1_m - xy1) )
xy_err_sqrd = sum( power(xy1_mAt - xy2_m, 2) , 0)
_inliers = find(xy_err_sqrd < xy_thresh_sqrd)
#
# Transform Ellipse Geometry (solved on paper)
if not sigma_thresh_sqrd is None:
scale1_mAt = (acd1_m[0]*Aff[0,0]) *\
(acd1_m[1]*Aff[1,0]+acd1_m[2]*Aff[1,1])
scale2_m = acd2_m[0] * acd2_m[2]
scale_err = np.abs(scale1_mAt - scale2_m)
_inliers_scale = find(scale_err < sigma_thresh_sqrd)
_inliers = np.bitwise_and(_inliers, _inliers_scale)
#If this hypothesis transformation is better than the ones we have
#previously seen then set it as the best
if len(_inliers) > len(aff_inliers):
aff_inliers = _inliers
#bst_xy_err = xy_err_sqrd
return aff_inliers
def __init__(self, paramfile):
for row in paramfile:
if row[0]=='BIAS_COMBINE': self.BIAS_COMBINE = row[1]
if row[0]=='COMP_COMBINE': self.COMP_COMBINE = row[1]
if row[0]=='FLAT_COMBINE': self.FLAT_COMBINE = row[1]
if row[0]=='FIBER_TRACE': self.FIBER_TRACE = row[1]
if row[0]=='BIAS_SUBTRACT': self.BIAS_SUBTRACT = row[1]
if row[0]=='LAMBDA_SOLVE': self.LAMBDA_SOLVE = row[1]
if row[0]=='COSMIC_RAYS': self.COSMIC_RAYS = row[1]
if row[0]=='WRITE_SPECTRA': self.WRITE_SPECTRA = row[1]
if row[0]=='FIBERS_TOTAL': self.FIBERS_TOTAL = int(row[1])
if row[0]=='FIBERS_EXCLUDE':
fibs = pl.array( [ int(i) for i in row[1].split(',') ] )
fibs.sort()
fibs = fibs[ pl.find( 0<fibs ) ]
fibs = fibs[ pl.find( fibs<self.FIBERS_TOTAL+1 ) ]
self.FIBERS_EXCLUDE = fibs
if row[0]=='FIBER_WIDTH': self.FIBER_WIDTH = int(row[1])
if row[0]=='FIBER_SEP': self.FIBER_SEP = int(row[1])
if row[0]=='LO_BUFFER': self.LO_BUFFER = int(row[1])
if row[0]=='HI_BUFFER': self.HI_BUFFER = int(row[1])
if row[0]=='COMP_HEADER': self.COMP_HEADER = row[1]
if row[0]=='COMP_NAME': self.COMP_NAME = row[1]
if row[0]=='POLYFIT_ORDER': self.POLYFIT_ORDER = int(row[1])
if row[0]=='CR_SCAN_DX': self.CR_SCAN_DX = int(row[1])
def detect_signals():
vector, label = weeklydataset_sg_ndata(
"/media/4AC0AB31C0AB21E5/Documents and Settings/Claudio/Documenti/Thesis/Workloads/MSClaudio/ews/access_log-20110805.csv",
[],
)
x, target = aggregatebymins_sg_ndata(vector[1])
starttime = time.time()
y = array(target)
t = array(x)
thr = max(y) * 2 / 3
print thr
I = pylab.find(y > thr)
# print I
# pylab.plot(t,y, 'b',label='signal')
# pylab.plot(t[I], y[I],'ro',label='detections')
# pylab.plot([0, t[len(t)-1]], [thr,thr], 'g--')
J = pylab.find(diff(I) > 1)
argpeak = []
targetpeak = []
for K in split(I, J + 1):
ytag = y[K]
peak = pylab.find(ytag == max(ytag))
# pylab.plot(peak+K[0],ytag[peak],'sg',ms=7)
argpeak.append(peak + K[0])
targetpeak.append(ytag[peak])
eta = time.time() - starttime
print "time elapsed %f" % eta
return list(itertools.chain(*argpeak)), list(itertools.chain(*targetpeak))
def find_continuum(self): """ Creates a list of el states that are in the continuum. Stores the list as an instance variable. """ f = tables.openFile(self.name_el_couplings) #Electronic basis state energies. r_grid = f.root.R_grid[:] index_middle = argmin(abs(r_grid - 4)) energies = f.root.E[:,index_middle] f.close() continuum = [] #Looping the q's. for i in range(max(self.index_array[:,1])+1): #Get the indices corresponding to given q. temp_ind = find(self.index_array[:,1] == i) #Corresponding energies. temp_E = energies[temp_ind] #Checks if there are positive energies. if temp_E[-1] > 0: #The continuum starts when gap between energies increases. #OBS: TODO DEBUG border_value = temp_ind[argmin(diff(temp_E))] + 0 continuum.append(temp_ind[find((temp_ind > border_value))]) self.continuum = [item for sublist in continuum for item in sublist]
def dct_cut_downsampled(data, cutoff, spacing=0.4):
'''
like dctCut but also lowers the sampling rate, creating a compact
representation from which the whole downsampled data could be
recovered
'''
h, w = np.shape(data)[:2]
f1 = fft.fftfreq(h * 2, spacing)
f2 = fft.fftfreq(w * 2, spacing)
wl = 1. / np.abs(reshape(f1, (2 * h, 1)) + 1j * f2.reshape(1, 2 * w))
mask = np.int32(wl >= cutoff)
mirror = reflect2D(data)
ff = fft.fft2(mirror, axes=(0, 1))
cut = (ff.T * mask.T).T # weird shape broadcasting constraints
empty_cols = find(np.all(mask == 0, 0))
empty_rows = find(np.all(mask == 0, 1))
delete_col = len(empty_cols) / 2 #idiv important here
delete_row = len(empty_rows) / 2 #idiv important here
keep_cols = w - delete_col
keep_rows = h - delete_row
col_mask = np.zeros(w * 2)
col_mask[:keep_cols] = 1
col_mask[-keep_cols:] = 1
col_mask = col_mask == 1
row_mask = np.zeros(h * 2)
row_mask[:keep_rows] = 1
row_mask[-keep_rows:] = 1
row_mask = row_mask == 1
cut = cut[row_mask, ...][:, col_mask, ...]
w, h = keep_cols, keep_rows
result = fft.ifft2(cut, axes=(0, 1))[:h, :w, ...]
return result
def plt_annual_WBCt(fign):
fig = plt.figure(num=fign,figsize=(8,5),facecolor='w')
ax = plt.gca()
argo = np.ones(soest_gs.latitude.shape[0])*np.nan
wind = np.ones(scow_gs.latitude.shape[0])*np.nan
d = py.find(soest_gs.depth == 1200)[0]
for i in xrange(argo.shape[0]):
gs = soestgs_annual_Ipsi[d,i,:]
good = py.find(~np.isnan(gs))
if len(good) > 0:
argo[i] = gs[good[0]]
del gs, good
for i in xrange(wind.shape[0]):
gs = scowgs_annual_psi[i,:]
good = py.find(~np.isnan(gs))
if len(good) > 0:
wind[i] = gs[good[0]]
del gs, good
ax.plot(soest_gs.latitude,argo,'b',lw=3.5,label='ARGO')
ax.plot(scow_gs.latitude,wind,'r',lw=3.5,label='Sverdrup - Ekman')
ax.legend(numpoints=1,loc=0,prop={'size':20},frameon=False)
ax.plot([-60,0],[0,0],'k--',lw=3.5)
ax.set_ylim(-40,40)
ax.set_xlim(-5,-50)
ax.set_yticks(range(-40,50,10))
ax.set_xticks(range(-50,0,5))
ax.tick_params(labelsize=18)
rstyle(ax)
labels = [ur'5$^{\circ}$S',ur'10$^{\circ}$S',ur'15$^{\circ}$S',ur'20$^{\circ}$S',
ur'25$^{\circ}$S',ur'30$^{\circ}$S',ur'35$^{\circ}$S',ur'40$^{\circ}$S',ur'45$^{\circ}$S',
ur'50$^{\circ}$S']
labels.reverse()
ax.set_xticklabels(labels)
ax.set_ylabel('Transport at X$_{w}$ (Sv)',fontsize=30,fontweight='demibold')
ax.set_xlabel(ur'Latitude',fontsize=30,fontweight='demibold')
plt.draw()
plt.show(block=False)
return fig
def res2_name_consistency(em, res):
'''Score a single query for name consistency
Input:
res - query result
Returns: Dict
error_chip - degree of chip error
name_error - degree of name error
gt_pos_name -
gt_pos_chip -
'''
# Defaults to -1 if no ground truth is in the top results
cm, nm = em.hs.get_managers('cm','nm')
qcx = res.rr.qcx
qnid = res.rr.qnid
qnx = nm.nid2_nx[qnid]
ret = {'name_error':-1, 'chip_error':-1,
'gt_pos_chip':-1, 'gt_pos_name':-1,
'chip_precision': -1, 'chip_recall':-1}
if qnid == nm.UNIDEN_NID: exec('return ret')
# ----
# Score Top Chips
top_cx = res.cx_sort()
gt_pos_chip_list = (1+pylab.find(qnid == cm.cx2_nid(top_cx)))
# If a correct chip was in the top results
# Reward more chips for being in the top X
if len(gt_pos_chip_list) > 0:
# Use summation formula sum_i^n i = n(n+1)/2
ret['gt_pos_chip'] = gt_pos_chip_list.min()
_N = len(gt_pos_chip_list)
_SUM_DENOM = float(_N * (_N + 1)) / 2.0
ret['chip_error'] = float(gt_pos_chip_list.sum())/_SUM_DENOM
# Calculate Precision / Recall (depends on the # threshold/max_results)
ground_truth_cxs = np.setdiff1d(np.array(nm.nx2_cx_list[qnx]), np.array([qcx]))
true_positives = top_cx[gt_pos_chip_list-1]
false_positives = np.setdiff1d(top_cx, true_positives)
false_negatives = np.setdiff1d(ground_truth_cxs, top_cx)
nTP = float(len(true_positives)) # Correct result
nFP = float(len(false_positives)) # Unexpected result
nFN = float(len(false_negatives)) # Missing result
#nTN = float( # Correct absence of result
ret['chip_precision'] = nTP / (nTP + nFP)
ret['chip_recall'] = nTP / (nTP + nFN)
#ret['true_negative_rate'] = nTN / (nTN + nFP)
#ret['accuracy'] = (nTP + nFP) / (nTP + nTN + nFP + nFN)
# ----
# Score Top Names
(top_nx, _) = res.nxcx_sort()
gt_pos_name_list = (1+pylab.find(qnid == nm.nx2_nid[top_nx]))
# If a correct name was in the top results
if len(gt_pos_name_list) > 0:
ret['gt_pos_name'] = gt_pos_name_list.min()
# N should always be 1
_N = len(gt_pos_name_list)
_SUM_DENOM = float(_N * (_N + 1)) / 2.0
ret['name_error'] = float(gt_pos_name_list.sum())/_SUM_DENOM
# ----
# RETURN RESULTS
return ret
def psd(act, total_time, bin_w=5., time_range=[]):
evs = act[:, 0]
ts = act[:, 1]
if time_range != []:
idx = (ts > time_range[0]) & (ts <= time_range[1])
spikes = ts[idx]
if time_range == []:
total_time = total_time
else:
total_time = time_range[1] - time_range[0]
if len(spikes) == 0:
print 'psd: spike array is empty'
return np.nan, np.nan, np.nan, np.nan
ids = np.unique(evs)
nr_neurons = len(ids)
#psd, max_value, freq,h = misc2.psd_sp(spikes[:,1],nr_bins,nr_neurons)
bins = np.arange(time_range[0], time_range[1], bin_w)
a, b = np.histogram(spikes, bins)
ff = abs(np.fft.fft(a - np.mean(a)))**2
Fs = 1. / (bin_w * 0.001)
freq2 = np.fft.fftfreq(len(bins))[0:len(bins / 2) + 1]
freq = np.linspace(0, Fs / 2, len(ff) / 2 + 1)
px = ff[0:len(ff) / 2 + 1]
max_px = np.max(px[1:])
idx = px == max_px
corr_freq = freq[pl.find(idx)]
new_px = px
max_pow = new_px[pl.find(idx)]
return new_px, freq, freq2, corr_freq[0], max_pow
def nCR2(data, tau, q, off, bp, bo):
t, fc, b = data
to = 1.0 # to=1.0 : sheath trailing edge
cc = t[1:-1] <= to # la recuperacion ocurre despues de 'to'
cx = find(cc)
dt = t[1:-1] - t[0:-2]
nCR = np.nan * np.ones(t.size)
fcc = fc[1:-1]
bc = b[1:-1] - bo
bc[bc <= 0.0] = 0.0
#---- zona sheath
for i in cx:
ind = cx[:(i + 1)]
nCR[i + 1] = q * np.sum(fcc[:(i + 1)] * dt[:(i + 1)])
cy = find(~cc)
no = cx[-1]
#---- despues de sheath
for i in cy:
# termino rms
nCR[i + 1] = q * sum(fcc[:(i + 1)] * dt[:(i + 1)])
# termino recovery-after-sheath
nCR[i + 1] += (-1.0 / tau) * sum(nCR[1:-1][no:i] * dt[no:i])
nCR[i + 1] += 1.0 * off # offset
nCR[i + 1] += bp * sum(bc[no:i] * dt[no:i])
return nCR
def ellipse(width, height):
horpow = 2
verpow = horpow
# to produce a Ellipse with horizontal axis == ceil(2*hor semi axis)
width = width + 0.5
#to produce a Ellipse with vertical axis == ceil(2*vert semi axis)
height = height + 0.5
[x, y] = np.meshgrid(np.arange(-width, width + 1),
np.arange(-height, height + 1))
#print width, height, x.shape, y.shape
bll = (abs(x / width)**horpow + abs(y / height)**verpow) < 1
xs = plt.find(bll.sum(0) == 0)
ys = plt.find(bll.sum(1) == 0)
bll = bll[ys[0] + 1:ys[1], :]
bll = bll[:, xs[0] + 1:xs[1]]
bll = bll.T
mask_inds = np.where(bll > 0)
mask_inds = np.array(mask_inds).T
inv_mask_inds = np.where(bll == 0)
inv_mask_inds = np.array(inv_mask_inds).T
box_inds = np.array(np.where(bll != 5555)).T
box_edges = np.array([[box_inds[:, 0].min(), box_inds[:, 1].min()],
[box_inds[:, 0].max(), box_inds[:, 1].max()]])
return mask_inds, inv_mask_inds, box_edges
def make_continuum_list(self): """ Creates a list of the states that are in the continuum, based on the density of states. """ energies = diag(self.input_function.H_0) self.energies = energies continuum = [] border_value_list = [] #Looping the q's. for i in range(max(self.input_function.index_array[:,1])+1): #Get the indices corresponding to given q. temp_ind = find(self.input_function.index_array[:,1] == i) #Corresponding energies. temp_E = energies[temp_ind] #The continuum starts when gap between energies increases. border_value = temp_ind[argmin(diff(temp_E))] border_value_list.append(energies[border_value]) continuum.append(temp_ind[find((temp_ind > border_value))]) self.continuum = [item for sublist in continuum for item in sublist] self.continuum_limit_dos = mean(border_value_list)
def energy_spectrum(self, psi, energy_grid):
"""
energy_grid, spectrum = energy_spectrum(psi, energy_grid)
Creates the energy spectrum on the <energy_grid>.
"""
all_energies = diag(self.input_function.H_0)
all_population = abs(psi)**2
spectrum = zeros(len(energy_grid))
#Density of states.
for q in range(max(self.input_function.index_array[:,1])+1):
q_indices = find(self.input_function.index_array[:,1] == q)
energies = all_energies[q_indices]
population = all_population[q_indices]
dos = 1 / diff(energies)
if True:
#Choose an appropriate output grid.
start_index = find(energy_grid > energies[0])[0]
end_index = find(energy_grid < energies[-1])[-1] + 1
spectrum[start_index:end_index] += UnivariateSpline(energies[1:],
population[1:] * dos, s=0)(energy_grid[start_index:end_index])
else:
spectrum += UnivariateSpline(energies[1:],
population[1:] * dos, s=0)(energy_grid)
#raise
return energy_grid, spectrum
def lognormal_cdf( x, mu, sigma ):
if sigma > 0:
small = 0.5 + 0.5*special.erf( (np.log(1e-6)-mu)/(np.sqrt(2.0)*sigma))
if x.__class__ == np.ndarray:
lp = np.zeros( len(x) )
I = pp.find( x > 1e-6 )
J = pp.find( x <= 1e-6)
lp[I] = 0.5 + 0.5*special.erf( (np.log(x)-mu)/(np.sqrt(2.0)*sigma))
lp[J] = small
return lp
else:
if x > 1e-6:
return 0.5 + 0.5*special.erf( (np.log(x)-mu)/(np.sqrt(2.0)*sigma))
else:
return small
else:
if x.__class__ == np.ndarray:
logx = np.log(x+1e-6)
lp = 0.5*np.ones( len(x))
I1 = pp.find( logx < mu )
I2 = pp.find( logx > mu )
lp[I1] = 0
lp[I2] = 1
return lp
else:
if np.log(x) < mu:
return 0
elif np.log(x) > mu:
return 1
else:
return 0.5
def _CalculateMaxActivity(self,
culture_levels,
activities):
"""Calculates the maximal promoter activity.
Args:
culture_levels: the culture levels.
activities: the activities computed.
Returns:
The maximal (smoothed) reporter level within the
allowed range of culture levels.
"""
abovemin_i = pylab.find(culture_levels >= self.min_culture_level)
abovemax_i = pylab.find(culture_levels >= self.max_culture_level)
if not abovemin_i.any() or not abovemax_i.any():
return numpy.NAN
lower_i = numpy.min(abovemin_i)
upper_i = numpy.min(abovemax_i)
if lower_i >= upper_i:
return numpy.NAN
window_size = upper_i - lower_i
activities_window = activities[lower_i:lower_i+window_size]
diff_activities = self._MovingAverage(activities_window)
max_activity = numpy.max(diff_activities)
if max_activity < 0:
return numpy.NAN
return max_activity
def chop(self, start=None, stop=None):
"""
restrict the spectrum to only this wavelength range
(start and stop are in nm)
"""
if start is None:
start = self.wavelen[0]
if stop is None:
stop = self.wavelen[-1]
if start < self.wavelen[0] or stop > self.wavelen[-1]:
raise ValueError("wavelength range is not contained within\
the spectrum.")
self.lum = self.lum[pylab.find(self.wavelen > start)]
self.wavelen = self.wavelen[find(self.wavelen > start)]
self.lum = self.lum[pylab.find(self.wavelen < stop)]
self.wavelen = self.wavelen[find(self.wavelen < stop)]
self.lum -= max(self.lum)
# return self.lum
# return [self.wavelen, self.lum]
return [array(self.wavelen),
array(self.lum)] # returns a copy of the chopped data
def logpdf(x, mu, sigma):
if type(x) == np.ndarray:
if sigma > 0:
small = np.log(0.5 + 0.5 * special.erf((np.log(1e-6) - mu) /
(np.sqrt(2.0) * sigma)))
I = pp.find(x > 1e-6)
log_x = np.log(x[I])
lp = small * np.ones(x.shape)
lp[I] = -log_x - 0.5 * np.log(2.0 * np.pi) - np.log(sigma) - \
0.5 * ((log_x - mu) ** 2) / (sigma ** 2)
else:
I = pp.find(x == mu)
lp = -np.inf * np.ones(x.shape)
lp[I] = 0
else:
if sigma > 0:
if x > 1e-6:
log_x = np.log(x)
lp = - log_x - 0.5 * np.log(2.0 * np.pi) - \
np.log(sigma) - \
0.5 * ((log_x - mu) ** 2) / (sigma ** 2)
else:
lp = np.log(0.5 +
0.5 * special.erf((np.log(1e-6) - mu) /
(np.sqrt(2.0) * sigma)))
else:
if x == mu:
lp = 0
else:
lp = -np.inf
return lp
def _CalculateMaxActivity(self, culture_levels, activities):
"""Calculates the maximal promoter activity.
Args:
culture_levels: the culture levels.
activities: the activities computed.
Returns:
The maximal (smoothed) reporter level within the
allowed range of culture levels.
"""
abovemin_i = pylab.find(culture_levels >= self.min_culture_level)
abovemax_i = pylab.find(culture_levels >= self.max_culture_level)
if not abovemin_i.any() or not abovemax_i.any():
return numpy.NAN
lower_i = numpy.min(abovemin_i)
upper_i = numpy.min(abovemax_i)
if lower_i >= upper_i:
return numpy.NAN
window_size = upper_i - lower_i
activities_window = activities[lower_i:lower_i + window_size]
diff_activities = self._MovingAverage(activities_window)
max_activity = numpy.max(diff_activities)
if max_activity < 0:
return numpy.NAN
return max_activity
def ConditionalNRLHist(self, nrls, labels):
"""
Analyze method
:param nrls:
:type nrls:
:param labels:
:type labels:
:rtype: Resultlist of figures
"""
figures = []
for m in range(0, self.scen.M):
q = labels[m, 1, :]
ind = find(q >= 0.5)
ind2 = find(q < 0.5)
r = nrls[ind]
xmin, xmax = min(nrls), max(nrls)
lnspc = linspace(xmin, xmax, 100)
m, s = stats.norm.fit(r)
pdf_g = stats.norm.pdf(lnspc, m, s)
r = nrls[ind2]
# xmin, xmax = min(r), max(r)
lnspc2 = linspace(xmin, xmax, 100) # len(r)
m, s = stats.norm.fit(r)
pdf_g2 = stats.norm.pdf(lnspc2, m, s)
fg = figure()
plot(lnspc, pdf_g / len(pdf_g), label="Norm")
hold(True)
plot(lnspc2, pdf_g2 / len(pdf_g2), 'k', label="Norm")
legend(['Posterior: Activated', 'Posterior: Non Activated'])
# xmin, xmax = min(xt), max(xt)
# ind2 = find(q <= 0.5)
figures.append(fg)
if self.shower:
show()
return figures
def named_state_population(self, psi, name_list):
"""
populations = named_state_population(psi, name_list)
Returns the population in a set of named states.
Parameters
----------
psi : 2D complex array. The wavefunction to be analyzed.
name_list : string list, e.g. ["2p", "3d"].
Returns
-------
populations : 1D float array, the population of the basis states.
"""
angular_names = {"s":0, "p":1, "d":2, "f":3}
el_indices = []
for i in name_list:
q_matches = find(self.index_array[:,1] == (angular_names[i[1]]))
n_matches = find(self.index_array[:,2] == int(i[0]) - angular_names[i[1]] -1)
el_indices.append(intersect1d(q_matches, n_matches))
el_indices = list(ravel(array(el_indices)))
populations = self.state_population(psi, el_indices)
return populations
def extract_time_series(self, filename, node_yxz):
dis = self.parent.get_package('DIS')
if (dis != None):
yy, xx, zz = dis.get_node_coordinates()
times, heads, strings = self.read_all(filename)
if self.silent == 0: print('Please wait. Extracting time series.')
ht = np.empty((1, len(node_yxz)))
rr = 0
for h in heads:
if (rr > 0):
ht = np.vstack((ht, np.empty(len(node_yxz))))
cc = 0
for yxz in node_yxz:
y = yxz[0]
x = yxz[1]
z = yxz[2]
r = find(abs(yy - y) < abs(y) / 1e6)[0]
c = find(abs(xx - x) < abs(x) / 1e6)[0]
l = find(abs(zz[r, c] - z) < abs(z) / 1e6)[0]
ht[rr, cc] = h[r, c, l]
cc = cc + 1
rr = rr + 1
return times, ht
def vl_test_hikmeans():
""" VL_TEST_HIKMEANS Test VL_HIKMEANS function
"""
K = 3
nleaves = 100
data = numpy.array(numpy.random.rand(2, 1000) * 255, 'uint8')
datat = numpy.array(numpy.random.rand(2, 10000) * 255, 'uint8')
[tree, A] = vlfeat.vl_hikmeans(data, K, nleaves, verb=1)
AT = vlfeat.vl_hikmeanspush(tree, datat)
pylab.figure()
plottree(tree)
pylab.xlim(0, 255)
pylab.ylim(0, 255)
print('hikmeans-tree') ;
pylab.figure()
gen = color_gen()
for k in range(K*K):
color = next(gen)
sel = pylab.find(A[-1, :] == k)
pylab.plot(data[0, sel], data[1, sel], '.', color=color)
sel = pylab.find(AT[-1, :] == k)
pylab.plot(datat[0, sel], datat[1, sel], '+', color=color)
plottree(tree, linewidth=4)
pylab.xlim(0, 255)
pylab.ylim(0, 255)
print('hikmeans-clusters') ;
def life_time(Zpy, ns):
"""
life-time criterion for automatic selection of the number of clusters
[porting from life-time implementation on matlab]
Input:
Zpy (array): input data array of shape (number samples x number features).
ns (int): number of samples.
Output:
res (dict): output dict with indexes for each cluster determined.
Example: res = {'-1': noise indexes list,
'0': cluster 0 indexes list,
'1': cluster 1 indexes list}
Configurable fields:{"name": "cluster.dbscan", "config": {"min_samples": "10", "eps": "0.95", "metric": "euclidean"}, "inputs": ["data"], "outputs": ["core_samples", "labels"]}
See Also:
Example:
References:
.. [1]
"""
Z = hierarchy.to_mlab_linkage(Zpy)
#dif=Z[1:,2]-Z[0:-1,2]
dif = np.diff(Z[:, 2])
indice = np.argmax(dif)
maximo = dif[indice]
indice = Z[find(Z[:, 2] > Z[indice, 2]), 2]
if indice == []:
cont = 1
else:
cont = len(indice) + 1
# th = maximo
#testing the situation when only 1 cluster is present
#max>2*min_interval -> nc=1
minimo = np.min(dif[pl.find(dif != 0)])
if minimo != maximo: #se maximo=minimo e' porque temos um matriz de assocs perfeita de 0s e 1s
if maximo < 2 * minimo:
cont = 1
nc_stable = cont
if nc_stable > 1:
labels = hierarchy.fcluster(hierarchy.from_mlab_linkage(Z), nc_stable,
'maxclust')
else: #ns_stable=1
labels = np.arange(ns, dtype="int")
return labels
def climatology(t, z, w=None, result='year'):
"""Returns monthly climatology of a time-series.
PARAMETERS
t (array like) :
Time in matplotlib time format.
z (array like) :
The data to calculate climatology.
w (array like) :
Data weight, should have same dimensions as 'z'.
result (string) :
If set to 'year', returns only one year of data. If set to
'full', returns the climatology for every time t.
RETURN
z_clim (array like) :
Climatological averages.
"""
# Checks for proper dimensions
shape = z.shape
if len(shape) == 1:
z = z[:, None, None]
c, b, a = z.shape
if w != None:
if w.shape != z.shape:
raise Warning, 'Data and weight arrays are not the same.'
# Starts converting time to datetime format. Determines the start and end
# of the relevant dataset to ensure that only whole years are used.
# Initializes climatology variable and calculates the averages.
Time = common.num2ymd
try:
start = pylab.find(Time[:, 1] == 1)[0]
except:
start = None
try:
end = pylab.find(Time[:, 1] == 12)[-1]
except:
end = None
time_clim = numpy.arange(12) + 1
z_clim = numpy.ma.zeros([12, b, a])
for i in range(12):
selt = pylab.find(Time[start:end+1, 1] == i + 1) + start
if w == None:
z_clim[i, :, :] = z[selt, :, :].mean(axis=0)
else:
z_clim[i, :, :] = ((z[selt, :, :] * w[selt, :, :]).sum(axis=0) /
w[selt, :, :].sum(axis=0))
#
z_clim.mask = z_clim.mask | numpy.isnan(z_clim.data)
if result == 'full':
z_clim = z_clim[Time[:, 1]-1]
return z_clim, Time
def __init__(self, name, version, zclust, sigmaz, alpha=50):
self.name = name # e.g. "NEP 200"
self.version = version # e.g. "nep200_v0.0.4"
self.zclust = zclust # cluster redshift
self.sigmaz = sigmaz # 1sigma scatter in (zphot-zspec)/(1+zspec)
self.zspec_lo = 0 # lower redshift bound for specz
self.zspec_hi = 0 # upper redshift bound for specz
self.substructures = []
self.cat = gunzip_read_gzip('%s/%s/%s.cat.gz' %
(data_dir, version, version),
readcat=1)
self.zout = gunzip_read_gzip('%s/%s/%s.zout.gz' %
(data_dir, version, version),
readzout=1)
self.fout = gunzip_read_gzip('%s/%s/%s.fout.gz' %
(data_dir, version, version),
readcat=1)
### UPDATING USE FLAG WITH REDUCE CHI**2 > 10
chi2red = self.zout.chi_p / (self.zout.nfilt - 1.)
cinds = pylab.find(chi2red > 10)
self.cat.use[cinds] = 0.
#print ' reading: %s_voronoi.pickle' % name
#self.voronoi = pickle.load(open('../data/%s_voronoi.pickle' % name, 'rb'))
xyrd1 = self.cat.x[0], self.cat.y[0], self.cat.ra[0], self.cat.dec[0]
xyrd2 = self.cat.x[1], self.cat.y[1], self.cat.ra[1], self.cat.dec[1]
d_arcsec = mypy.radec_sep(xyrd1[2], xyrd1[3], xyrd2[2], xyrd2[3])
d_pixel = ((xyrd1[0] - xyrd2[0])**2 + (xyrd1[1] - xyrd2[1])**2)**0.5
self.px_scale = d_arcsec / d_pixel
### getting z band magnitude
try:
self.zmagnitude = 25 - 2.5 * pylab.log10(self.cat.fluxauto_z)
except:
pass
try:
self.zmagnitude = 25 - 2.5 * pylab.log10(self.cat.fluxauto_Zplus)
except:
pass
### setting spatial flag based on Convex Hull method
#zspecinds = pylab.find(self.cat.z_spec > 0)
#self.spatial_flag = checkPoint(self.cat.ra[zspecinds], self.cat.dec[zspecinds], self.cat.ra, self.cat.dec)
#self.inds_spatial = pylab.find(self.spatial_flag == 1)
### setting spatial flag based on Concave Hull method (creates a "tighter" spatial selection than Cnvex Hull)
zspecinds = pylab.find(self.cat.z_spec > 0)
points = pylab.array(zip(self.cat.x[zspecinds], self.cat.y[zspecinds]))
self.concave_hull, self.edge_points = alpha_shape(points, alpha)
self.inds_spatial = CheckPoints(self.concave_hull.buffer(10),
self.cat.x, self.cat.y)
self.area_pix2 = self.concave_hull.buffer(10).area
self.area_arcmin2 = self.area_pix2 * (self.px_scale / 60.)**2
print ''
def GetAverageRange(X, lower_lim, domainheight): # # get range of X for averaging such that lower_lim<X<h0 try: start_val = pylab.find(numpy.array(X)>lower_lim)[0] except IndexError: print "not enough values of X, \n lower lim = " + str(lower_lim) + "\n X = " + str(X); sys.exit(1) try: end_val = pylab.find(numpy.array(X)>0.4-domainheight)[0] except IndexError: end_val = len(X) # return start_val, end_val
def climatology(t, z, w=None, result='year'):
"""Returns monthly climatology of a time-series.
PARAMETERS
t (array like) :
Time in matplotlib time format.
z (array like) :
The data to calculate climatology.
w (array like) :
Data weight, should have same dimensions as 'z'.
result (string) :
If set to 'year', returns only one year of data. If set to
'full', returns the climatology for every time t.
RETURN
z_clim (array like) :
Climatological averages.
"""
# Checks for proper dimensions
shape = z.shape
if len(shape) == 1:
z = z[:, None, None]
c, b, a = z.shape
if w != None:
if w.shape != z.shape:
raise Warning, 'Data and weight arrays are not the same.'
# Starts converting time to datetime format. Determines the start and end
# of the relevant dataset to ensure that only whole years are used.
# Initializes climatology variable and calculates the averages.
Time = common.num2ymd
try:
start = pylab.find(Time[:, 1] == 1)[0]
except:
start = None
try:
end = pylab.find(Time[:, 1] == 12)[-1]
except:
end = None
time_clim = numpy.arange(12) + 1
z_clim = numpy.ma.zeros([12, b, a])
for i in range(12):
selt = pylab.find(Time[start:end + 1, 1] == i + 1) + start
if w == None:
z_clim[i, :, :] = z[selt, :, :].mean(axis=0)
else:
z_clim[i, :, :] = ((z[selt, :, :] * w[selt, :, :]).sum(axis=0) /
w[selt, :, :].sum(axis=0))
#
z_clim.mask = z_clim.mask | numpy.isnan(z_clim.data)
if result == 'full':
z_clim = z_clim[Time[:, 1] - 1]
return z_clim, Time
def pdf_from_cv(cv, origin, roundoff=1e-9):
"""
Returns a gaussian pdf from a given covariance matrix
@param cv (n-by-n): covariance matrix
@param origin (n-by-1): origin
@param roundoff (float): ratio of largest to smallest eigenvalue
s.th. smallest eigenvalue is not considered 0 (degenerate case)
@return Returns a pdf function
"""
if not np.allclose(cv - cv.T, 0):
raise ValueError("argument must be a positive definite symmetric "
"covariance matrix")
dim = cv.shape[0]
x_ref = np.array(origin)
if len(x_ref) != dim:
raise ValueError("x_ref dimension must match covariance dimension")
# this is numerically stable here since cv is symmetric
# -> eigenvectors are normal
eigvals, eigvecs = np.linalg.eig(cv)
if not all(eigvals > 0):
raise ValueError("argument must be a positive definite "
"symmetric covariance matrix")
max_ev = max(eigvals)
small_ev = find(eigvals < roundoff*max_ev)
large_ev = find(eigvals >= roundoff*max_ev)
eigvals_i = eigvals.copy()
eigvals_i[small_ev] = 0
eigvals_i[large_ev] = 1./eigvals_i[large_ev]
# compute the pseudo-inverse using eigenvalues
# (numerically stable here, see above)
cv_i = np.dot(eigvecs,
np.dot(np.diag(eigvals_i), eigvecs.T))
p_det = reduce(lambda x, y: x*y, eigvals[large_ev]) # pseudo-determinant
scale = (1. / np.sqrt((2*np.pi)**dim * p_det))
projector = np.dot(cv, cv_i)
def fun(xp, threshold=1e-10, debug=False):
x = np.dot(projector, xp)
if np.linalg.norm(x - xp) > threshold:
if debug:
print "debug:", svd(projector, 0, 0)
print "threshold exceeded:", x, xp
print ("(norm of", x - xp, ":", np.linalg.norm(x - xp), " > ",
threshold, ") ")
return 0
return scale * np.exp(-0.5 * np.dot(x - x_ref, np.dot(cv_i, x - x_ref)))
return fun
def get_max_counts_in_histogram2D(self):
high_Gbins = abs(np.log10(self.Gbins) - -2)
low_Gbins = abs(np.log10(self.Gbins) - -6)
low_idx = pl.find(min(low_Gbins) == low_Gbins)[0]
high_idx = pl.find(min(high_Gbins) == high_Gbins)[0]
tmp = self.histo2D_breaking_sum[low_idx:high_idx, :]
Cmax = 0
for y in range(0, high_idx - low_idx):
Cmax = max(max(tmp[y, :]), Cmax)
return int(Cmax)
def assign_woe_discrete(woes, bins, values):
results = np.zeros(values.shape[0])
others = np.ones(values.shape[0])
for i in range(len(woes)-1):
inds = values==bins[i]
others[plb.find(inds==True)] = 0
if inds.shape[0] > 0:
results[inds] = woes[i]
results[plb.find(others==1)] = woes[-1]
return results
def nonna_select_data(data, outlier_threshold, level='high'): """ This function returns a list of indexed after identifying the main outliers. It applies a cut on the data to remove exactly a fraction (1-outlier_threshold) of all data points. By default the cut is applied only at the higher end of the data values, but the parameter level can be used to change this Input arguments: data = vector containing all data points outlier_threshold = remove outliers until we are left with exactly this fraction of the original data level = 'high|low|both' determines if the outliers are removed only from the high values end, the low values end of both ends. Output: idx = index of selected (good) data """ # histogram all the data values n,x = scipy.histogram(data, len(data)/10) # compute the cumulative distribution and normalize nn = scipy.cumsum(n) nn = nn / float(max(nn)) if level=='high': # select the value such that a fraction outlier_threshold of the data lies below it if outlier_threshold < 1: val = x[pylab.find(nn/float(max(nn)) >= outlier_threshold)[0]] else: val = max(data) # use that fraction of data only idx = data <= val elif level=='low': # select the value such that a fraction outlier_threshold of the data lies above it if outlier_threshold < 1: val = x[pylab.find(nn/float(max(nn)) <= (1-outlier_threshold))[-1]] else: val = min(data) # use that fraction of data only idx = data >= val elif level=='both': # select the value such that a fraction outlier_threshold/2 of the data lies below it if outlier_threshold < 1: Hval = x[pylab.find(nn/float(max(nn)) >= 1-(1-outlier_threshold)/2)[0]] else: Hval = max(data) # select the value such that a fraction outlier_threshold/2 of the data lies above it if outlier_threshold < 1: Lval = x[pylab.find(nn/float(max(nn)) <= (1-outlier_threshold)/2)[-1]] else: Lval = min(data) # use that fraction of data only idx = scipy.logical_and(data >= Lval, data <= Hval) return idx
def positive_normal_rand( mu, stddevs, N = 1 ):
X = mu + stddevs*np.random.randn( N )
I = pp.find( X <= 0 )
i = 0
while len(I) > 0:
X[I] = mu + stddevs*np.random.randn( len(I) )
J = pp.find( X[I] <= 0 )
I = I[J]
return X
def positive_normal_rand(mu, stddevs, N=1):
X = mu + stddevs * np.random.randn(N)
I = pp.find(X <= 0)
i = 0
while len(I) > 0:
X[I] = mu + stddevs * np.random.randn(len(I))
J = pp.find(X[I] <= 0)
I = I[J]
return X
def plot_partition(data, datat, C, A, AT): K = C.shape[1] colors = ['r', 'g', 'b', 'y', '#444444'] for k in range(K): sel = pylab.find(A==k) selt = pylab.find(AT==k) vl_plotframe(data[:,sel], color=colors[k]) vl_plotframe(datat[:,selt], color=colors[k]) pylab.plot(C[0,:],C[1,:],'ko', markersize=14, linewidth=6) pylab.plot(C[0,:],C[1,:],'yo', markersize=10, linewidth=1)
def plot_partition(data, datat, C, A, AT):
K = C.shape[1]
colors = ["r", "g", "b", "y", "#444444"]
for k in range(K):
sel = pylab.find(A == k)
selt = pylab.find(AT == k)
vl_plotframe(data[:, sel], color=colors[k])
vl_plotframe(datat[:, selt], color=colors[k])
pylab.plot(C[0, :], C[1, :], "ko", markersize=14, linewidth=6)
pylab.plot(C[0, :], C[1, :], "yo", markersize=10, linewidth=1)
def mixing_stats():
pylab.figure(num=2, figsize=(16.5, 11.5))
pylab.suptitle('Mixing')
times = []
mixing_stats_lower = []
mixing_stats_mixed = []
mixing_stats_upper = []
stat_files, time_index_end = le_tools.GetstatFiles('./')
for sf in stat_files:
stat = stat_parser(sf)
for i in range(
time_index_end[pylab.find(numpy.array(stat_files) == sf)]):
times.append(stat['ElapsedTime']['value'][i])
mixing_stats_lower.append(stat['fluid']['Temperature']
['mixing_bins%cv_normalised'][0][i])
mixing_stats_mixed.append(stat['fluid']['Temperature']
['mixing_bins%cv_normalised'][1][i])
mixing_stats_upper.append(stat['fluid']['Temperature']
['mixing_bins%cv_normalised'][2][i])
pylab.plot(times, mixing_stats_lower, label='T < -0.4')
pylab.plot(times, mixing_stats_mixed, label='-0.4 < T < 0.4')
pylab.plot(times, mixing_stats_upper, label='0.4 < T')
time = le_tools.ReadLog('diagnostics/logs/time.log')
X_ns = [x - 0.4 for x in le_tools.ReadLog('diagnostics/logs/X_ns.log')]
X_fs = [0.4 - x for x in le_tools.ReadLog('diagnostics/logs/X_fs.log')]
try:
index = pylab.find(numpy.array(X_ns) < 0.4 - 1E-3)[-1]
pylab.fill_between([time[index], time[index + 1]], [0, 0], [0.5, 0.5],
color='0.3')
index = pylab.find(numpy.array(X_fs) < 0.4 - 1E-3)[-1]
pylab.fill_between([time[index], time[index + 1]], [0, 0], [0.5, 0.5],
color='0.6')
except IndexError:
print 'not plotting shaded regions on mixing plot as front has not reached end wall'
pylab.axis([0, times[-1], 0, 0.5])
pylab.grid("True")
pylab.legend(loc=0)
pylab.text(
times[-1] - (times[-1] / 5),
0.005,
'shaded regions show when \nfronts near the end wall \ndark: free-slip, light: no-slip',
bbox=dict(facecolor='white', edgecolor='black'))
pylab.xlabel('time (s)')
pylab.ylabel('domain fraction')
pylab.savefig('diagnostics/plots/mixing.png')
return
def get_edges(signal):
'''
Assuming a binary signal, get the start and stop times of each
treatch of "1s"
'''
if len(signal) < 1:
return np.array([[], []])
starts = list(find(np.diff(np.int32(signal)) == 1))
stops = list(find(np.diff(np.int32(signal)) == -1))
if signal[0]: starts = [0] + starts
if signal[-1]: stops = stops + [len(signal)]
return np.array([starts, stops])
def print_batch(self, X, b):
from pylab import find
if len(X[0][0])==1:
M = X
ans = ['01'[int(m[b].asarray())] for m in M]
else:
inds = [int(find(x[b].asarray())) for x in X
if len(find(x[b].asarray()))>0]
ans = [self.chars_inv_dict[i] for i in inds]
return ''.join(ans).replace('\n', ' ').replace('\t', ' ')
def align_spike_peaks(waveform, threshold_mv):
"""Align each spike by its peak and scrunch down length apropriately"""
#waveform -= pylab.matrix(waveform[:,:10].mean(axis=1)).T*pylab.matrix(pylab.ones((1,waveform.shape[1])))
idx = pylab.find(waveform.max(axis=1) < threshold_mv)
waveform = waveform[idx,:]
peak_idx = waveform.argmin(axis=1)#a row vector of the peak indices for each spike
idx = pylab.find((peak_idx > 5) & (peak_idx < 15))
wv = pylab.zeros((idx.size,30))
for n in range(idx.size):
wv[n,:] = waveform[idx[n],peak_idx[idx[n]]-6:peak_idx[idx[n]]+24]
return wv
def get_files():
if os.name == 'nt':
sep = "\\" #windows
else:
sep = "/" #unix, MacOs, Java, others
header_loc, arousal_loc, signal_loc, is_training = [], [], [], []
rootDir = p_DATASET_DIR
print("eseguo phic get files: " + str(rootDir))
for dirName, subdirList, fileList in sorted(
os.walk(rootDir, followlinks=True)): #sorted(
if dirName != rootDir and dirName != rootDir + sep + 'test' and dirName != rootDir + sep + 'training':
if dirName.startswith(rootDir + '' + sep + 'training' + sep + ''):
is_training.append(True)
#print("TRAIN### : "+dirName)
for fname in fileList:
if '.hea' in fname:
header_loc.append(dirName + sep + fname)
if '-arousal.mat' in fname:
arousal_loc.append(dirName + sep + fname)
if 'mat' in fname and 'arousal' not in fname:
signal_loc.append(dirName + sep + fname)
elif dirName.startswith(rootDir + '' + sep + 'test' + sep + ''):
is_training.append(False)
#print("TEST### : " + dirName)
arousal_loc.append('')
for fname in fileList:
if '.hea' in fname:
header_loc.append(dirName + sep + fname)
if 'mat' in fname and 'arousal' not in fname:
signal_loc.append(dirName + sep + fname)
# combine into a data frame
data_locations = {
'header': header_loc,
'arousal': arousal_loc,
'signal': signal_loc,
'is_training': is_training
}
#print (data_locations)
# Convert to a data-frame
df = pd.DataFrame(data=data_locations)
# Split the data frame into training and testing sets.
tr_ind = list(find(df.is_training.values))
te_ind = list(find(df.is_training.values == False))
training_files = df.loc[tr_ind, :]
testing_files = df.loc[te_ind, :]
return training_files, testing_files
def print_batch(self, X, b):
from pylab import find
if len(X[0][0])==1:
M = X
ans = ['01'[int(m[b].asarray())] for m in M]
else:
inds = [int(find(x[b].asarray())) for x in X
if len(find(x[b].asarray()))>0]
ans = [self.chars_inv_dict[i] for i in inds]
return ''.join(ans).replace('\n', ' ').replace('\t', ' ')
def svdInverse(mat,maxEig=1e10,minEig=1e-10): #1e10,1e-10
u,w,vt = scipy.linalg.svd(mat)
if any(w==0.):
raise ZeroDivisionError, "Singular matrix."
wInv = w ** -1
largeIndices = pylab.find( abs(wInv) > maxEig )
if len(largeIndices) > 0: print "svdInverse:",len(largeIndices),"large singular values out of",len(w)
wInv[largeIndices] = maxEig*scipy.sign(wInv[largeIndices])
smallIndices = pylab.find( abs(wInv) < minEig )
if len(smallIndices) > 0: print "svdInverse:",len(smallIndices),"small singular values out of",len(w)
wInv[smallIndices] = minEig*scipy.sign(wInv[smallIndices])
return scipy.dot( scipy.dot(vt.T,scipy.diag(wInv)), u.T )
def normal_drawer(plane_normal, plane_seed):
print( " Drawing plane Normals")
for plane in range(len(plane_normal)):
pl= plane_seed[plane]
pl=pl[pl.find('(') :]
pl=pl[pl.find('[')+1 :]
pl=pl[: pl.find(']')]
pt=pl.split(',')
#print(pt)
p0x=float(pt[0])
p0y=float(pt[1])
p0z=float(pt[2])
pn=plane_normal[plane]
p0 = [p0x, p0y, p0z]
p1 = [p0x+pn[0], p0y+pn[1], p0z+pn[2]]
# Create a vtkPoints object and store the points in it
points = vtk.vtkPoints()
#points.InsertNextPoint(origin)
points.InsertNextPoint(p0)
points.InsertNextPoint(p1)
# Create a cell array to store the lines in and add the lines to it
lines = vtk.vtkCellArray()
line = vtk.vtkLine()
line.GetPointIds().SetId(0,0)
line.GetPointIds().SetId(1,1)
lines.InsertNextCell(line)
# Create a polydata to store everything in
linesPolyData = vtk.vtkPolyData()
# Add the points to the dataset
linesPolyData.SetPoints(points)
# Add the lines to the dataset
linesPolyData.SetLines(lines)
# Setup actor and mapper
mapper = vtk.vtkPolyDataMapper()
if vtk.VTK_MAJOR_VERSION <= 5:
mapper.SetInput(linesPolyData)
else:
mapper.SetInputData(linesPolyData)
actor = vtk.vtkActor()
actor.SetMapper(mapper)
actor.GetProperty().SetColor(1,0,0)
renderer.AddActor(actor)
def homogAnalyze(merged,numRuns=10,genFigs=False,maintainNetwork=False):
agroups=merged.groupby('pid')
ethnicitysets=[tmp[1].ethnicity.tolist() for tmp in iter(agroups)]
# Testing for homogeneous sets
################################
homog=lambda x:np.all([tmp==x[0] for tmp in x])
homogs=[homog(tmp) for tmp in ethnicitysets if len(tmp)>1]
mainlyhomog=lambda x:pd.value_counts(x).iloc[0] >= 0.75*len(x)
mainlyhomogs=[mainlyhomog(tmp) for tmp in ethnicitysets if len(tmp)>1]
homogcount=len(pl.find(homogs))
mainlyhomogcount=len(pl.find(mainlyhomogs))
# Randomly generated sets
############################
random.seed(10)
results=[]
mainlyresults=[]
for count in range(numRuns):
if not maintainNetwork:
elist=merged.ethnicity.tolist()
random.shuffle(elist)
randomsets=[]
for length in [len(tmp) for tmp in ethnicitysets]:
randomsets.append([elist.pop() for tmp in range(length)])
else: # Shuffle in a way which maintains the overall structure of the network
mixedauthors=np.unique(merged['aid'])
random.shuffle(mixedauthors)
shuffledict={mixedauthors[tmp]:mixedauthors[tmp+1] for tmp in range(0,len(mixedauthors)/2)}
randomhomogs=[homog(tmp) for tmp in randomsets if len(tmp)>1]
results.append(len(pl.find(randomhomogs)))
randommainlyhomogs=[mainlyhomog(tmp) for tmp in randomsets if len(tmp)>1]
mainlyresults.append(len(pl.find(randommainlyhomogs)))
randomhomogcount=np.mean(results)
randommainlyhomogcount=np.mean(mainlyresults)
return homogcount,mainlyhomogcount,randomhomogcount,randommainlyhomogcount
def normal_drawer(plane_normal, plane_seed):
print(" Drawing plane Normals")
for plane in range(len(plane_normal)):
pl = plane_seed[plane]
pl = pl[pl.find('('):]
pl = pl[pl.find('[') + 1:]
pl = pl[:pl.find(']')]
pt = pl.split(',')
#print(pt)
p0x = float(pt[0])
p0y = float(pt[1])
p0z = float(pt[2])
pn = plane_normal[plane]
p0 = [p0x, p0y, p0z]
p1 = [p0x + pn[0], p0y + pn[1], p0z + pn[2]]
# Create a vtkPoints object and store the points in it
points = vtk.vtkPoints()
#points.InsertNextPoint(origin)
points.InsertNextPoint(p0)
points.InsertNextPoint(p1)
# Create a cell array to store the lines in and add the lines to it
lines = vtk.vtkCellArray()
line = vtk.vtkLine()
line.GetPointIds().SetId(0, 0)
line.GetPointIds().SetId(1, 1)
lines.InsertNextCell(line)
# Create a polydata to store everything in
linesPolyData = vtk.vtkPolyData()
# Add the points to the dataset
linesPolyData.SetPoints(points)
# Add the lines to the dataset
linesPolyData.SetLines(lines)
# Setup actor and mapper
mapper = vtk.vtkPolyDataMapper()
if vtk.VTK_MAJOR_VERSION <= 5:
mapper.SetInput(linesPolyData)
else:
mapper.SetInputData(linesPolyData)
actor = vtk.vtkActor()
actor.SetMapper(mapper)
actor.GetProperty().SetColor(1, 0, 0)
renderer.AddActor(actor)
def radialprofile_errors(odprofiles, angles, od_prof, od_cutoff, \
showfig=False, savefig_name=None, report=True):
"""Calculate errors in radial profiles as a function of angle
**Inputs**
* odprofiles: 2D array, containing radial OD profiles along angles
* angles: 1D array, angles at which radial profiles are taken
(zero is postive x-axis)
* od_prof: 1D array, radially averaged optical density
* od_cutoff: integer, index of profiles at which maximum fit-OD is reached
**Outputs**
* av_err: float, sum of absolute values of errors in errsum
**Optional inputs**
* showfig: bool, determines if a figure is shown with density profile
and fit
* report: bool, if True print the sums of the mean and rms errors
* savefig_name: string, if not None and showfig is True, the figure is
not shown but saved as png with this string as filename.
"""
err = (odprofiles[od_cutoff:, :].transpose() - \
od_prof[od_cutoff:]).sum(axis=1)
av_err = np.abs(err).mean()
if report:
print 'mean error is ', err.mean()
print 'rms error is ', av_err
if showfig:
# angular plot of errors, red for positive, blue for negative values
pylab.figure()
poserr = pylab.find(err > 0)
negerr = pylab.find(err < 0)
pylab.polar(angles[negerr], np.abs(err[negerr]), 'ko', \
angles[poserr], np.abs(err[poserr]), 'wo')
pylab.title('Angular dependence of fit error')
pylab.text(np.pi/2+0.3, np.abs(err).max()*0.85, \
r'$\sum_{\phi}|\sum_r\Delta_{OD}|=%1.1f$'%av_err)
if not savefig_name:
pylab.show()
else:
pylab.savefig(''.join([os.path.splitext(savefig_name)[0], '.png']))
return av_err
def radialprofile_errors(odprofiles, angles, od_prof, od_cutoff, \
showfig=False, savefig_name=None, report=True):
"""Calculate errors in radial profiles as a function of angle
**Inputs**
* odprofiles: 2D array, containing radial OD profiles along angles
* angles: 1D array, angles at which radial profiles are taken
(zero is postive x-axis)
* od_prof: 1D array, radially averaged optical density
* od_cutoff: integer, index of profiles at which maximum fit-OD is reached
**Outputs**
* av_err: float, sum of absolute values of errors in errsum
**Optional inputs**
* showfig: bool, determines if a figure is shown with density profile
and fit
* report: bool, if True print the sums of the mean and rms errors
* savefig_name: string, if not None and showfig is True, the figure is
not shown but saved as png with this string as filename.
"""
err = (odprofiles[od_cutoff:, :].transpose() - \
od_prof[od_cutoff:]).sum(axis=1)
av_err = np.abs(err).mean()
if report:
print 'mean error is ', err.mean()
print 'rms error is ', av_err
if showfig:
# angular plot of errors, red for positive, blue for negative values
pylab.figure()
poserr = pylab.find(err > 0)
negerr = pylab.find(err < 0)
pylab.polar(angles[negerr], np.abs(err[negerr]), 'ko', \
angles[poserr], np.abs(err[poserr]), 'wo')
pylab.title('Angular dependence of fit error')
pylab.text(np.pi/2+0.3, np.abs(err).max()*0.85, \
r'$\sum_{\phi}|\sum_r\Delta_{OD}|=%1.1f$'%av_err)
if not savefig_name:
pylab.show()
else:
pylab.savefig(''.join([os.path.splitext(savefig_name)[0], '.png']))
return av_err
def GetAverageRange(X, lower_lim, domainheight):
#
# get range of X for averaging such that lower_lim<X<h0
try:
start_val = pylab.find(numpy.array(X)>lower_lim)[0]
end_val = pylab.find(numpy.array(X)>0.4-domainheight)[0]
average = True
except IndexError:
start_val = 0
end_val = 0
average = False
#
return start_val, end_val, average
def GetAverageRange(X, lower_lim, domainheight):
#
# get range of X for averaging such that lower_lim<X<h0
try:
start_val = pylab.find(numpy.array(X) > lower_lim)[0]
end_val = pylab.find(numpy.array(X) > 0.4 - domainheight)[0]
average = True
except IndexError:
start_val = 0
end_val = 0
average = False
#
return start_val, end_val, average
