def sudokuChecker(matrix):
'''
sudokuChecker takes a 9x9 sudoku and return True if it is a valid sudoku,
i.e. it meets all the rules.
Parameters
----------
matrix: array_like
`matrix` is a numpy.array that contains a 9x9 sudoku.
Returns
-------
valid : bool
If any of the numbers in `matrix` does not follow the 3 sudoku's rules,
`out` will be False. Otherwise will be True.
'''
valid = True
if len( np.hstack((find(matrix <1), find(matrix > 9))) ) != 0:
valid = False
return valid
for i in range(0,9):
for j in range(0,9):
k = matrix[i,j]
if len(find(matrix[i,:] == k)) > 1 or \
len(find(matrix[:,j] == k)) > 1 or \
len(find(mySquare(matrix,i,j) == k)) > 1:
valid = False
return valid
def getdrift_raw(filename,id3,interval,datetime_wanted):
# range_time is a number,unit by one day. datetime_wanted format is num
d=ml.load(filename)
lat1=d[:,8]
lon1=d[:,7]
idd=d[:,0]
year=[]
for n in range(len(idd)):
year.append(str(idd[n])[0:2])
h=d[:,4]
day=d[:,3]
month=d[:,2]
time1=[]
for i in range(len(idd)):
time1.append(date2num(datetime.datetime.strptime(str(int(h[i]))+' '+str(int(day[i]))+' '+str(int(month[i]))+' '+str(int(year[i])), "%H %d %m %y")))
idg1=list(ml.find(idd==id3))
idg2=list(ml.find(np.array(time1)<=datetime_wanted+interval/24))
"'0.25' means the usual Interval, It can be changed base on different drift data "
idg3=list(ml.find(np.array(time1)>=datetime_wanted-0.1))
idg23=list(set(idg2).intersection(set(idg3)))
# find which data we need
idg=list(set(idg23).intersection(set(idg1)))
print 'the length of drifter data is '+str(len(idg)),str(len(set(idg)))+' . if same, no duplicate'
lat,lon,time=[],[],[]
for x in range(len(idg)):
lat.append(round(lat1[idg[x]],4))
lon.append(round(lon1[idg[x]],4))
time.append(round(time1[idg[x]],4))
# time is num
return lat,lon,time
def relabel(complex_objs, old_tagging, majority=True):
'''flatten+label(majority or consistent)'''
val_map={}
for i,obj in enumerate(complex_objs):
tag= old_tagging[i]
for item in obj: #each entity
if not val_map.has_key(item):
val_map[item]={}
if not val_map[item].has_key(tag):
val_map[item][tag]=0
val_map[item][tag]+=1
blorf= [[a] for a,counts in val_map.items() if len(counts.values())==1 or #it is not the case that there is full equality for all tags, so it's 1-1-1 or 2-2-2
sum(counts.values()) % len(counts.values()) != 0]
items= array(blorf, dtype=object)
tags=[]
for i,item in enumerate(items):
label_counts=val_map[item[0]]
#label counts is mapping of tag->apperances
if majority:
tags.append(label_counts.keys()[argmax(label_counts.values())])
else:
vals=array(label_counts.values())
inds= find(vals>0)
if len(inds) == 1: #exactly one class
tags.append(array(label_counts.keys())[inds])
else:
tags.append(-1)
tags=array(tags)
if majority:
return items,tags
idxs=find(tags>=0)
return items[idxs], tags[idxs]
def extr(x):
"""Extract the indices of the extrema and zero crossings.
:param x: input signal
:type x: array-like
:return: indices of minima, maxima and zero crossings.
:rtype: tuple
"""
m = x.shape[0]
x1 = x[:m - 1]
x2 = x[1:m]
indzer = find(x1 * x2 < 0)
if np.any(x == 0):
iz = find(x == 0)
indz = []
if np.any(np.diff(iz) == 1):
zer = x == 0
dz = np.diff([0, zer, 0])
debz = find(dz == 1)
finz = find(dz == -1) - 1
indz = np.round((debz + finz) / 2)
else:
indz = iz
indzer = np.sort(np.hstack([indzer, indz]))
indmax = argrelmax(x)[0]
indmin = argrelmin(x)[0]
return indmin, indmax, indzer
def __get_level_index_from_chan_slot__(slotnum, channel):
""" given slot and channel, returns corresponding the level|keep column index """
global slot
global __boardTypes__
# 1. determine board type from slot
for boardname in slot.keys():
if slotnum in slot[boardname]:
# 1a. for driver channels, multiply the chan number by 2
# to get the UniqueStateArr index
if boardname == 'drvr':
channel *= 2
# 1b. check that channel is valid for board type.
if channel >= __chan_per_board__[boardname]:
print "*** INVALID channel (%d) specified for slot (%d,%s) ***"%(channel,slotnum,boardname)
return -1
# 2. determine index in global slot for board type (0 for the first board of that type, etc)
indx_slot = mlab.find(np.array(slot[boardname]) == slotnum)[0]
# 3. calculate the base index (boardname index * slot index)
signalPartitions = np.cumsum([ 0,
__chan_per_board__['drvr'] * len(slot['drvr']), ## !driver-speed-keep
__chan_per_board__['lvds'] * len(slot['lvds']),
__chan_per_board__['adc'] * len(slot['adc']),
__chan_per_board__['back'] * len(slot['back']),
__chan_per_board__['hvbd'] * len(slot['hvbd']) ])
indx_LVL_boardname = mlab.find(np.array(__boardTypes__) == boardname)[0]
indx_base = signalPartitions[indx_LVL_boardname] + indx_slot * __chan_per_board__[boardname]
# 4. add the channel offset
return (indx_base + channel)
def cb_active_plot(self,start_ms,stop_ms,line_color='b'):
"""
Plot timing information of time spent in the callback. This is similar
to what a logic analyzer provides when probing an interrupt.
cb_active_plot( start_ms,stop_ms,line_color='b')
"""
# Find bounding k values that contain the [start_ms,stop_ms]
k_min_idx = mlab.find(np.array(self.DSP_tic)*1000 < start_ms)
if len(k_min_idx) < 1:
k_min = 0
else:
k_min = k_min_idx[-1]
k_max_idx = mlab.find(np.array(self.DSP_tic)*1000 > stop_ms)
if len(k_min_idx) < 1:
k_max= len(self.DSP_tic)
else:
k_max = k_max_idx[0]
for k in range(k_min,k_max):
if k == 0:
plt.plot([0,self.DSP_tic[k]*1000,self.DSP_tic[k]*1000,
self.DSP_toc[k]*1000,self.DSP_toc[k]*1000],
[0,0,1,1,0],'b')
else:
plt.plot([self.DSP_toc[k-1]*1000,self.DSP_tic[k]*1000,self.DSP_tic[k]*1000,
self.DSP_toc[k]*1000,self.DSP_toc[k]*1000],[0,0,1,1,0],'b')
plt.plot([self.DSP_toc[k_max-1]*1000,stop_ms],[0,0],'b')
plt.xlim([start_ms,stop_ms])
plt.title(r'Time Spent in the callback')
plt.ylabel(r'Timing')
plt.xlabel(r'Time (ms)')
plt.grid();
def dct_cut_downsampled(data,cutoff,spacing=0.4):
'''
like dctCut but also lowers the sampling rate, creating a compact
representation from which the whole downsampled data could be
recovered
'''
h,w = np.shape(data)[:2]
f1 = fftfreq(h*2,spacing)
f2 = fftfreq(w*2,spacing)
wl = 1./abs(reshape(f1,(2*h,1))+1j*reshape(f2,(1,2*w)))
mask = int32(wl>=cutoff)
mirror = reflect2D(data)
ff = fft2(mirror,axes=(0,1))
cut = (ff.T*mask.T).T # weird shape broadcasting constraints
empty_cols = find(all(mask==0,0))
empty_rows = find(all(mask==0,1))
delete_col = len(empty_cols)/2 #idiv important here
delete_row = len(empty_rows)/2 #idiv important here
keep_cols = w-delete_col
keep_rows = h-delete_row
col_mask = np.zeros(w*2)
col_mask[:keep_cols] =1
col_mask[-keep_cols:]=1
col_mask = col_mask==1
row_mask = np.zeros(h*2)
row_mask[:keep_rows] =1
row_mask[-keep_rows:]=1
row_mask = row_mask==1
cut = cut[row_mask,...][:,col_mask,...]
w,h = keep_cols,keep_rows
result = ifft2(cut,axes=(0,1))[:h,:w,...]
return result
def streaks(rats):
""" Okay, I want to find consecutive hits during uncued blocks
So, what I'll do first is grab the
"""
d_list = []
for rat in rats.itervalues():
trials = rat.sessions['trials']
uncued = [ find(trial['block'] == 'uncued') for trial in trials ]
cued = [ find(trial['block'] == 'cued') for trial in trials ]
un_blk_lens = [ np.diff(session)-1 for session in cued ]
un_blk_lens = np.array([ sess[find(sess != 0)] for sess in uncued_lens ])
un_hits = [ trial['hits'][uncued[ii]] for ii, trial in enumerate(trials) ]
un_hits = [ find(hts == 1) for hts in un_hits ]
# Grab the first block of uncued trials
un_blk_lens
d_list.append(diffs)
return d_list
def find_spinning(mag,gyros): """ return the indicies in the magnetometer data when the gyro indicates it is spinning on the z axis """ import numpy import scipy.signal from matplotlib.mlab import find threshold = 40 spinning = scipy.signal.medfilt(abs(gyros['z'][:,0]),kernel_size=5) > threshold # make sure to always find end elements spinning = numpy.concatenate((numpy.array([False]),spinning,numpy.array([False]))) start = find(spinning[1:] & ~spinning[0:-1]) stop = find(~spinning[1:] & spinning[0:-1])-1 tstart = gyros['time'][start] tstop = gyros['time'][stop] idx = numpy.zeros((0),dtype=int) for i in arange(tstart.size): i1 = abs(mag['time']-tstart[i]).argmin() i2 = abs(mag['time']-tstop[i]).argmin() idx = numpy.concatenate((idx,arange(i1,i2,dtype=int))) return idx
def make_batches(data, batch_size=100):
"""
Split the data into minibatches of size batch_size
This procedure generates subsamples ids for batches by only considering
features that are active in the minibatch
Args:
data - the data to be split into minibatches (must be rank 2)
batch_size - the size of the minibatches
Returns:
batches - a list: [(batch, subsample_ids) for batch in minibatchs]
"""
n = data.shape[0]
perm = random.permutation(range(n))
i = 0
batches = []
while i < n:
batch = perm[i:i+batch_size]
i += batch_size
batches.append(data[batch])
try:
ids = [find((b.sum(0) != 0).A.flatten()) for b in batches]
except AttributeError:
ids = [find((b.sum(0) != 0).flatten()) for b in batches]
batches = [(b[:,i].toarray(), i) for b,i in zip(batches, ids)]
return batches
def Extract_Range(self, begin, end):
#Extract a date range and use it to make a new time series.
try:
datetime.datetime.now() > begin
except TypeError:
begin = parser.parse(begin)
try:
datetime.datetime.now() > end
except TypeError:
end = parser.parse(end)
#Find which rows of timestamps lie in this range.
upper_rows = mlab.find( self.timestamps >= begin )
lower_rows = mlab.find( self.timestamps <= end )
rows = filter( lambda x: x in upper_rows, lower_rows )
#Now create a time series with the data in this range.
range = {}
range['headers'] = self.headers
range['timestamps'] = self.timestamps[rows]
range['data'] = self.data[rows,:]
return Series(series=range)
def FWHM(X,Y):
half_max = (max(Y)+min(Y)) / 2
d = np.sign(half_max - np.array(Y[0:-1])) - np.sign(half_max - np.array(Y[1:]))
#find the left and right most indexes
left_idx = find(d > 0)[0]
right_idx = find(d < 0)[-1]
return X[right_idx], X[left_idx], half_max #return the difference (full width)
def add_trials_from_file(self, filename):
if self.span_type=='period' and filename:
bci_stream=FileReader.bcistream(filename)
sig,states=bci_stream.decode(nsamp='all')
sig,chan_labels=bci_stream.spatialfilteredsig(sig)
erpwin = [int(bci_stream.msec2samples(ww)) for ww in bci_stream.params['ERPWindow']]
x_vec = np.arange(bci_stream.params['ERPWindow'][0],bci_stream.params['ERPWindow'][1],1000/bci_stream.samplingfreq_hz,dtype=float)
trigchan = bci_stream.params['TriggerInputChan']
trigchan_ix = find(trigchan[0] in chan_labels)
trigthresh = bci_stream.params['TriggerThreshold']
trigdetect = find(np.diff(np.asmatrix(sig[trigchan_ix,:]>trigthresh,dtype='int16'))>0)+1
intensity_detail_name = 'dat_TMS_powerA' if self.detail_values.has_key('dat_TMS_powerA') else 'dat_Nerve_stim_output'
#Get approximate data segments for each trial
trig_ix = find(np.diff(states['Trigger'])>0)+1
for i in np.arange(len(trigdetect)):
ix = trigdetect[i]
dat = sig[:,ix+erpwin[0]:ix+erpwin[1]]
self.trials.append(Datum(subject_id=self.subject_id\
, datum_type_id=self.datum_type_id\
, span_type='trial'\
, parent_datum_id=self.datum_id\
, IsGood=1, Number=0))
my_trial=self.trials[-1]
my_trial.detail_values[intensity_detail_name]=str(states['StimulatorIntensity'][0,trig_ix[i]])
if int(bci_stream.params['ExperimentType']) == 1:#SICI intensity
my_trial.detail_values['dat_TMS_powerB']=str(bci_stream.params['StimIntensityB'])#TODO: Use the state.
my_trial.detail_values['dat_TMS_ISI']=str(bci_stream.params['PulseInterval'])
my_trial.store={'x_vec':x_vec, 'data':dat, 'channel_labels': chan_labels}
Session.commit()
def split_and_subtree(query_chosen, recursive_step_obj):
query_results=array([query_chosen(x) for x in recursive_step_obj.objects])
pos_inds=find(query_results==1)
neg_inds=find(query_results!=1)
recursive_step_obj.left_son= TreeRecursiveSRLStep(recursive_step_obj.objects[neg_inds], recursive_step_obj.tagging[neg_inds], recursive_step_obj.relations, recursive_step_obj.transforms, recursive_step_obj.n, recursive_step_obj.MAX_DEPTH, recursive_step_obj.SPLIT_THRESH, recursive_step_obj.cond)
recursive_step_obj.right_son=TreeRecursiveSRLStep(recursive_step_obj.objects[pos_inds], recursive_step_obj.tagging[pos_inds], recursive_step_obj.relations, recursive_step_obj.transforms, recursive_step_obj.n, recursive_step_obj.MAX_DEPTH, recursive_step_obj.SPLIT_THRESH, recursive_step_obj.cond)
return query_chosen,recursive_step_obj.left_son,recursive_step_obj.right_son
def reject_outliers(x,percent=10,side='both'):
'''
Reject outliers from data based on percentiles.
Parameters
----------
x : ndarary
1D numeric array of data values
percent : number
percent between 0 and 100 to remove
side : str
'left' 'right' or 'both'. Default is 'both'. Remove extreme
values from the left / right / both sides of the data
distribution. If both, the percent is halved and removed
from both the left and the right
Returns
-------
ndarray
Values with outliers removed
kept
Indecies of values kept
removed
Indecies of values removed
'''
N = len(x)
remove = outliers(x,percent,side)
to_remove = find(remove==True)
to_keep = find(remove==False)
return x[to_keep], to_keep, to_remove
def durations():
import os
import re
from matplotlib.mlab import find
save_file = 'durations.pkl'
save_dir = '/home/mat/Dropbox/Working-memory/Metadata/'
# I don't like regex...
datadir = '/media/hippocampus/NDAQ'
filelist = os.listdir(datadir)
ML = find(['ML' in filename for filename in filelist])
data = [ filelist[ind] for ind in ML ]
# Now I have a list of the filenames for all of my data
# But, I only want the ns5 data, so do the same thing again.
ns5 = find(['.ns5' in filename for filename in data])
data = [ data[ind] for ind in ns5 ]
# Let's check if we've already done this for some data so we
# don't do it twice.
check_files = os.listdir(save_dir)
if save_file in check_files:
with open(save_file,'r') as f:
info_list = pkl.load(f)
for file in data:
reg=re.search('datafile_ML_(\w+)_(\w+)_001.ns5', file)
rat = reg.group(1)
date = reg.group(2)
for info in info_list:
if (info['rat'] == rat) & (info['date'] == date):
info_list.remove(info)
else:
info_list = []
for file in data:
filepath = os.path.join(datadir,file)
print "Loading %s" % file
loader = ns5.loader(filepath)
loader.load_header()
print "File %s loaded" % file
samples = loader.header.n_samples
sample_rate = 30000
duration = samples/float(sample_rate)
print 'Duration is %s' % duration
reg=re.search('datafile_ML_(\w+)_(\w+)_001.ns5', file)
rat = reg.group(1)
date = reg.group(2)
info = {'rat':rat, 'date':date, 'duration':duration}
info_list.append(info)
with open(''.join([save_dir,save_file]), 'w') as f:
pkl.dump(info_list, f)
def get_valid_trials(session, area=None):
if dowarn():
print("TRIAL IS 1 INDEXED FOR MATLAB COMPATIBILITY")
if area is None:
trials = set(find(metaloadvariable(session, "M1", "eventsByTrial")[:, 0]) + 1)
trials &= set(find(metaloadvariable(session, "PMv", "eventsByTrial")[:, 0]) + 1)
trials &= set(find(metaloadvariable(session, "PMd", "eventsByTrial")[:, 0]) + 1)
return np.array(list(trials))
else:
return np.array(find(1 == metaloadvariable(session, area, "eventsByTrial")[:, 0]))
def tight(I): B = I>0 if I.ndim==2: ix = B.any(1); ix0,ix1 = min(find(ix)) , max(find(ix))+1 iy = B.any(0); iy0,iy1 = min(find(iy)) , max(find(iy))+1 return I[ix0:ix1,iy0:iy1] elif I.ndim==3: ix = B.max(2).any(1); ix0,ix1 = min(find(ix)) , max(find(ix))+1 iy = B.max(2).any(0); iy0,iy1 = min(find(iy)) , max(find(iy))+1 iz = B.max(0).any(0); iz0,iz1 = min(find(iz)) , max(find(iz))+1 return I[ix0:ix1,iy0:iy1,iz0:iz1]
def fortgaugeread (datafile="fort.gauge",setgaugefile="setgauges.data"):
"""
Read data from fort.gauge files output by GeoClaw.
Reads the gauge data and returns a list with mgauge elements.
Each element of the list is a dictionary containing the data for a particular gauge.
example:
for N gauges: allgaugedata=[dictionary_1,...,dictionary_N] where
dictionary_n.keys() = gauge#,x,y,level,t,q1,...qmq+1.
where level,t,q1...q mq+1 are numpy arrays. 'q mq+1' is the surface elevation.
"""
fid=open(setgaugefile)
inp='#'
while inp == '#':
inpl=fid.readline()
inp=inpl[0]
inp = fid.readline()
mgauges=int(inp.split()[0])
gaugelocs=[]
linesread=0
while linesread < mgauges :
row=string.split(fid.readline())
if row!=[]:
gaugelocs.append(row)
linesread=linesread+1
fid.close()
data=loadtxt(datafile)
allgaugedata=[]
for n in xrange(mgauges) :
onegaugedata=data[mlab.find(data[:,0]==int(gaugelocs[n][0]))]
dict={}
dict['gauge']=int(gaugelocs[n][0])
dict['x']=float(gaugelocs[n][1])
dict['y']=float(gaugelocs[n][2])
onegaugedata = data[mlab.find(data[:,0]==dict['gauge'])]
dict['level']=onegaugedata[:,1]
dict['t']=onegaugedata[:,2]
dict['mq'] = len(onegaugedata[0])-3
for m in xrange(dict['mq']) :
dict['q'+str(m+1)]=onegaugedata[:,3 + m]
allgaugedata.append(dict)
return allgaugedata
def CheckElevation(vec):
#print vec
vec_diff = np.diff(np.array(vec))
elChange = vec[-1] - vec[1];
if (elChange > ELEVATION_THRESHOLD):
#print 'DOWN'
if (mlab.find(vec_diff >= 0).size > VECTOR_SIZE*ELEVATION_TREND_PRECENT):
return elChange,HAND_DOWN
elif (elChange < -ELEVATION_THRESHOLD):
#print 'UP'
if (mlab.find(vec_diff <= 0).size > VECTOR_SIZE*ELEVATION_TREND_PRECENT):
return elChange,HAND_UP
return 0,HAND_UNKWON
def jump_frequencies(s,t,valid): split_ds = 5e3 split_dt = 1e-3 ds = np.diff(s)/split_ds dt = np.diff(t)/split_dt #compute distance of jump from origin dr = np.sqrt(np.power(ds,2) + np.power(dt,2)) window = np.ones(t.size-1) #Discard points within 10% of edge window[0:0.1*t.size] = 0 window[0.9*t.size:t.size] = 0 dr = window*dr l = label(dr<1) #jump points have dr<1 index = find(l[0]==0) #indices of jump points #remove points whose indices are within 5 places of each other remove = np.sort(np.append(find(np.diff(index)<5), find(np.diff(index)<5)+1)) index = np.delete(index, remove) remove = np.array([]) for i in range(0,index.size): if dr[index[i]]<2: remove = np.append(remove,i) remove.dtype = int index = np.delete(index,remove) jumps = np.zeros((100,4)) for i in range(0,len(index)): #This if statement checks several conditions. If any of them are #satisified the jump point is rejected. 1) Is jump point in high #entropy region, 2) Is the before or the after jump frequency to #high or too low, 3) Is the the difference between before and after #frequencies too high or too low if (find(valid[index[i]-15:index[i]+15]==0)!=[] or s[index[i]]<15e3 or s[index[i]+1]<15e3 or s[index[i]]>80e3 or s[index[i]+1]>80e3 or abs(s[index[i]]-s[index[i]+1])<7e3 or abs(s[index[i]+1]-s[index[i]])>25e3): continue else: #not sure what this does but leave it there if i != len(index)-1 and abs(index[i+1]-index[i])<10: continue if i!=0 and abs(index[i-1]-index[i])<10: continue jumps[i,:] = np.array([s[index[i]], s[index[i]+1], t[index[i]], t[index[i]+1]]) jumps = np.delete(jumps,find(jumps==0)) jumps.shape = (len(jumps)/4, 4) return jumps
def drop_disjoint(train, test):
"""
Removes all features from train and test that are strictly all 0 in either the train
set or test set
train, and test are scipy.sparse CSR matricies
"""
in_training = find(train.sum(0).A.flatten() != 0)
train = train[:, in_training]
test = test[:, in_training]
in_test = find(test.sum(0).A.flatten() != 0)
train = train[:, in_test]
test = test[:, in_test]
return sparse.vstack((train, test)).tocsr()
def calc_stats(predicted, actual):
'''accuracy,precision,recall,F1'''
sums_of_things= [0.0,0.0,0.0]
for cat in frozenset(actual):
pos_idxs= find(predicted==cat)
tp= len(find(predicted[pos_idxs]==actual[pos_idxs]))
tp_fp= len(pos_idxs)
tp_fn= len(find(actual==cat))
#print tp, tp_fp, tp_fn, cat, predicted, mean(predicted!=actual)
sums_of_things[0]+= tp*1.0/tp_fp if tp_fp>0 else 0.0
sums_of_things[1]+= tp*1.0/tp_fn
sums_of_things[2]+= tp*2.0/(tp_fp+tp_fn)
means_of_things= [x/len(frozenset(actual)) for x in sums_of_things]
return array([mean(predicted==actual),means_of_things[0],means_of_things[1],means_of_things[2]])
def Autocorrelation(data): corr = fftconvolve(data, data[::-1], mode='full') # 傅立叶卷积 corr = corr[len(corr)/2:] # 对称取一半 d = np.diff(corr) # 相关性变化情况 if len(find(d > 0)) <= 0: return (-1, -1, None, None) start = find(d > 0)[0] peak = np.argmax(corr[start:]) + start # 相关性最大的位移 frequencey, py = parabolic(corr, peak) # 加权平均位移 strength = np.max(d) # 变化最大值: 强度 return (int(frequencey), strength, corr, d)
def _error_check_onsets(self, sound_bool):
# Find when the threshhold crossings first happen. `onsets` and
# `offsets` are inclusive bounds on audio power above threshhold.
onsets = mlab.find(np.diff(np.where(sound_bool, 1, 0)) == 1) + 1
offsets = mlab.find(np.diff(np.where(sound_bool, 1, 0)) == -1)
# check that we don't start or end in the middle of a sound
try:
if onsets[0] > offsets[0]:
# Extra offset at the beginning
offsets = offsets[1:]
if onsets[-1] > offsets[-1]:
# Extra onset at the end
onsets = onsets[:-1]
except IndexError:
# apparently no onsets or no offsets
print "No sounds found!"
onsets = np.array([])
offsets = np.array([])
if len(onsets) > 0:
# First do some error checking.
assert (len(onsets) == len(offsets)) and (np.all(onsets <= offsets))
# Remove sounds that violate min_duration requirement.
too_short_sounds = ((offsets - onsets) < self._minimum_duration_samples)
if np.any(too_short_sounds):
if self.verbose:
print "Removing %d sounds that violate duration requirement" % \
len(mlab.find(too_short_sounds))
onsets = onsets[np.logical_not(too_short_sounds)]
offsets = offsets[np.logical_not(too_short_sounds)]
# Warn when onsets occur very close together. This might occur if the
# sound power briefly drops below threshhold.
if np.any(np.diff(onsets) < self._minimum_duration_samples):
print "WARNING: %d onsets were suspiciously close together." % \
len(find(np.diff(onsets) < self._minimum_duration_samples))
# Print the total number of sounds identified.
if self.verbose:
print "Identified %d sounds with average duration %0.3fs" % \
(len(onsets), (offsets-onsets).mean() / self.F_SAMP)
# Store detected onsets
self.detected_onsets = onsets
self.detected_offsets = offsets
def knn(training,label,sample,k):
dit = {}
#----------检查输入数据格式是否正确------------------
r_t,c_t = numpy.shape(training)
r_s,c_s = numpy.shape(sample)
label_shape = numpy.shape(label)
if len(label_shape) >1:
raise Exception #标签数组有多列，报错
else:
label_list = []
label_num = [] #分析标签数组，初始化分类结果字典
for n,l in enumerate(label):
if l not in label[0:n]:
label_list.append(l)
label_num.append(len(mlab.find(label[n+1:] == l)))
class_num = len(label_list) #class_num类问题
for c in range(class_num-1): #检查样本数据是否均衡，不均衡的训练样本会造成结果的偏差
if label_num[c] != label_num[c+1]:
print "WARNING: 样本数据不均衡\n"
print 'class NO.1 amount: '+str(label_num[c])
print 'class NO.2 amount: '+str(label_num[c+1])
break
if c_t != c_s or r_t != label_shape[0]: #样本数据和训练集数据列长不一致，或标签数与训练样本数不一致，报错
raise Exception
#----------------end--------------------------------
#----------------实现算法--------------------------
else:
training = numpy.array(training) #格式
sample = numpy.array(sample)
label = numpy.array(label)
result_list = []
for s in sample: #计算欧式距离，得到的数值实际上是距离的平方
label_dict = dit.fromkeys(label_list,0)
distance1 = (s-training)**2 #dictance存放每个样本点到训练集之间的距离
distance = [sum(temp) for temp in distance1]
#print distance
distance_label = zip(distance,label) #dictance_label存放每个样本点到训练集之间的距离与标签
#print distance_label
result_temp = []
distance_sort = numpy.sort(distance)[0:k]
for d in distance_sort:
index = mlab.find(d == numpy.array(distance_label)[:,0])
target_label = distance_label[index[0]][1]
label_dict[target_label] += 1
for key in label_list: #归一化
result_temp.append(round(label_dict[key]/float(k),3))
result_list.append(result_temp)
return result_list,label_list
def CheckElevation(vec):
#print vec
step = 1.0/len(vec)
vec_after_polyfit = createElevationVectorAfterPolyfit(mlab.frange(0,1-step,step),vec)
vec_diff = np.diff(np.array(vec_after_polyfit))
elChange = vec_after_polyfit[-1] - vec_after_polyfit[1];
if (elChange > ELEVATION_THRESHOLD):
#print 'DOWN'
if (mlab.find(vec_diff >= 0).size > VECTOR_SIZE*ELEVATION_TREND_PRECENT):
return elChange,HAND_DOWN
elif (elChange < -ELEVATION_THRESHOLD):
#print 'UP'
if (mlab.find(vec_diff <= 0).size > VECTOR_SIZE*ELEVATION_TREND_PRECENT):
return elChange,HAND_UP
return 0,HAND_UNKWON
def Partition_Data(data, headers, year):
"""Partition the supplied data set into training and validation sets"""
# model_data is the set of observations that we'll use to train the model.
rows_year = mlab.find(data[:, headers.index("year")] >= year)
model_data = np.delete(data, rows_year, axis=0)
# validation_data is the set of observations we'll use to test the model's predictive ability.
rows_year = mlab.find(data[:, headers.index("year")] == year)
validation_data = data[rows_year, :]
model_dict = dict(zip(headers, np.transpose(model_data)))
validation_dict = dict(zip(headers, np.transpose(validation_data)))
return [model_data, validation_data]
def Cross_Validation(self, cv_method=0, **args):
# select ncomp by the requested CV method
validation = self.Extract("validation")
# method 0: select the fewest components with PRESS within 1 stdev of the least PRESS (by the bootstrap)
if cv_method == 0: # Use the bootstrap to find the standard deviation of the MSEP
cv = np.array(self.Extract("pred", extract_from=validation)).squeeze()
PRESS = map(lambda x: sum((cv[:, x] - self.actual) ** 2), range(cv.shape[1]))
ncomp = np.argmin(PRESS)
cv_squared_error = (cv[:, ncomp] - self.actual) ** 2
sample_space = xrange(cv.shape[0])
PRESS_stdev = list()
# Cache random number generator and int's constructor for a speed boost
_random, _int = random.random, int
for i in np.arange(100):
PRESS_bootstrap = list()
for j in np.arange(100):
PRESS_bootstrap.append(sum([cv_squared_error[_int(_random() * cv.shape[0])] for i in sample_space]))
PRESS_stdev.append(np.std(PRESS_bootstrap))
med_stdev = np.median(PRESS_stdev)
# Maximum allowable PRESS is the minimum plus one standard deviation
good_ncomp = mlab.find(PRESS < min(PRESS) + med_stdev)
self.ncomp = np.min(good_ncomp) + 1
# method 1: select the fewest components w/ PRESS less than the minimum plus a 4% of the range
if cv_method == 1:
# PRESS stands for predicted error sum of squares
PRESS0 = list(self.Extract("PRESS0", extract_from=validation))
PRESS = list(self.Extract("PRESS", extract_from=validation))
# the range is the difference between the greatest and least PRESS values
PRESS_range = abs(PRESS0 - np.min(PRESS))
# Maximum allowable PRESS is the minimum plus a fraction of the range.
max_CV_error = np.min(PRESS) + PRESS_range / 25
good_ncomp = mlab.find(PRESS < max_CV_error)
# choose the most parsimonious model that satisfies that criterion
self.ncomp = np.min(good_ncomp) + 1
def solve_multiclass(trn, trn_lbl, lbl_vals, tst, tst_lbl, relations, d=0):
clfs=[]
for i,lbl in enumerate(lbl_vals):
for j,lbl2 in enumerate(lbl_vals):
if j<=i:
continue
idxs= find(any(vstack((trn_lbl==lbl,trn_lbl==lbl2)),axis=0))
new_trn= trn[idxs]
new_lbl= ((trn_lbl[idxs]-lbl2)/(lbl-lbl2)).astype(int) #binarize
clf= alg7.TreeRecursiveSRLClassifier(new_trn, new_lbl, relations, [], 100*(d**2), d, 3)
clf.train()
clfs.append((clf, lbl, lbl2))
def predict_label(x, descriptors=clfs):
mappify={}
mappify.update([t[::-1] for t in enumerate(lbl_vals)]) #switch i,val for val,i
votes= zeros(len(lbl_vals))
for clf,lbl1,lbl2 in clfs:
label= clf.predict(x)
if label==1:
votes[mappify[lbl1]]+=1
else:
votes[mappify[lbl2]]+=1
return lbl_vals[argmax(votes)]
tst_predict= array([predict_label(x) for x in tst])
err= mean(tst_predict!=tst_lbl)
return err, tst_predict, clfs, predict_label
def hereRelabelClustersByDistance(labels, D, th=20):
nLabels = np.unique(labels).size - 1
L = deepcopy(labels)
#### find close clusters:
D = D < th
for i in range(nLabels - 1, 0, -1):
ind = find(D[:i, i])
if len(ind) > 0:
L[labels == (i + 1)] = (ind[0] + 1)
return L
def getdrift_raw(filename, id3, interval, datetime_wanted):
# range_time is a number,unit by one day. datetime_wanted format is num
d = np.genfromtxt(filename)
lat1 = d[:, 8]
lon1 = d[:, 7]
idd = d[:, 0]
year = []
for n in range(len(idd)):
year.append(str(idd[n])[0:2])
h = d[:, 4]
day = d[:, 3]
month = d[:, 2]
time1 = []
for i in range(len(idd)):
time1.append(
date2num(
datetime.datetime.strptime(
str(int(h[i])) + ' ' + str(int(day[i])) + ' ' +
str(int(month[i])) + ' ' + str(int(year[i])),
"%H %d %m %y")))
idg1 = list(ml.find(idd == id3))
idg2 = list(ml.find(np.array(time1) <= datetime_wanted + interval / 24))
"'0.25' means the usual Interval, It can be changed base on different drift data "
idg3 = list(ml.find(np.array(time1) >= datetime_wanted - 0.1))
idg23 = list(set(idg2).intersection(set(idg3)))
# find which data we need
idg = list(set(idg23).intersection(set(idg1)))
print 'the length of drifter data is ' + str(len(idg)), str(len(
set(idg))) + ' . if same, no duplicate'
lat, lon, time = [], [], []
for x in range(len(idg)):
lat.append(round(lat1[idg[x]], 4))
lon.append(round(lon1[idg[x]], 4))
time.append(round(time1[idg[x]], 4))
drifter_data = {}
drifter_data['lat'] = lat
drifter_data['lon'] = lon
drifter_data['time'] = time
# time is num
return drifter_data
def zero_cross(time, sig_actual, sig_filter):
''' Freq. from Zero Crossing Estimate '''
t, sig = np.transpose(time), np.transpose(sig_filter)
indices = find((sig[1:] >= 0) & (sig[:-1] < 0))
tf = t[indices]
f = 1 / np.diff(tf, axis=0)
tz = np.zeros((len(tf), 1))
return tz, tf, f
def Pitch(signal):
signal = np.fromstring(signal, 'Int16')
maXabs = sum(abs(i) for i in signal) / float(len(signal))
if maXabs > MinDetect:
crossing = [math.copysign(1.0, s) for s in signal]
index = find(np.diff(crossing))
f0 = round(len(index) * RATE / (2 * np.prod(len(signal))))
return f0, maXabs
else:
return 0, 0
def freq_from_crossings(sig, fs): indices = find((sig[1:] >= 0) & (sig[:-1] < 0)) crossings = [i - sig[i] / (sig[i+1] - sig[i]) for i in indices] diffc = diff(crossings) # BUG: np.average does not like empty lists if len(diffc) > 0: return 1.0 * fs / average(diff(crossings)) else: return 0
def get_neracoos_temp_data(
url, id_s, id_e,
id_max_url): #get temperature and salinity data from neracoos.
url1 = url + 'depth[0:1:0],time[' + id_s + ':1:' + id_e + '],temperature[' + id_s + ':1:' + id_e + '][0:1:0][0:1:0][0:1:0],salinity[' + id_s + ':1:' + id_e + '][0:1:0][0:1:0][0:1:0]'
#database=open_url(url1)['temperature'][int(id_s):int(id_e)]
#database_s=open_url(url1)['salinity'][int(id_s):int(id_e)]
database = netCDF4.Dataset(url1)
database_time = database.variables['time']
database_s = database.variables['salinity']
database_t = database.variables['temperature']
database_depth = database.variables['depth']
#salinity=database_s['salinity']
salinity = database_s[0:].tolist()
#lat=database['lat']
#lat=round(lat[0],2)
#lon=database['lon']
#lon=round(lon[0],2)
depth = database_depth
period = database_time
#temp=database['temperature']
temp = database_t[0:].tolist()
period = num2date(period[0:] + date2num(dt.datetime(1858, 11, 17, 0, 0)))
period_str, depth_temp = [], []
for i in range(len(period)): #convert format to list
period_str.append(dt.datetime.strftime(period[i], '%Y-%m-%d-%H-%M'))
#period_str.append(period[i])
depth_temp.append([
round(depth[0], 1),
round(temp[i][0][0][0], 2),
round(salinity[i][0][0][0], 2)
])
temp1, salinity1 = [], [] #get rid of bad data
for i in range(len(depth_temp)):
temp1.append(depth_temp[i][1])
salinity1.append(depth_temp[i][2])
id_bad = ml.find((np.array(temp1) > 30) | (np.array(temp1) < -4)
| (np.array(salinity1) > 37) | (np.array(salinity1) < 25))
#print id_bad
id_bad = list(id_bad)
id_bad.reverse()
for m in id_bad:
del period_str[m]
del depth_temp[m]
for i in range(len(depth_temp)):
if i >= len(depth_temp):
break
if abs(depth_temp[i - 1][1] - depth_temp[i][1]) > 2:
del depth_temp[i]
del period_str[i]
depth_temp = depth_temp
period_str = period_str
i = i - 1
#df=DataFrame(np.array(depth_temp),index=period_str,columns=['depth','temp','lat','lon'])
return depth_temp, period_str
def get_neracoos_current_data(
url, id_s, id_e, id_max_url): #get surface current data from neracoos.
#url1=url+'current_speed[0:1:'+id_max_url+'][0:1:0][0:1:0][0:1:0],current_direction[0:1:'+id_max_url+'][0:1:0][0:1:0][0:1:0],current_u[0:1:'+id_max_url+'][0:1:0][0:1:0][0:1:0],current_v[0:1:'+id_max_url+'][0:1:0][0:1:0][0:1:0]'
url1 = url + 'time[' + id_s + ':1:' + id_e + '],current_speed[' + id_s + ':1:' + id_e + '][0:1:0][0:1:0][0:1:0],current_direction[' + id_s + ':1:' + id_e + '][0:1:0][0:1:0][0:1:0],current_u[' + id_s + ':1:' + id_e + '][0:1:0][0:1:0][0:1:0],current_v[' + id_s + ':1:' + id_e + '][0:1:0][0:1:0][0:1:0]'
database = netCDF4.Dataset(url1)
database_s = database.variables['current_speed']
database_d = database.variables['current_direction']
database_u = database.variables['current_u']
database_v = database.variables['current_v']
period = database.variables['time']
#database_s=open_url(url1)['current_speed'][int(id_s):int(id_e)]
#database_d=open_url(url1)['current_direction'][int(id_s):int(id_e)]
#database_u=open_url(url1)['current_u'][int(id_s):int(id_e)]
#database_v=open_url(url1)['current_v'][int(id_s):int(id_e)]
#lat=database_s['lat']
#lat=round(lat[0],2)
#lon=database_s['lon']
#lon=round(lon[0],2)
#period=database_s['time']
#speed=database_s['current_speed']
speed = database_s[0:].tolist()
period = num2date(period[0:] + date2num(dt.datetime(1858, 11, 17, 0, 0)))
#direction=database_d['current_direction']
direction = database_d[0:].tolist()
#u=database_u['current_u']
u = database_u[0:].tolist()
#v=database_v['current_v']
v = database_v[0:].tolist()
period_str, current_all = [], []
for i in range(len(period)): #convert format to list
period_str.append(dt.datetime.strftime(period[i], '%Y-%m-%d-%H-%M'))
current_all.append([
round(speed[i][0][0][0], 2),
round(direction[i][0][0][0], 2),
round(u[i][0][0][0], 2),
round(v[i][0][0][0], 2)
])
current, u, v, direction = [], [], [], [] # figure out bad data and delete
for i in range(len(current_all)):
current.append(current_all[i][0])
direction.append(current_all[i][1])
u.append(current_all[i][2])
v.append(current_all[i][3])
id_bad = ml.find((np.array(current) > 200) | (np.array(current) < -1)
| (np.array(direction) < 0) | (np.array(direction) > 360)
| (np.array(u) < -200) | (np.array(u) > 200)
| (np.array(v) < -200) | (np.array(v) > 200))
#print id_bad
id_bad = list(id_bad)
id_bad.reverse()
for m in id_bad:
del period_str[m]
del current_all[m]
return period_str, current_all
def estimateFrequency_zeroCross(signal, frameRate):
# Find all indices before a rising-edge zero crossing
indices = find((signal[1:] >= 0) & (signal[:-1] < 0))
# Use linear interpolation to find zero-crossings inbetween samples, more accurate
crosses = [i - signal[i] / (signal[i+1] - signal[i]) for i in indices]
return frameRate/mean(diff(crosses))
def range_latlon(filename,
driftnumber): # function neede in case of "raw" drifter date
d = ml.load(filename)
id = ml.find(d[:, 0] == int(driftnumber))
lat1 = d[id, 8]
lon1 = d[id, 7]
maxlon = max(lon1)
minlon = min(lon1)
maxlat = max(lat1)
minlat = min(lat1)
return maxlon, minlon, maxlat, minlat, lat1, lon1
def bary_eval(yy, xs, lam, x):
"""
Baryzentrische Interpolations Formel
Keyword Arguments:
yy -- f_i
xs -- x_i
lam -- lambda_i
x -- evaluation points
"""
# Funktionswerte an der Stelle x
y = np.zeros_like(x)
xs = xs.reshape(-1, 1)
x = x.reshape(1, -1)
# finde Evaluationspunkte welche sehr nahe an den Stuetzstellen liegen
lxx = np.product(xs - x, axis=0)
tol = 1e-20
# array of boolean's
idx = abs(lxx) < tol
idx_n = mlab.find(idx)
# finde die zugehoerigen Funktionswerte
yidx = np.zeros(len(idx_n), dtype=int)
for i, ii in enumerate(idx_n):
d = abs(np.squeeze(x)[ii] - xs)
yidx_local = mlab.find(d == min(d))
yidx[i] = yidx_local
# an den Stuetzstellen settz man direkt die
# bekannten Funktionswerte
y[idx_n] = yy[yidx]
# x, mit genuegend Abstand zu Stuetzstellen
idx = np.logical_not(idx) # invertiere idx
xr = np.squeeze(x)[idx].reshape(1, -1)
tmp = (np.squeeze(lam) * np.squeeze(yy)).reshape(-1, 1)
y2 = (lxx[idx]).reshape(1, -1) * np.sum(tmp / (xs - xr), axis=0)
y[idx.flatten()] = y2.squeeze()
return y
def run(self):
S1, S2 = self._geometric_sample()
beta, mellin1, mellin2 = self._mellin_transform(S1, S2)
# Computation for lambda dilations/compressions
waf = np.zeros((2 * self.m1, self.n_voices), dtype=complex)
_x = -(2 * 1j * np.pi * beta + 0.5)
for n in range(1, 2 * self.m1 + 1):
_y = np.log(
np.sqrt(1 + (self.u[n - 1] / 2.0)**2) - self.u[n - 1] / 2.0)
mx1 = np.exp(_x * _y) * mellin1
_y = np.log(
np.sqrt(1 + (self.u[n - 1] / 2.0)**2) + self.u[n - 1] / 2.0)
mx2 = np.exp(_x * _y) * mellin2
fx1 = np.fft.fft(np.fft.fftshift(mx1))[:self.n_voices]
fx2 = np.fft.fft(np.fft.fftshift(mx2))[:self.n_voices]
waf[n - 1, :] = fx1 * np.conj(fx2)
if self.form != "A":
waf[n - 1, :] *= 1 / np.sqrt(1 + (self.u[n] / 2.0)**2)
waf = np.vstack((waf[self.m1:(2 * self.m1), :], waf[:self.m1, :]))
waf *= np.repeat(self.geo_f.reshape((1, self.geo_f.shape[0])),
2 * self.m1,
axis=0)
tffr = np.fft.ifft(waf, axis=0)
tffr = np.real(
np.rot90(np.vstack(
(tffr[self.m1:(2 * self.m1 + 1), :], tffr[:self.m1, :])),
k=-1))
# conversion from tff to tf using 1d interpolation
tfr = np.zeros((self.n_voices, self.ts.shape[0]))
ts2 = (self.signal.shape[0] - 1.0) / 2
gamma = np.linspace(-self.geo_f[self.n_voices - 1] * ts2,
self.geo_f[self.n_voices - 1] * ts2, 2 * self.m1)
for i in range(self.n_voices):
ind = find(
np.logical_and(gamma >= -self.geo_f[i] * ts2,
gamma <= self.geo_f[i] * ts2))
x = gamma[ind]
y = tffr[i, ind]
xi = (self.ts - ts2 - 1) * self.geo_f[i]
tck = splrep(x, y)
tfr[i, :] = splev(xi, tck).ravel()
t = self.ts
f = self.freqs = self.geo_f[:self.n_voices].ravel()
sp1_ana, sp2_ana = self._normalize()
if self.kind == "auto":
multiplier = np.linalg.norm(sp1_ana)**2
else:
multiplier = np.dot(sp1_ana, sp2_ana)
tfr = tfr * multiplier / integrate_2d(tfr, t, f) / self.n_voices
self.tfr = tfr
return tfr, t, f
def zerocrossings(filepath, size, h_size):
''' calculate and return zero crossings per size-block '''
x, sr, fmt = wavread(filepath)
output_blocks = []
# modified from https://gist.github.com/255291
# for each block
for n in range(0, len(x), h_size):
current_samples = x[n:n + size]
indices = find((current_samples[1:] >= 0) & (current_samples[:-1] < 0))
# crossings = sum([1 for m in range(n, n+size) if x[m+1] > 0 and x[m] <= 0])
output_blocks.append(len(indices))
return output_blocks
def undistortedCoordinates(map1, map2, x, y, maxDistance = 1.):
'''Returns the coordinates of a point in undistorted image
map1 and map2 are the mapping functions from undistorted image
to distorted (original image)
map1(x,y) = originalx, originaly'''
distx = npabs(map1-x)
disty = npabs(map2-y)
indices = logical_and(distx<maxDistance, disty<maxDistance)
closeCoordinates = unravel_index(find(indices), distx.shape) # returns i,j, ie y,x
xWeights = 1-distx[indices]
yWeights = 1-disty[indices]
return dot(xWeights, closeCoordinates[1])/npsum(xWeights), dot(yWeights, closeCoordinates[0])/npsum(yWeights)
def solveRec(self, i, j):
'''
This is a helper recursive function used by solver.
It stores the solved sudoku in `self.solution`
Parameters
----------
i : int
Current row.
j : int
Current column.
'''
if self.matbool[i, j] == False:
for k in range(1, 10):
self.original[i, j] = k
#checking if value is plausible in row, col and square
if len(find(self.original[i,:] == k)) == 1 and \
len(find(self.original[:,j] == k)) == 1 and \
len(find(self.mySquare(i,j) == k)) == 1:
if i == 8 and j == 8:
self.solution = self.original.copy()
elif i < 8 and j == 8:
self.solveRec(i + 1, 0)
else:
self.solveRec(i, j + 1)
self.original[i, j] = 0
else:
if i == 8 and j == 8:
self.solution = self.original.copy()
elif i < 8 and j == 8:
self.solveRec(i + 1, 0)
else:
self.solveRec(i, j + 1)
def script(outfile=sys.stdout, quiet=False):
"""generate ACF scripts and calculates times. Reports state of
Catalog if a consistent script cannot be generated """
global Catalog
global Parameters
jj_nocalc = mlab.find(np.isnan(Catalog['Time']))
N_nocalc = len(jj_nocalc) # number of segments with uncalculated time
while N_nocalc > 0:
for kk in jj_nocalc: # was reversed before
if Catalog['TimeSegment'][kk] != None:
Catalog['TimeSegment'][kk].script('/dev/null')
N_old = N_nocalc
jj_nocalc = mlab.find(np.isnan(Catalog['Time']))
N_nocalc = len(jj_nocalc)
if N_nocalc == N_old:
break
if N_nocalc == 0:
# remove the existing file before writing to it.
if type(outfile) == str:
#os.remove(outfile)
outfilehandle = open(outfile, 'w')
else:
outfilehandle = outfile
if len(Parameters) > 0:
outfilehandle.write('[PARAMETER#]\n')
for param in Parameters:
outfilehandle.write(param + '\n')
outfilehandle.write('[LINE#]\n')
if outfilehandle.name != '<stdout>':
outfilehandle.close()
for TS in Catalog['TimeSegment']:
TS.script(outfile)
elif not quiet:
print "Timing did not converge:"
catalog()
print "No script output due to undefined sequence or waveform."
return False
return True
def Get_Column(self, column, **args):
#Return the values from a particular column.
index = self.Get_Index(column)
#if we've been asked to strip the imputed values, then do so
if 'strip_imputed' in args:
if args['strip_imputed'] == True:
good_rows = mlab.find(self.imputed == False)
return self.data[good_rows, index]
#Otherwise, return all data
else:
return self.data[:, index]
def match_points(self, coords, cutoff):
gps_points = self.gps_points()
points, _, _ = self.project_coordinates(coords, self._proj_src, self._proj_dst)
tree = scipy.spatial.cKDTree(points)
dist = list()
ind = list()
for p in gps_points:
(d, i) = tree.query(p, distance_upper_bound=cutoff)
dist.append(d)
ind.append(i)
ind = np.array(ind)
return ind, ml.find(ind == coords.shape[0]), np.array(dist)
def freq_from_crossings(self, sig):
"""Estimate frequency by counting zero crossings"""
# From https://gist.github.com/endolith/255291:
fs = self.sampleRate
# Find all indices right before a rising-edge zero crossing
indices = find((sig[1:] >= 0) & (sig[:-1] < 0))
# More accurate, using linear interpolation to find intersample
# zero-crossings (Measures 1000.000129 Hz for 1000 Hz, for instance)
crossings = [i - sig[i] / (sig[i + 1] - sig[i]) for i in indices]
# Some other interpolation based on neighboring points might be better. Spline, cubic, whatever
return fs / np.mean(np.diff(crossings))
def getdrift_raw_range_latlon(filename,id3,interval,datetime_wanted_1,num,step_size):
# this is for plot all the data in same range of lat and lon. id3 means int format of drift number
#'interval' means range of time, 'num' means how many pictures we will get
d=ml.load(filename)
lat1=d[:,8]
lon1=d[:,7]
idd=d[:,0]
year=[]
for n in range(len(idd)):
year.append(str(idd[n])[0:2])
h=d[:,4]
day=d[:,3]
month=d[:,2]
time1=[]
for i in range(len(idd)):
time1.append(date2num(datetime.datetime.strptime(str(int(h[i]))+' '+str(int(day[i]))+' '+str(int(month[i]))+' '+str(int(year[i])), "%H %d %m %y")))
idg1=list(ml.find(idd==id3))
idg2=list(ml.find(np.array(time1)<=datetime_wanted_1+step_size/24.0*(num-1)+0.25))
"'0.25' means the usual Interval, It can be changed base on different drift data "
idg3=list(ml.find(np.array(time1)>=datetime_wanted_1-interval/24.0))
idg23=list(set(idg2).intersection(set(idg3)))
# find which data we need
idg=list(set(idg23).intersection(set(idg1)))
# print len(idg),len(set(idg))
lat,lon,time=[],[],[]
for x in range(len(idg)):
lat.append(round(lat1[idg[x]],4))
lon.append(round(lon1[idg[x]],4))
maxlon=max(lon)
minlon=min(lon)
maxlat=max(lat)
minlat=min(lat)
# time is num
return maxlon,minlon,maxlat,minlat
def get_frequency_ac(self, signal):
"""
"""
# autocorrelation using numpy
signal = np.fromstring(signal, 'Int16')
corr = fftconvolve(signal, signal[::-1], mode='full')
corr = corr[len(corr) // 2:]
d = np.diff(corr)
start = find(d > 0)[0]
peak = np.argmax(corr[start:]) + start
px, py = self.parabolic(corr, peak)
return self.rate / px
def freq_from_autocorr(sig, fs):
"""
Estimate frequency using autocorrelation
"""
# Calculate autocorrelation (same thing as convolution, but with
# one input reversed in time), and throw away the negative lags
corr = fftconvolve(sig, sig[::-1], mode='full')
corr = corr[len(corr) // 2:]
# Find the first low point
d = diff(corr)
#
#
#start = find(d > 0)[0]
#
# 萬一 find(d > 0) == np.array([]) ！ 上面這行就有 bug 了
#
# 例如：
#
'''
>>> sig= np.array([1,1,1,1,1,1,1,1,1])
>>> corr = fftconvolve(sig, sig[::-1], mode='full')
>>> corr = corr[len(corr)//2:]
>>> d = diff(corr)
>>> d>0
array([False, False, False, False, False, False, False, False], dtype=bool)
>>> find(d>0)
array([], dtype=int64)
>>> find(d>0)[0]
Traceback (most recent call last):
File "<pyshell#19>", line 1, in <module>
find(d>0)[0]
IndexError: index 0 is out of bounds for axis 0 with size 0
>>>
'''
try:
start = find(d > 0)[0]
except:
print('start = find(d > 0)[0] 此行有 bug, 先擋一下吧。')
#start= 0
fs_px = 0
return fs_px
# Find the next peak after the low point (other than 0 lag). This bit is
# not reliable for long signals, due to the desired peak occurring between
# samples, and other peaks appearing higher.
# Should use a weighting function to de-emphasize the peaks at longer lags.
peak = argmax(corr[start:]) + start
px, py = parabolic(corr, peak)
return fs / px
def stimuli(self,CS):
compareA = np.append(find(CS),find(CS)[-1])
compareB = np.append(find(CS)[0],find(CS))
delta = compareA-compareB
# stimulus on
on = np.zeros(len(CS))
on[compareA[find(delta>1)]] = 1
on[compareA[0]] = 1
# stimulus off
off= np.zeros(len(CS))
off[compareA[find(delta>1)-1]] = 1
off[compareA[-1]] = 1
return find(on), find(off)
def find_final(xp, yp, ind=-1):
"""
Loop through drifters and find final location of drifters within the
tracks arrays. This can be necessary because when drifters exit the
numerical domain, they are nan'ed out. default is to search for the final
non-nan location (-1), but can search for others instead, for example,
the first non-nan position, which is helpful if we are looking at the
flipped output from a backward run.
"""
# Find final position for drifters (since they are nan'ed out after they
# hit the open boundary)
# Make this a separate function later
xpc = []
ypc = []
for idrift in xrange(xp.shape[0]):
# Find last non-nan and make sure it is in the desired month start
# time
ind3 = ~np.isnan(xp[idrift, :])
# only plot if last non-nan (back in time) is in 1 month period
# in order to plot the tracks that "started" in the plotted month
if np.sum(ind3) > 1: # don't want tracks that start on land
# This is for if we care when the drifter stopped
# if t[find(ind3)[ind]] >= datetime(year,startMonth,startDay,0) and \
# t[find(ind3)[ind]] <= datetime(year,startMonth+1,startDay,0):
# ind2 = ~np.isnan(xp[idrift,:])
# if there is a nan
if np.sum(np.isnan(xp[idrift, :])) > 0 and \
np.sum(np.isnan(xp[idrift, :])) < xp.shape[1]:
# ax.plot(xp[idrift,find(ind2)[ind]].T,yp[idrift,find(ind2)[ind]].T,'o',color='orange',linewidth=.5,label='_nolegend_')
xpc.append(xp[idrift, find(ind3)[ind]])
ypc.append(yp[idrift, find(ind3)[ind]])
else:
# ax.plot(xp[idrift,ind].T,yp[idrift,ind].T,'o',color='orange',linewidth=.5,label='_nolegend_')
xpc.append(xp[idrift, find(ind3)[ind]])
ypc.append(yp[idrift, find(ind3)[ind]])
return xpc, ypc
def inputChecker(matrix):
'''
inputChecker takes a sudoku and return True if it is a valid initial
sudoku, i.e. it's shape is 9x9 and it meets all the rules for non-zero values.
Parameters
----------
matrix: array_like
`matrix` is a numpy.array that contains a 9x9 sudoku.
Returns
-------
valid : bool
If the input matrix is correct, a True value will be returned.
'''
valid = True
if matrix.shape != (9,9) or\
matrix.min() < 0 or \
matrix.max() > 9:
valid = False
return valid
for i in range(0, 9):
for j in range(0, 9):
k = matrix[i, j]
if k != 0:
if len(find(matrix[i,:] == k)) > 1 or \
len(find(matrix[:,j] == k)) > 1 or \
len(find(sudokuCommonTools.mySquare(matrix,i,j) == k)) > 1:
valid = False
return valid
def draw_f0_amdf(self, y_or, sr, window_len):
duration = float(len(y_or)) / sr
# print("duration:", duration)
x = window_len
time_ptr = []
f0_list = []
while x < duration - window_len * 1.01:
try:
y_windows = y_or[int((x - window_len / 2) *
sr):int((x + window_len / 2) * sr)]
z = librosa.zero_crossings(y_windows)
N = len(y_windows)
Xk = np.fft.fft(y_windows)
E_fft = np.sum(np.abs(Xk)**2) / N
if (len(np.nonzero(z)[0]) > 90) or (E_fft < 4):
f0_list.append(0)
time_ptr.append(x)
x += window_len / 2
continue
dk_list = compute_dk_list(y_or=y_windows)
d = diff(dk_list)
start = find(d > 0)[0]
min_index1 = np.argmin(dk_list[start:]) + start
dk_second_list = dk_list[(min_index1 + 5):]
d = diff(dk_second_list)
start = find(d > 0)[0]
min_index2 = np.argmin(dk_second_list[start:]) + start
f0_list.append(float(self.sr) / (min_index2 + 1))
except Exception as e:
print("loi draw_f0_amdf: ", e)
f0_list.append(0)
time_ptr.append(x)
x += window_len / 2
average = np.average(f0_list)
print(average, "; ", np.median(f0_list))
self.ax5.set_ylim([0, 400])
self.ax5.scatter(time_ptr, f0_list, s=2)
pass
def cb_active_plot(self, start_ms, stop_ms, line_color='b'):
"""
Plot timing information of time spent in the callback. This is similar
to what a logic analyzer provides when probing an interrupt.
cb_active_plot( start_ms,stop_ms,line_color='b')
"""
# Find bounding k values that contain the [start_ms,stop_ms]
k_min_idx = mlab.find(np.array(self.DSP_tic) * 1000 < start_ms)
if len(k_min_idx) < 1:
k_min = 0
else:
k_min = k_min_idx[-1]
k_max_idx = mlab.find(np.array(self.DSP_tic) * 1000 > stop_ms)
if len(k_min_idx) < 1:
k_max = len(self.DSP_tic)
else:
k_max = k_max_idx[0]
for k in range(k_min, k_max):
if k == 0:
plt.plot([
0, self.DSP_tic[k] * 1000, self.DSP_tic[k] * 1000,
self.DSP_toc[k] * 1000, self.DSP_toc[k] * 1000
], [0, 0, 1, 1, 0], 'b')
else:
plt.plot([
self.DSP_toc[k - 1] * 1000, self.DSP_tic[k] * 1000,
self.DSP_tic[k] * 1000, self.DSP_toc[k] * 1000,
self.DSP_toc[k] * 1000
], [0, 0, 1, 1, 0], 'b')
plt.plot([self.DSP_toc[k_max - 1] * 1000, stop_ms], [0, 0], 'b')
plt.xlim([start_ms, stop_ms])
plt.title(r'Time Spent in the callback')
plt.ylabel(r'Timing')
plt.xlabel(r'Time (ms)')
plt.grid()
def freq_from_crossings(signal, fs):
signal = np.asarray(signal) + 0.0 # Cute :) DG
indices = find((signal[1:] >= 0) & (signal[:-1] < 0))
crossings = [i - signal[i] / (signal[i + 1] - signal[i]) for i in indices]
diffs = np.diff(crossings)
# Check for odd length:
try:
averagediffs = np.median(diffs) # Try mean, etc.
except:
averagediffs = np.mean(diffs)
if averagediffs != 0:
return fs / averagediffs
else:
return 0
def fortgaugeread(datafile="fort.gauge", setgaugefile="setgauges.data"):
"""
fortgaugeread (datafile="fort.gauge",setgaugefile="setgauges.data"):
Read data from fort.gauge files output by GeoClaw.
Reads the gauge data and returns a list of dictionaries, with mgauge elements.
Each element of the list is a dictionary containing the data for a particular gauge.
Each dictionary has keys for each variable: t,h,hu,hv,eta,x,y,gauge.
example:
for N gauges: allgaugedata=[dictionary_1,...,dictionary_N] where
dictionary_n = {'t': (one-dimensional numpy array), 'h': (one-dimensional numpy array), ...,}
"""
fid = open(setgaugefile)
inp = '#'
while inp == '#':
inpl = fid.readline()
inp = inpl[0]
inp = fid.readline()
mgauges = int(inp.split()[0])
gaugelocs = []
linesread = 0
while linesread < mgauges:
row = string.split(fid.readline())
if row != []:
gaugelocs.append(row)
linesread = linesread + 1
fid.close()
data = loadtxt(datafile)
allgaugedata = []
for n in xrange(mgauges):
dict = {}
dict['gauge'] = int(gaugelocs[n][0])
dict['x'] = float(gaugelocs[n][1])
dict['y'] = float(gaugelocs[n][2])
onegaugedata = data[mlab.find(data[:, 0] == dict['gauge'])]
dict['level'] = onegaugedata[:, 1]
dict['t'] = onegaugedata[:, 2]
dict['mq'] = len(onegaugedata[0]) - 2
for m in xrange(1, dict['mq']):
dict['q' + str(m)] = onegaugedata[:, 2 + m]
allgaugedata.append(dict)
return allgaugedata
def wplot(TimingObjectLabel):
""" Plot the waveform of the specified sequence """
global Catalog
if TimingObjectLabel not in Catalog['Name']:
print Catalog['Name']
return
seqnum = mlab.find(np.array(Catalog['Name']) == TimingObjectLabel)
Catalog['TimeSegment'][seqnum].plot()
return
def assign_coords(self, space='brainsight'):
if self.span_type == 'period' and self.datum_type.Name == 'mep_mapping':
#Find and load the brainsight file
dir_stub = get_or_create(System, Name='bci_dat_dir').Value
bs_file_loc = dir_stub + '/' + self.subject.Name + '/mapping/' + str(
self.Number) + '_' + space + '.txt'
#Parse the brainsight file for X-Y coordinates
data = [line.split('\t') for line in file(bs_file_loc)]
data = [line for line in data if 'Sample' in line[0]]
starti = find(['#' in line[0] for line in data])[0]
data = data[starti:]
headers = data[0]
data = data[1:]
x_ind = find(['Loc. X' in col for col in headers])[0]
y_ind = find(['Loc. Y' in col for col in headers])[0]
z_ind = find(['Loc. Z' in col for col in headers])[0]
i = 0
for tt in self.trials:
tt.detail_values['dat_TMS_coil_x'] = float(data[i][x_ind])
tt.detail_values['dat_TMS_coil_y'] = float(data[i][y_ind])
tt.detail_values['dat_TMS_coil_z'] = float(data[i][z_ind])
i = i + 1
