def getSG(y, filtwidth=21, filtorder=2, deriv=1):
if filtwidth < filtorder + 2:
filtwidth = filtorder + 3
if filtwidth % 2 == 0:
filtwidth += 1
print 'WARN: window size reset to {0}'.format(filtwidth)
return sg.savitzky_golay(y, filtwidth, filtorder, deriv=deriv)
def update(self):
if self._running:
try:
img = np.rot90(self._cam.image, 3).astype(
np.int16) - self._bckg
self._viewImg.setImage(img)
"""Construct x and y arrays"""
crop = self._roi.getArrayRegion(img, self._viewImg)
horMax, verMax = np.unravel_index(crop.argmax(), crop.shape)
arr = np.mean(crop, axis=1)
reg = [
self._roi.pos()[0],
self._roi.pos()[0] + self._roi.size()[0]
]
xf = np.arange(reg[0], reg[1])
"""Smooth the signal"""
if self._smoothorder != 0:
arr = savitzky_golay(arr, self._smoothwindow,
self._smoothorder)
"""Initial guess for fit parameters"""
off = np.mean(arr[0:25] + arr[-26:-1]) / 2
c = xf[-1] - (xf[-1] - xf[0]) / 2
s = 50
p0 = [off, np.max(arr), c, s]
coeff, var_matrix = curve_fit(self.gaus, xf, arr, p0=p0)
fit = self.gaus(xf, *coeff)
self._curveRaw.setData(arr)
self._curveFit.setData(fit)
# FWHM = 2 * sqrt( 2 * log(2) ) * sigma
fwhm_f = 2.3548 * coeff[3]
fwhm_d = len(arr[arr > np.max(arr) / 2])
self.lab_fwhm_fit.setText(str(format(fwhm_f, '.4f')) + ' px')
self.lab_fwhm_data.setText(str(fwhm_d) + ' px')
self.lab_bl_fwhm_fit.setText(
str(
format((1e12 * self._mm_pix * fwhm_f * 1e-3) /
self._C, '.4f')) + ' ps')
self.lab_bl_fwhm_data.setText(
str(
format((1e12 * self._mm_pix * fwhm_d * 1e-3) /
self._C, '.4f')) + ' ps')
self.lab_bl_sigma_fit.setText(
str(
format((1e12 * self._mm_pix * coeff[3] * 1e-3) /
self._C, '.4f')) + ' ps')
self.lab_bl_sigma_data.setText(
str(format(np.std(arr), '.4f')) + ' px')
self._output.setText('')
except Exception as e:
logging.error('Error somewhere! ' + str(e))
self._output.setText(str(e))
def call_peaks(scores, min_dist, iters, window, order):
peaks = []
for i in range(iters):
scores = savitzky_golay(scores, window, order, deriv=0, rate=1)
med_score = np.median(scores)
if max(scores) < 6 * med_score:
return peaks
peaks, _ = find_peaks(scores, distance=min_dist, height=med_score * 3)
return peaks
def create_data ( n_per=4, noise=0.15, obs_off=0.33, \
window_size=0, order=4):
"""
Create synthetic "NDVI-like" data for a fictitious time series. We return
the original data, noisy data (using IID Gaussian noise), the QA flag as well
as the time axis.
Missing observations are simulated by drawing a random number between 0 and 1
and checking against obs_off.
Parameters
----------
n_per : integer
Observation periodicity. By default, assumes every 8 days
noise : float
The noise standard deviation. By default, 0.15
obs_off : float
The threshold to decide on missing observations in the time series.
window_size : integer, odd
window size for savitzky_golay filtering. A large window size will lead
to larger data gaps by correlating the noise. Set to zero by default
which applies no smoothing.
order : integer
order of the savitzky_golay filter. By default 4.
"""
from savitzky_golay import savitzky_golay
import numpy as np
doys = np.arange(1, 365 + 1, n_per)
ndvi_clean = np.clip(np.sin((doys - 1) / 72.), 0, 1)
ndvi = np.clip(np.sin(doys / 72.), 0, 1)
# add Gaussian noise of sd noise
ndvi = np.random.normal(ndvi, noise, ndvi.shape[0])
# set the qa flags for each sample to 1 (good data)
qa_flag = np.ones_like(ndvi).astype(np.int32)
passer = np.random.rand(ndvi.shape[0])
if window_size > 0:
# force odd
window_size = 2 * (window_size / 2) + 1
passer = savitzky_golay(passer, window_size=window_size, order=order)
# assign a proportion of the qa to 0 from an ordering of the smoothed
# random numbers
qa_flag[np.argsort(passer)[:passer.size * obs_off]] = 0
return (doys, ndvi_clean, ndvi, qa_flag)
def create_data ( n_per=4, noise=0.15, obs_off=0.33, \
window_size=0, order=4):
"""
Create synthetic "NDVI-like" data for a fictitious time series. We return
the original data, noisy data (using IID Gaussian noise), the QA flag as well
as the time axis.
Missing observations are simulated by drawing a random number between 0 and 1
and checking against obs_off.
Parameters
----------
n_per : integer
Observation periodicity. By default, assumes every 8 days
noise : float
The noise standard deviation. By default, 0.15
obs_off : float
The threshold to decide on missing observations in the time series.
window_size : integer, odd
window size for savitzky_golay filtering. A large window size will lead
to larger data gaps by correlating the noise. Set to zero by default
which applies no smoothing.
order : integer
order of the savitzky_golay filter. By default 4.
"""
from savitzky_golay import savitzky_golay
import numpy as np
doys = np.arange ( 1, 365+1, n_per)
ndvi_clean = np.clip(np.sin((doys-1)/72.), 0,1)
ndvi = np.clip(np.sin(doys/72.), 0,1)
# add Gaussian noise of sd noise
ndvi = np.random.normal(ndvi,noise,ndvi.shape[0])
# set the qa flags for each sample to 1 (good data)
qa_flag = np.ones_like ( ndvi).astype( np.int32 )
passer = np.random.rand( ndvi.shape[0])
if window_size >0:
# force odd
window_size = 2*(window_size/2)+1
passer = savitzky_golay(passer, window_size=window_size, order=order)
# assign a proportion of the qa to 0 from an ordering of the smoothed
# random numbers
qa_flag[np.argsort(passer)[:passer.size * obs_off]] = 0
return ( doys, ndvi_clean, ndvi, qa_flag )
def get_split_prep_data(train_start, train_end,
test_start, test_end):
data = np.loadtxt(path_to_dataset)
data = savitzky_golay(data[:, 1], 11, 3) # smoothed version
print("Length of Data", len(data))
# train data
print "Creating train data..."
result = []
for index in range(train_start, train_end - sequence_length):
result.append(data[index: index + sequence_length])
result = np.array(result) # shape (samples, sequence_length)
result, result_mean = z_norm(result)
print "Mean of train data : ", result_mean
print "Train data shape : ", result.shape
train = result[train_start:train_end, :]
np.random.shuffle(train) # shuffles in-place
X_train = train[:, :-1]
y_train = train[:, -1]
X_train, y_train = dropin(X_train, y_train)
# test data
print "Creating test data..."
result = []
for index in range(test_start, test_end - sequence_length):
result.append(data[index: index + sequence_length])
result = np.array(result) # shape (samples, sequence_length)
result, result_mean = z_norm(result)
print "Mean of test data : ", result_mean
print "Test data shape : ", result.shape
X_test = result[:, :-1]
y_test = result[:, -1]
print("Shape X_train", np.shape(X_train))
print("Shape X_test", np.shape(X_test))
X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], 1))
X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1], 1))
return X_train, y_train, X_test, y_test
def get_split_prep_data(train_start, train_end, test_start, test_end):
data = np.loadtxt(path_to_dataset)
data = savitzky_golay(data[:, 1], 11, 3) # smoothed version
print("Length of Data", len(data))
# train data
print("Creating train data...")
result = []
for index in range(train_start, train_end - sequence_length):
result.append(data[index:index + sequence_length])
result = np.array(result) # shape (samples, sequence_length)
result, result_mean = z_norm(result)
print("Mean of train data : ", result_mean)
print("Train data shape : ", result.shape)
train = result[train_start:train_end, :]
np.random.shuffle(train) # shuffles in-place
X_train = train[:, :-1]
y_train = train[:, -1]
X_train, y_train = dropin(X_train, y_train)
# test data
print("Creating test data...")
result = []
for index in range(test_start, test_end - sequence_length):
result.append(data[index:index + sequence_length])
result = np.array(result) # shape (samples, sequence_length)
result, result_mean = z_norm(result)
print("Mean of test data : ", result_mean)
print("Test data shape : ", result.shape)
X_test = result[:, :-1]
y_test = result[:, -1]
print("Shape X_train", np.shape(X_train))
print("Shape X_test", np.shape(X_test))
X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], 1))
X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1], 1))
return X_train, y_train, X_test, y_test
def getArgMin(self,Vpp=1,offset=0.,samples=1000,readChan="ai0",writeChan="ao0",rate=1000,fig=None,ax=None,filter=51): inData, outData = self.syncRamp(Vpp,offset,samples,readChan,writeChan,rate) filteredData = savitzky_golay(outData,filter,3) minimum = min(filteredData) inVoltageAtMin = inData[filteredData.argmin()] if fig != None: plt.figure(fig) ax = plt.gca() if fig or ax: print "Plotting ", ax.plot(inData,outData,'k') ax.plot(inData,filteredData,'g') ax.plot(inVoltageAtMin,minimum,'r+') plt.draw() plt.show(block=False) print "[DONE]" return minimum,inVoltageAtMin
def getRGBThreshold(im):
y = np.zeros((3, 256))
for j in range(3):
hist = skimage.exposure.histogram(im[:, :, j])
y[j, hist[1]] = hist[0]
ymin = np.min(y, axis=0)
ymin = savitzky_golay(ymin, 11, 3)
m = np.max(ymin)
ym = np.argmax(ymin)
coeff, _ = curve_fit(gaussian, range(256), ymin, [90.0, ym, m])
## MONITORING ###
rg = range(256)
MONITORING(3, 1, plotCurve, rg, ymin)
MONITORING(3, 2, plotCurve, rg, gaussian(rg, *coeff), clear=False)
## ###
return coeff[1] - abs(coeff[0]) * 2.7
def find_histogram_peaks( ndata, num_peaks=5 ) :
hist,bins = np.histogram(ndata,bins=256)
np.save("hist.npy",hist)
# zero pad the end so clipped white will show up as a peak
tmp = np.zeros(257)
tmp[0:256] = hist
hist = tmp
del tmp
if 1 :
# smoothed 1st derivative of the signal
svg = savitzky_golay.savitzky_golay( hist, 21, 5, 1 )
np.save("svg.npy",svg)
peaks_list = find_peaks( svg )
# look for zero crossing
zeros_list = find_zeros( svg )
print "zeros_list=",zeros_list
highest_peaks = find_highest_peaks( zeros_list, hist, num_peaks )
print "highest_peaks=",highest_peaks
else :
# nothing fancy; just use argmax repeatedly
highest_peaks = simple_highest_peaks( ndata, hist, num_peaks )
svg = None
# svg = np.diff(hist)
# 2nd derivative
# svg = np.diff(np.diff(hist))
# highest_peaks = np.argsort(svg)[0:num_peaks]
# highest_peaks = np.argsort(svg)[-num_peaks:]
print highest_peaks
save_gray_histogram_img( hist, svg, highest_peaks )
sorted_peaks = sorted(highest_peaks)
histo_values = [ hist[p] for p in sorted_peaks ]
# return the pixel values of the peaks and the histogram counts
return sorted_peaks, histo_values
def plot_savitzky_golay( ndata ) :
# kill the clipped-to-white pixels
# ndata = ndata[:-1]
smoothed = savitzky_golay.savitzky_golay( ndata, 21, 5, 1 )
# print "smoothed=", smoothed
fig = Figure()
ax = fig.add_subplot(111)
ax.grid()
ax.plot(ndata)
# ax = fig.add_subplot(212)
# ax.grid()
ax.plot(smoothed)
outfilename = "svgy.pdf"
canvas = FigureCanvasAgg(fig)
canvas.print_figure(outfilename)
print "wrote",outfilename
return smoothed
def get_split_prep_data(train_ratio):
file = [
100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 111, 112, 113, 114,
115, 116, 117, 118, 119, 121, 122, 123, 124, 200, 201, 202, 203, 205,
207, 208, 209, 210, 212, 213, 214, 215, 217, 219, 220, 221, 222, 223,
228, 230, 231, 232, 233, 234
]
file = [str(i) for i in file]
normal = []
fusion = []
atrial = []
ventri = []
for f in file:
nor = np.load('mitdb/N_' + f + '.npy')
atr = np.load('mitdb/A_' + f + '.npy')
fus = np.load('mitdb/F_' + f + '.npy')
ven = np.load('mitdb/V_' + f + '.npy')
np.reshape(nor, (nor.shape[0], 2))
np.reshape(fus, (fus.shape[0], 2))
np.reshape(atr, (atr.shape[0], 2))
np.reshape(ven, (ven.shape[0], 2))
print 'Ori_Shape:', nor.shape, fus.shape, atr.shape, ven.shape
if nor.shape[0] > 0:
nor[:, 0] = savitzky_golay(nor[:, 0], 11, 3)
nor[:, 1] = savitzky_golay(nor[:, 1], 11, 3)
if atr.shape[0] > 0:
atr[:, 0] = savitzky_golay(atr[:, 0], 11, 3)
atr[:, 1] = savitzky_golay(atr[:, 1], 11, 3)
if fus.shape[0] > 0:
fus[:, 0] = savitzky_golay(fus[:, 0], 11, 3)
fus[:, 1] = savitzky_golay(fus[:, 1], 11, 3)
if ven.shape[0] > 0:
ven[:, 0] = savitzky_golay(ven[:, 0], 11, 3)
ven[:, 1] = savitzky_golay(ven[:, 1], 11, 3)
normal = ladd(normal, nor, [1, 0, 0, 0])
atrial = ladd(atrial, atr, [0, 1, 0, 0])
fusion = ladd(fusion, fus, [0, 0, 1, 0])
ventri = ladd(ventri, ven, [0, 0, 0, 1])
normal = np.array(normal)[:class_size]
atrial = np.array(atrial)[:class_size]
fusion = np.array(fusion)[:class_size]
ventri = np.array(ventri)[:class_size]
normal_size = len(normal)
fusion_size = len(fusion)
atrial_size = len(atrial)
ventri_size = len(ventri)
print 'Normal_size', normal_size
print 'Fusion_size', fusion_size
print 'Atrial_size', atrial_size
print 'Ventri_size', ventri_size
result = np.concatenate((normal, atrial, fusion, ventri))
np.random.shuffle(result)
label = np.array(result[:, -1].tolist())
data = np.array(result[:, :-1].tolist())
result = data
print('Result shape', result.shape, label.shape)
result, result_mean = z_norm(result)
print('Result, Result_mean', result.shape, result_mean)
print "Mean of train data : ", result_mean
print "Train data shape : ", result.shape
# label = []
# idx = 0
# while idx < normal_size:
# label.append([1,0,0,0])
# idx += 1
# idx = 0
# while idx < atrial_size:
# label.append([0,1,0,0])
# idx += 1
# idx = 0
# while idx < fusion_size:
# label.append([0,0,1,0])
# idx += 1
# idx = 0
# while idx < ventri_size:
# label.append([0,0,0,1])
# idx += 1
print 'Resule Label Shape', result.shape, label.shape
train_size = int(result.shape[0] * train_ratio)
X_train, y_train = result[:train_size], label[:train_size]
X_test, y_test = result[train_size:], label[train_size:]
print 'X_shape', X_train.shape, X_test.shape
print 'y_shape', y_train.shape, y_test.shape
X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], 2))
X_test = np.reshape(X_test, (X_test.shape[0], X_train.shape[1], 2))
return X_train, y_train, X_test, y_test
def smooth_chromatogram_using_Savitzky_Golay(i_list):
i_list2 = sg.savitzky_golay(
np.array(i_list), 7,
3) # window size 11, polynomial order 3. Optimized for chrom
return i_list2.tolist()
def smooth_chromatogram_using_Savitzky_Golay(i_list):
i_list2 = sg.savitzky_golay(np.array(i_list), 7, 3) # window size 11, polynomial order 3. Optimized for chrom
return i_list2.tolist()
T = data[:,spec['Frame']]/fps - data[0,spec['Frame']]/fps
X = (data[:,spec['X']]-Xo)*scale
Y = (data[:,spec['Y']]-Yo)*scale
#Y = -data[:,spec['Y']]/9.67 + 90.
Rad = np.pi*data[:,spec['Phi']]/180.
Angle = Rad #data[:,spec['Phi']]
#print(T.shape, X.shape, Y.shape, Angle.shape)
"""
Calculate forward, sideward, angular and translational speed
"""
Vf = np.multiply( (X[1:]-X[:-1]), np.cos(Rad[:-1]) ) + np.multiply( Y[1:]-Y[:-1], np.sin(Rad[:-1]))
Vf = Vf * fps
Vs = np.multiply( -(X[1:]-X[:-1]), np.sin(Rad[:-1]) ) + np.multiply( Y[1:]-Y[:-1], np.cos(Rad[:-1]))
Vs = Vs * fps
AngleSav = savitzky_golay(Angle, 51., 3.) # window size 51, polynomial order 3
Vr = fps*(np.diff(AngleSav))
Vt = np.sqrt(np.multiply(Vf,Vf) + np.multiply(Vs,Vs));
aVt = np.abs(np.diff(Vt))
"""
Clear jumps
"""
peaks = detect_peaks(aVt, mph=10, mpd=5)
#peaks = peakutils.peak.indexes(aVt, thres=0.3, min_dist=2)
Vf_tmp = Vf
peaks_tmp = np.hstack( (np.zeros(1), peaks, (np.ones(1)*Vf.size)) )
numjumps = peaks_tmp.size - 1
print("#Jump events:", numjumps)
for i in range(numjumps):
def apply_savitzky_golay(series): result = savitzky_golay.savitzky_golay(np.array(series), 31, 3) return result.tolist()
# 9 rows
p = '/mnt/hit4/hit4user/PycharmProjects/my_pytorch_lstm/YNDX_191211_191223.csv'
data = augum(p, [7])
data = data[:500, :] # limit
# replace date
dat = list(range(data.shape[0]))
data[:, 0] = scaler_simple(dat)
# test
# data[:, 1] = scaler_simple(dat)
# print(data[1101, 0])
# SCALING
data = my_scaler(data) # (0,1)
# SMOOTHING
s = data[:, 1].copy()
data[:, 1] = savitzky_golay(data[:, 1], 87, 7)
# save original
torch.save(data, open('traindata_ya_orig.pt', 'wb'))
# PRINT origin
plt.plot(np.arange(data.shape[0]), s, 'r', linewidth=2.0)
plt.plot(np.arange(data.shape[0]), data[:, 1], 'b', linewidth=2.0)
plt.show()
# BATCHING
# 1) strict windows
# for di in range(0, len(data), batch_num):
# window = train_data[di:di + batch_num]
# if len(window) < 1000: # last short window
for yr in range(start_year,end_year+1):
SOS_dbl_2rd.append(-9999)
EOS_dbl_2rd.append(-9999)
SOS_Gmax20.append(-9999)
EOS_Gmax20.append(-9999)
SOS_Gmax50.append(-9999)
EOS_Gmax50.append(-9999)
SOS_poly.append(-9999)
EOS_poly.append(-9999)
SOS_dbl_1st.append(-9999)
EOS_dbl_1st.append(-9999)
continue
### replace negative NDVIS
multi_year_vi[ind] = np.min(multi_year_vi[~ind])
multi_year_vi_sg=savitzky_golay.savitzky_golay(multi_year_vi,15,4)
# multiyear average NDVI and gs NDVI/ nongs NDVI
avg_multi = np.nanmean(np.reshape(multi_year_vi_sg,[len(multi_year_vi_sg)/24,24]),axis=0)
avg_vi = np.nanmean(avg_multi)
gs_avg_vi = np.nanmean(avg_multi[6:18])
nongs_avg_vi = (np.nanmean(avg_multi[0:6])+np.nanmean(avg_multi[18:24]))/2
if((avg_vi>0.1) and (gs_avg_vi > (nongs_avg_vi + 0.1))):
if(~np.isnan(np.sum(multi_year_vi_sg))):
[onset_ndvi,dormacy_ndvi] = Phen_Est.get_onset_dormancy_ndvi(multi_year_vi_sg,ndata_year)
else:
onset_ndvi = np.nan
dormacy_ndvi = np.nan
multi_year_vi_sg = np.squeeze(multi_year_vi_sg)
def execution():
pz = []
ln = []
dz = []
cvi = 0
mainth = 1.0
thresh = 100.0
vth = 0.0
filtwidth = 51
filtorder = 2
for c in cv:
if len(c) > 0:
cvi += 1
x = c[2].z
y = c[2].f
ifrom = np.argmax(y)
x = x[ifrom:]
y = y[ifrom:]
pylab.subplot(2, 2, cvi)
pylab.plot(x, y, 'k.')
pylab.title(c.basename)
der = sg.savitzky_golay(y, filtwidth, filtorder, deriv=1)
xi = np.where(der > mainth)
segments = []
js = []
trovati = xi[0]
for j in range(len(y)):
if j in trovati:
if len(js) > 0:
xx = x[js]
yy = y[js]
segments.append(seg(xx, yy))
js = []
else:
js.append(j)
if len(js) > 0:
xx = x[js]
yy = y[js]
segments.append(seg(xx, yy))
i = 1
for s in segments:
if s.len() > vth:
s.plot(i, False)
i = i + 1
if s.len() > thresh:
pz.append(s.val())
ln.append(s.len())
prev = segments[-1].val()
for s in segments[::-1]:
if s.len() > thresh:
if s.val() != prev:
dz.append(s.val() - prev)
prev = s.val()
y_filtered['tv'] = y_tv * y_max #---------------------------- # LOWESS (locally weighted scatter plot smooth) #---------------------------- import statsmodels.api as sm y_filtered['lowess'] = sm.nonparametric.lowess(np.copy(y), np.copy(x), frac=0.05)[:,1] #---------------------------- # Savitzky Golay filter #---------------------------- y_filtered['savitzky_golay'] = SG1.savitzky_golay(np.copy(y), window_size=5, order=2, deriv=0) #---------------------------- # Derivatives #---------------------------- y_filtered['savitzky_golay_1'] = SG1.savitzky_golay(y_filtered['gaussian'], window_size=5, order=2, deriv=1) y_filtered['savitzky_golay_2'] = SG1.savitzky_golay(y_filtered['gaussian'], window_size=5, order=2, deriv=2) #------------------------------------------- # Principal Component Analysis (time series)
def DigitizePeaklist(peaklist,
resolution=0.5,
debug=False,
sg=False,
inputtype='tuple',
returntype='tuple'):
"""
Bin the list of peaks so that resolution decreases
--------
Keyword arguments:
peaklist -- input peaklist as list of tuples \n
resolution -- size of the bins, default 0.5 \n
debug -- True activates verbose output
sg -- apply savitzky golay filtering
inputtype -- tuple or list
returntype -- tuple is a outputs two lists of mz and intensities, list
outputs one list of tuples of mz, int
"""
import numpy as np
import sys
import savitzky_golay as sg
if inputtype == 'tuple':
mz_in, intens_in = zip(*peaklist)
mz = np.array(mz_in)
counts_double = np.array(intens_in)
elif inputtype == 'list':
mz = np.array(peaklist[0])
counts_double = np.array(peaklist[1])
else:
sys.exit('You must provide a valid inputtype (list or tuple)')
bins = np.arange(0, max(mz) + 1, resolution)
inds = np.digitize(mz, bins)
if debug:
print(mz)
print(bins)
for n in range(round(mz.size / resolution)):
print(bins[inds[n] - 1], "<=", mz[n], "<", bins[inds[n]])
mz_bins_double = []
for m in range(mz.size):
mz_bins_double.append(bins[inds[m] - 1])
mz_bins_double = np.array(mz_bins_double)
mz_counts_tuple = {}
for index, mass in enumerate(mz_bins_double):
if mass in mz_counts_tuple.keys():
mz_counts_tuple[mass] += counts_double[index]
else:
mz_counts_tuple[mass] = counts_double[index]
mz_bins = []
counts = []
for element in sorted(list(mz_counts_tuple.items())):
mz_bins.append(element[0])
counts.append(element[1])
if sg:
counts = sg.savitzky_golay(counts, 49, 3)
if returntype == 'tuple':
return mz_bins, counts
if returntype == 'list':
return zip(mz_bins, counts)
def plotv_slice(arr,model=None,s=None,dq=None,tau=1.,r=3,boxtext=None,show=True,rf=10,tight=True):
'''
Plots vertical slices of arr at y=cst and x=cst respectively.
Depending on optional parameters, also plots tau=tau (=1 by
default) isosurface. Marks in red the "eye" of the nMBP.
'''
if not model: model=_model
if dq == None: dq=bool(_parfile)
if dq:
tau1_level=np.array(pybold.level(model.dq.tau,tau),dtype=int)
tau1=int(round(np.mean(tau1_level)))
else: tau1=int(np.size(model.z.xc3)/2.)
if s is None: s=snake.snake_from_box(model.z.rho,radius=r,start=tau1)
sx, sy, sz = np.shape(arr)
x, y, z = np.array(s, dtype=int).T
xc, yc, zc = model.z.xc1[:,0,0]/1.e5, model.z.xc2[0,:,0]/1.e5, model.z.xc3[0,0,:]/1.e5
if dq:
tau1_sindex = int(list(s[:,2]).index(tau1))
xmean = int(s[tau1_sindex,0])
ymean = int(s[tau1_sindex,1])
else:
xmean = int(round(np.mean(x)))
ymean = int(round(np.mean(y)))
tau1 = 0
sharey = False
if tight: z0 = z
else:
if dq:
xp,yp,z1p,z2p = xc, yc, zc[tau1_level[:,ymean]]-zc[tau1],zc[tau1_level[xmean,:]]-zc[tau1]
z0min = z[0]
z0max = z[-1]+1
z0min = min(z0min,np.min(tau1_level[:,ymean]))
z0max = max(z0max,1+np.max(tau1_level[:,ymean]))
z0min = min(z0min,np.min(tau1_level[xmean,:]))
z0max = max(z0max,1+np.max(tau1_level[ymean,:]))
delta = z0max - z0min
z0min = max(0, int(z0min-.1*delta))
z0max = min(sz, int(z0max+.1*delta))
z0 = np.arange(z0min, z0max)
sharey=True
else:
z0 = range(sz)
sliceY=arr[np.ix_(range(sx),[ymean],z0)][:,0,:].T
sliceX=arr[np.ix_([xmean],range(sy),z0)][0,:,:].T
vY=model.z.v2[np.ix_(range(sx),[ymean],z0)][:,0,:].T
vX=model.z.v1[np.ix_([xmean],range(sy),z0)][0,:,:].T
if rf > 1: # Refine factor
vY = refine(vY,8)
vX = refine(vX,8)
vY=(vY>=0.)
vX=(vX>=0.)
extY=np.array([xc[0],xc[sx-1],zc[z0[0]]-zc[tau1],zc[z0[-1]]-zc[tau1]])
extX=np.array([yc[0],yc[sy-1],zc[z0[0]]-zc[tau1],zc[z0[-1]]-zc[tau1]])
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(16,5), sharey=sharey)
ax1.set_aspect('equal')
ax2.set_aspect('equal')
plt.tight_layout(pad=4)
if _modelfile:
title = _modelfile.split('.')
title = '.'.join(title[:max(1,len(title)-1)])
plt.suptitle(title,fontsize=16)
vmin = min(np.min(sliceY), np.min(sliceX))
vmax = max(np.max(sliceY), np.max(sliceX))
im1 = ax1.imshow(sliceY, origin='bottom',alpha=1.,cmap=None, extent=extY, vmin=vmin, vmax=vmax)
im2 = ax2.imshow(sliceX, origin='bottom',alpha=1.,cmap=None, extent=extX, vmin=vmin, vmax=vmax)
f.subplots_adjust(right=0.8)
cbar_ax = f.add_axes([0.83, 0.15, 0.02, 0.69])
cbar = f.colorbar(im2, cax=cbar_ax)
cbar.set_label('Density [g$\,$cm$^{-3}$]', rotation=270, labelpad=20)
ax1.imshow(-vY, origin='bottom',alpha=.4,cmap=cm.gray, extent=extY)
ax2.imshow(vX, origin='bottom',alpha=.4,cmap=cm.gray, extent=extX)
xp,yp,zp = xc[x],yc[y],zc[z]-zc[tau1]
if rf > 1:
xp,yp,zp = refine(xp,rf), refine(yp,rf), refine(zp,rf)
xp = savitzky_golay(xp, rf**2+1, 3)
yp = savitzky_golay(yp, rf**2+1, 3)
zp = savitzky_golay(zp, rf**2+1, 3)
ax1.plot(xp,zp,'r')
ax2.plot(yp,zp,'r')
#ax1.plot(xc[x],zc[z]-zc[tau1],'r')
#ax2.plot(yc[y],zc[z]-zc[tau1],'r')
if dq:
xp,yp,z1p,z2p = xc, yc, zc[tau1_level[:,ymean]]-zc[tau1],zc[tau1_level[xmean,:]]-zc[tau1]
if rf > 1:
xp,yp,z1p,z2p = refine(xp,rf), refine(yp,rf), refine(z1p,rf), refine(z2p,rf)
xp = savitzky_golay(xp, rf*(rf-1)+1, 3)
yp = savitzky_golay(yp, rf*(rf-1)+1, 3)
z1p = savitzky_golay(z1p, rf*(rf-1)+1, 3)
z2p = savitzky_golay(z2p, rf*(rf-1)+1, 3)
ax1.plot(xp,z1p,'b')
ax2.plot(yp,z2p,'b')
# ax1.plot(xc,zc[tau1_level[:,ymean]]-zc[tau1],'b')
# ax2.plot(yc,zc[tau1_level[xmean,:]]-zc[tau1],'b')
ax1.set_xlim((extY[0],extY[1]))
ax1.set_ylim((extY[2],extY[3]))
ax2.set_xlim((extX[0],extX[1]))
ax2.set_ylim((extX[2],extX[3]))
ax1.set_xlabel("Spatial horizontal position (X axis) [km]")
ax2.set_xlabel("Spatial horizontal position (Y axis) [km]")
ax1.set_ylabel("Height [km]")
if boxtext:
props = dict(boxstyle='round, pad=.7, rounding_size=.2', facecolor='white', edgecolor='black', alpha=.8)
ax1.text(0.05, 0.9, boxtext, size='smaller', ha='left', va='top', transform=ax1.transAxes, bbox=props)
if show: plt.show()
return sliceY, sliceX
#%% Normalize to number of reads/variant, peak detect on raw and smoothed data
fivedata123 = forexpressionanalysis.NormalizeToReadNumbers(fivecov123fnorm)
fivedata123.to_pickle(homedir + 'MappingProtein/fivedata123.pkl')
from peakdetect import peakdet
from savitzky_golay import savitzky_golay
fivedata123smoothed = fivedata123.copy()
peaks123=pd.DataFrame()
peaksinfo123=[]
for var,group in fivedata123.groupby(level=0):
peaks123.loc[int(var), 'rawnumberofpeaks'] = len(peakdet(list(group.summedreads), 0.02, list(xval))[0])
smoothed = savitzky_golay(list(group.summedreads), 3, 1)
for binn in np.arange(1,17):
fivedata123smoothed.loc[int(var)].loc[binn] = smoothed[binn-1]
peaks123.loc[int(var), 'smoothednumberofpeaks'] = len(peakdet(smoothed, 0.02, list(xval))[0])
peaksinfo123.append(peakdet(smoothed, 0.02, list(xval)))
peakspositions123=pd.DataFrame(peaksinfo123)
peakspositions123.columns=['maxima','minima']
peakspositions123.index = peaks123.index
for var in peakspositions123.index:
if (peaks123.loc[var,'smoothednumberofpeaks']>0):
peaks123.loc[var, 'xpeak1'] = peakspositions123.ix[var,0][0][0]
peaks123.loc[var, 'ypeak1'] = peakspositions123.ix[var,0][0][1]
if (peaks123.loc[var,'smoothednumberofpeaks']>1):
peaks123.loc[var, 'xpeak2'] = peakspositions123.ix[var,0][1][0]
def calc_ld(data, W=85., plotfits=True):
W *= lbs2N
m = W / g
t = data["timeelapsed"]["values"] - data["timeelapsed"]["values"][0]
ti = np.linspace(t[0], t[-1], 100)
dt = np.diff(ti)
h = data["altitude"]["values"] * ft2m
m, b, rval, _, _ = stats.linregress(t, h)
hi = m * ti + b
dh = hi[:-1] - hi[1:]
if plotfits:
fig, ax = plt.subplots()
ax.plot(ti, hi)
ax.grid()
ax.scatter(t, h, facecolor="None")
ax.set_xlabel("Time [sec]")
ax.set_ylabel("Altitude [m]")
ax.set_title("R-squared value: %.3f" % (rval**2))
fig.savefig("htest.pdf")
r = data["roll"]["values"]
fr = interp1d(t, r, "cubic")
ri = fr(ti)
from savitzky_golay import savitzky_golay
ri = savitzky_golay(ri, window_size=41, order=2)
if plotfits:
fig, ax = plt.subplots()
ax.plot(ti, ri)
ax.grid()
ax.scatter(t, r, facecolor="None")
ax.set_xlabel("Time [sec]")
ax.set_ylabel("Roll [radians]")
fig.savefig("rtest.pdf")
ri = 0.5 * (ri[1:] + ri[:-1])
cosr = np.cos(ri)
V = np.average(data["speed"]["values"]) * kts2ms
LD = V / cosr / dh * dt
if plotfits:
fig, ax = plt.subplots()
ax.plot(ti[:-1], LD)
ax.grid()
ax.set_xlabel("Time [sec]")
ax.set_ylabel("L/D")
fig.savefig("LD_constantV.pdf")
V = data["speed"]["values"] * kts2ms
fv = interp1d(t, V, "cubic")
Vi = fv(ti)
Vi = savitzky_golay(Vi, window_size=31, order=4)
if plotfits:
fig, ax = plt.subplots()
ax.plot(ti, Vi)
ax.grid()
ax.scatter(t, V, facecolor="None")
ax.set_xlabel("Time [sec]")
ax.set_ylabel("Speed [m/s]")
fig.savefig("Vtest.pdf")
dKE = 0.5 * m * (Vi[1:]**2 - Vi[:-1]**2)
V = 0.5 * (Vi[1:] + Vi[:-1])
LD = 1. / ((m * g * dh / dt + dKE) / m / g * cosr / V)
if plotfits:
fig, ax = plt.subplots()
ax.plot(ti[:-1], LD)
ax.grid()
ax.set_xlabel("Time [sec]")
ax.set_ylabel("L/D")
# ax.set_ylim([0, 70])
fig.savefig("LD.pdf")
return LD
print(data.shape)
# replace date
dat = list(range(data.shape[0]))
data[:, 0] = scaler_simple(dat)
# test
# data[:, 1] = scaler_simple(dat)
# print(data[1101, 0])
# SCALING
# data = my_scaler(data) # (0,1)
data[:, 1] = scaler_simple(data[:, 1]) # price
data[:, 2] = scaler_simple(data[:, 2]) # value
# SMOOTHING
s = data[:, 1].copy()
data[:, 1] = savitzky_golay(data[:, 1], 41, 7)
data[:, 2] = savitzky_golay(data[:, 2], 41, 7)
# PLOT
plt.plot(np.arange(data.shape[0]), s, 'r', linewidth=2.0)
plt.plot(np.arange(data.shape[0]), data[:, 1], 'b', linewidth=2.0)
plt.show()
# save original
data = data[:, 1:3] # remove date
data = data.transpose((1, 0))
# print(data)
print(data.shape) # 2, 500 #price/volume, step
np.save('123', data)
torch.save(data, open('traindata.pt', 'wb'))
# PRINT origin
window_size = 7
order = 3
print("Savitzky Golay hyperaparameters --> Window size: %d ; Order: %d" %
(window_size, order))
data_rgb_smooth = np.zeros((3, 300, 25, 2), dtype=np.float32)
print(data_rgb.shape)
for per in range(data_rgb.shape[3]):
for joint in range(data_rgb.shape[2]):
x_coord = data_rgb[0, :, joint, per]
y_coord = data_rgb[1, :, joint, per]
z_coord = data_rgb[2, :, joint, per]
#print (x_coord)
x_coord_smooth = savitzky_golay(x_coord,
window_size,
order,
deriv=0)
#print (x_coord_smooth)
y_coord_smooth = savitzky_golay(y_coord,
window_size,
order,
deriv=0)
z_coord_smooth = savitzky_golay(z_coord,
window_size,
order,
deriv=0)
data_rgb_smooth[0, :, joint, per] = x_coord_smooth
data_rgb_smooth[1, :, joint, per] = y_coord_smooth
data_rgb_smooth[2, :, joint, per] = z_coord_smooth
seconds_arr = np.arange(0, clip_length_sec, w) * 1000
#####################################################
seconds_cuts = []
for cut in cuts:
seconds_cuts.append(np.argmin(np.abs(seconds_arr - cut * 1000)))
#print(second)
#%%
res2 = np.zeros((len(arr_test_subjects), len(seconds_arr)))
tmp_velocities = []
for cur_iter, user in enumerate(arr_test_subjects):
tmp_clip = []
tmp_arr = []
current_sec_barrier = 0
i = 0
current_trajectory = savitzky_golay(np.array(data[user][clip]['yaw']),
window_size=11,
order=3,
deriv=0)
velocity = np.gradient(current_trajectory, data[user][clip]['time']) * 1000
tmp_velocities.append(velocity)
while (i < len(data[user][clip]['time'])
and current_sec_barrier < len(seconds_arr)):
if (data[user][clip]['time'][i] >= seconds_arr[current_sec_barrier]
and tmp_clip != []):
tmp_arr.append(np.mean(np.abs(tmp_clip)))
current_sec_barrier += 1
tmp_clip = []
# if current_sec_barrier >= 159:
# break
tmp_clip.append(velocity[i])
if current_sec_barrier == 0:
tmp_clip[-1] = 0
