def bandpass(image, lshort, llong, threshold=None, truncate=4):
"""Remove noise and background variation.
Convolve with a Gaussian to remove short-wavelength noise and subtract out
long-wavelength variations, retaining features of intermediate scale.
This implementation relies on scipy.ndimage.filters.gaussian_filter, and it
is the fastest way known to the authors of performing a bandpass in
Python.
Parameters
----------
image : ndarray
lshort : small-scale cutoff (noise)
llong : large-scale cutoff
for both lshort and llong:
give a tuple value for different sizes per dimension
give int value for same value for all dimensions
when 2*lshort >= llong, no noise filtering is applied
threshold : float or integer
By default, 1 for integer images and 1/256. for float images.
Returns
-------
result : array
the bandpassed image
See Also
--------
legacy_bandpass, legacy_bandpass_fftw
"""
lshort = validate_tuple(lshort, image.ndim)
llong = validate_tuple(llong, image.ndim)
if np.any([x * 2 >= y for (x, y) in zip(lshort, llong)]):
raise ValueError("The smoothing length scale must be more" +
"than twice the noise length scale.")
if threshold is None:
if np.issubdtype(image.dtype, np.integer):
threshold = 1
else:
threshold = 1 / 256.
boxcar = image.copy()
result = np.array(image, dtype=np.float)
for axis, (sigma, smoothing) in enumerate(zip(lshort, llong)):
if smoothing > 1:
uniform_filter1d(boxcar,
2 * smoothing + 1,
axis,
output=boxcar,
mode='nearest',
cval=0)
if sigma > 0:
correlate1d(result,
gaussian_kernel(sigma, truncate),
axis,
output=result,
mode='constant',
cval=0.0)
result -= boxcar
return np.where(result > threshold, result, 0)
def types_temp_draw3(all_recs):
all_recs=types_draw
# maxi=[]
# for rec in all_recs:
# maxi.append(max(rec[1]))
# print 'Max value of y-axis is %d' %(max(maxi))
x_ticks=['sRequest','Paging', 'Attach']
with PdfPages('types_temp3.pdf') as pdf:
font_size='30'
fig = plt.figure()
ax=fig.add_subplot(111)
ax.plot(np.arange(0,288), uniform_filter1d(all_recs[0],2),'r-',marker='D',markersize=5, markevery=7,linewidth=2)
ax.plot(np.arange(0,288), uniform_filter1d(all_recs[1],2),'b--',marker='o',markersize=5, markevery=7,linewidth=2)
ax.plot(np.arange(0,288), uniform_filter1d(all_recs[2],2),'g:',marker='s',markersize=5, markevery=7,linewidth=2)
plt.subplots_adjust(top=0.95, right=0.9, left=0.21, bottom=0.21)
plt.xticks(np.arange(0,289,step=48),[0,4,8,12,16,20,24],size=font_size,fontweight='bold')
# plt.yticks(size=font_size,fontweight='bold')
plt.yticks(np.arange(0,1.1,step=0.2),[0,0.2,0.4,0.6,0.8,1],size=font_size,fontweight='bold')
plt.yticks(np.arange(0,35001,step=5000),[0,5,10,15,20,25,30,35],size=font_size,fontweight='bold')
plt.xlabel('Time of Day (h)', fontsize=font_size,fontweight='bold')
plt.ylabel('# of Records (k)',fontsize=font_size,fontweight='bold')
# plt.ylim(0,100)
plt.legend(x_ticks,fontsize=15, loc='lower right')
plt.xlim(0,289)
plt.grid()
plt.show()
pdf.savefig(fig)
def CorrMeasures(xplot,
Feats,
plotFeatures,
plotFeaturesSynch,
chSet=0,
Nmodels=1,
FeatsMulti=0):
#correlation measures between features (Feats and FeatsMulti) and shifted parameter (xplot)
MeasMI = [None] * len(
stimAmp) #mutual information [StimAmps][Realizations,features]
MeasCorr = [None] * len(stimAmp)
MeasSpCorr = [None] * len(stimAmp)
for ui in range(len(stimAmp)): #for each stimulus type
MeasMI[ui], MeasSpCorr[ui], MeasCorr[ui] = {}, {}, {}
for fkey in plotFeatures: #for each univariate feature
ysmooth = uniform_filter1d(Feats[ui][chSet][fkey],
size=avrgWinSize,
axis=0) #smooth feature series
MeasMI[ui][fkey] = mutual_info_regression(
ysmooth, xplot[stimTS]) #mutual information
MeasCorr[ui][fkey] = np.corrcoef(
ysmooth.T, xplot[stimTS])[-1, :-1] #correlation coefficient
spTemp = spearmanr(
ysmooth, xplot[stimTS])[0] #spearman correlation coefficient
if Nsims > 1:
spTemp = spTemp[-1, :-1]
MeasSpCorr[ui][fkey] = spTemp
if Nmodels > 1: #synchronization features
for fkey in plotFeaturesSynch:
ysmooth = uniform_filter1d(FeatsMulti[ui][fkey][:, :, 0],
size=avrgWinSize,
axis=0)
MeasMI[ui][fkey] = mutual_info_regression(
ysmooth, xplot[stimTS - 1])
MeasCorr[ui][fkey] = np.corrcoef(ysmooth.T,
xplot[stimTS])[-1, :-1]
spTemp = spearmanr(ysmooth, xplot[stimTS])[0]
if Nsims > 1:
spTemp = spTemp[-1, :-1]
MeasSpCorr[ui][fkey] = spTemp
MeasMI[ui]["StimAmp"] = stimAmp[ui] * np.ones(Nsims)
MeasCorr[ui]["StimAmp"] = stimAmp[ui] * np.ones(Nsims)
MeasSpCorr[ui]["StimAmp"] = stimAmp[ui] * np.ones(Nsims)
MeasMI[ui]["channel"] = chSet
MeasCorr[ui]["channel"] = chSet
MeasSpCorr[ui]["channel"] = chSet
#seizure onset time
MeasMI[ui] = pd.DataFrame(MeasMI[ui])
MeasCorr[ui] = pd.DataFrame(MeasCorr[ui])
MeasSpCorr[ui] = pd.DataFrame(MeasSpCorr[ui])
MeasMI_df = pd.concat([ii for ii in MeasMI], ignore_index=True)
MeasSpCorr_df = pd.concat([ii for ii in MeasSpCorr], ignore_index=True)
MeasCorr_df = pd.concat([ii for ii in MeasCorr], ignore_index=True)
return MeasMI_df, MeasSpCorr_df, MeasCorr_df
def boxcar(image, size):
"""Compute a rolling (boxcar) average of an image.
The kernel is square or rectangular.
Parameters
----------
image : ndarray
size : number or tuple
Size of rolling average (square or rectangular kernel) filter. Should
be odd and larger than the particle diameter.
Provide a tuple for different sizes per dimension.
Returns
-------
result : array
the rolling average image
See Also
--------
bandpass
"""
size = validate_tuple(size, image.ndim)
if not np.all([x & 1 for x in size]):
raise ValueError("Smoothing size must be an odd integer. Round up.")
result = image.copy()
for axis, _size in enumerate(size):
if _size > 1:
uniform_filter1d(result, _size, axis, output=result,
mode='nearest', cval=0)
return result
def main():
# Import pickle resuts
with open('plot_10000.pkl', 'rb') as f:
training_reward = pickle.load(f)
# Create the episode list
episode = list(range(100, 10000, 10))
# Create the average list for plotting
avg_list = [np.mean(training_reward[(i - 100):i]) for i in episode]
# Smooth the average list
avg_list_smoothed = uniform_filter1d(avg_list, 1)
# Create the standard deviation list
stdev_list = [np.std(training_reward[(i - 100):i]) for i in episode]
stdev_list_smoothed = uniform_filter1d(stdev_list, 1)
# Plot the episodic raw reward
plt.plot(episode, training_reward[100:10000:10])
tikz_save('raw_reward_plot.tikz')
plt.close()
# Plot the average reward for and standard deviation band
plt.plot(episode, avg_list)
plt.fill_between(episode,
avg_list_smoothed + stdev_list_smoothed,
avg_list_smoothed - stdev_list_smoothed,
alpha=0.3,
edgecolor="#ff7f0e")
tikz_save('average_reward_plot.tikz')
plt.close()
def show_training_results(hits, wins, lifetime, rewards, time, heatmap,
heatmap_start):
fig, ax = plt.subplots(3, 2, figsize=(12, 12))
fig.tight_layout(pad=3.0)
n = len(time) // 10
hits = uniform_filter1d(hits, size=n)
ax[0, 0].plot(time, hits, 'r') # row=0, col=0
ax[0, 0].set_title('Hits')
ax[1, 0].plot(time, wins, 'b') # row=1, col=0
ax[1, 0].set_title('Win probability')
lifetime = uniform_filter1d(lifetime, size=n)
ax[0, 1].plot(time, lifetime, 'g') # row=0, col=1
ax[0, 1].set_title('Lifetime')
rewards = uniform_filter1d(rewards, size=n)
ax[1, 1].plot(time, rewards, 'k') # row=1, col=1
ax[1, 1].set_title('Rewards')
ax[2, 0].imshow(heatmap_start, cmap='hot', interpolation='nearest')
ax[2, 0].set_title('Initial HeatMap')
ax[2, 1].imshow(heatmap, cmap='hot', interpolation='nearest')
ax[2, 1].set_title('HeatMap')
plt.show()
def boxcar(image, size):
"""Compute a rolling (boxcar) average of an image.
The kernel is square or rectangular.
Parameters
----------
image : ndarray
size : number or tuple
Size of rolling average (square or rectangular kernel) filter. Should
be larger than the particle diameter.
Provide a tuple for different sizes per dimension.
Returns
-------
result : array
the rolling average image
See Also
--------
bandpass
"""
size = validate_tuple(size, image.ndim)
result = image.copy()
for axis, _size in enumerate(size):
if _size > 1:
uniform_filter1d(result,
_size,
axis,
output=result,
mode='nearest',
cval=0)
return result
def window_stdev(arr, radius):
c1 = uniform_filter1d(arr,
radius * 2,
axis=0,
mode='constant',
origin=-radius)
c2 = uniform_filter1d(arr * arr,
radius * 2,
axis=0,
mode='constant',
origin=-radius)
return ((c2 - c1 * c1)**.5)[:-radius * 2 + 1, :-radius * 2 + 1]
def window_stdev(data, radius):
'''Explanation on https://stackoverflow.com/questions/18419871/improving-code-efficiency-standard-deviation-on-sliding-windows'''
c1 = uniform_filter1d(data.astype(np.float32),
radius * 2,
mode='constant',
origin=-radius)
c2 = uniform_filter1d(data.astype(np.float32) *
data.astype(np.float32),
radius * 2,
mode='constant',
origin=-radius)
return ((c2 - c1 * c1)**.5)[:-radius * 2 + 1]
def bandpass(image, lshort, llong, threshold=None, truncate=4):
"""Remove noise and background variation.
Convolve with a Gaussian to remove short-wavelength noise and subtract out
long-wavelength variations, retaining features of intermediate scale.
This implementation relies on scipy.ndimage.filters.gaussian_filter, and it
is the fastest way known to the authors of performing a bandpass in
Python.
Parameters
----------
image : ndarray
lshort : small-scale cutoff (noise)
llong : large-scale cutoff
for both lshort and llong:
give a tuple value for different sizes per dimension
give int value for same value for all dimensions
when 2*lshort >= llong, no noise filtering is applied
threshold : float or integer
By default, 1 for integer images and 1/256. for float images.
Returns
-------
result : array
the bandpassed image
See Also
--------
legacy_bandpass, legacy_bandpass_fftw
"""
lshort = validate_tuple(lshort, image.ndim)
llong = validate_tuple(llong, image.ndim)
if np.any([x*2 >= y for (x, y) in zip(lshort, llong)]):
raise ValueError("The smoothing length scale must be more" +
"than twice the noise length scale.")
if threshold is None:
if np.issubdtype(image.dtype, np.integer):
threshold = 1
else:
threshold = 1/256.
boxcar = image.copy()
result = np.array(image, dtype=np.float)
for axis, (sigma, smoothing) in enumerate(zip(lshort, llong)):
if smoothing > 1:
uniform_filter1d(boxcar, 2*smoothing+1, axis, output=boxcar,
mode='nearest', cval=0)
if sigma > 0:
correlate1d(result, gaussian_kernel(sigma, truncate), axis,
output=result, mode='constant', cval=0.0)
result -= boxcar
return np.where(result > threshold, result, 0)
def to_label(self,
smoothing=0.250,
percentile=75,
plot=True,
figsize=(13, 5)):
"""Generates binary label (movement vs. rest) from accelererometer data by
1. Uniformally smoothing acc data with `smoothing` seconds
2. Thresholding according to `percentile`
Args:
smoothing (float, optional): Width of smoothing window in seconds. Defaults to 0.250.
percentile (int, optional): Percentile of (smoothed signal) used to compute threshold. Defaults to 75.
plot (bool, optional): Flag indicating whether to generate plot. Defaults to True.
figsize (tuple, optional): figsize, only used when `plot = True`. Defaults to (13, 5).
Raises:
RuntimeError: Accelerometer labels have to be computed from single-channel signal. Please call .to_magnitude() method to get single accelerometer channel.
Returns:
AccData: Object of class AccData.
"""
if not self.highpassed:
_ = self.highpass()
if len(self.data.shape) > 1:
raise RuntimeError(
"Accelerometer labels have to be computed from single-channel signal. Please call .to_magnitude() method to get single accelerometer channel."
)
smoothed = uniform_filter1d(self.data, size=int(smoothing * self.fs))
thresh = np.percentile(smoothed, percentile)
self.label = (np.abs(smoothed) > thresh).astype(np.float)
label_smoothed = uniform_filter1d(self.label, size=int(self.fs))
# assert np.allclose(self.label, label_smoothed)
self.label = (label_smoothed > 0.5).astype(np.float)
if plot:
fig, (ax1, ax2) = plt.subplots(2, 1, sharex=True, figsize=figsize)
ax1.plot(self.time(), smoothed)
ax1.plot(self.time(), self.label * thresh * 5)
ax1.set_title("Smoothed + label")
ax2.plot(self.time(), self.data)
ax2.set_title("Raw data")
return self
def boxcar(image, size):
size = validate_tuple(size, image.ndim) #convert size in 2 dimension
if not np.all([x & 1 for x in size]): #check if size is odd integer
raise ValueError("Smoothing size must be an odd integer. Round up.")
result = image.copy() #store image as results
for axis, _size in enumerate(
size): #boxcar average the image about the x and y axis
if _size > 1:
uniform_filter1d(result,
_size,
axis,
output=result,
mode='nearest',
cval=0)
return result
def poc_gradient_zero_crossing(force, ret_details=False):
"""Gradient zero-crossing of indentation part
1. Apply a moving average filter to the curve
2. Compute the gradient
3. Cut off gradient at maximum with a 10 point reserve
4. Apply a moving average filter to the gradient
5. The POC is the index of the averaged gradient curve where
the values are below 1% of the gradient maximum, measured
from the indentation maximum (not from baseline).
"""
cp = np.nan
details = {}
# Perform a median filter to smooth the array
filtsize = max(5, int(force.size * .01))
y = uniform_filter1d(force, size=filtsize)
if y.size > 1:
# Cutoff at maximum plus some reserve
cutoff = y.size - np.argmax(y) + 10
grad = np.gradient(y)[:-cutoff]
if grad.size > 50:
# Use the point where the gradient becomes small enough.
gradn = uniform_filter1d(grad, size=filtsize)
thresh = 0.01 * np.max(gradn)
gradpos = gradn <= thresh
if np.sum(gradpos):
# The gradient contains positive values.
# Flip `gradpos`, because we want the first value from the
# end of the array.
# Weight with two times "filtsize//2", because we actually
# want the rolling median filter from the edge and not at the
# center of the array (and two times, because we did two
# filter operations).
cp = y.size - np.where(gradpos[::-1])[0][0] - cutoff + filtsize
if ret_details:
x = np.arange(gradn.size)
details["plot force gradient"] = [x, gradn]
details["plot threshold"] = [[x[0], x[-1]],
[thresh, thresh]]
details["plot poc"] = [[cp, cp],
[gradn.min(),
gradn.max()]]
if ret_details:
return cp, details
else:
return cp
def plot_all_train_dice(self):
self.log_dir = 'results/' + task_name + '/train/pred/GLUCOLD/' + number + '/'
self.file_name = os.path.join(self.log_dir, file_name)
fig = plt.figure(facecolor='w', figsize=(6, 6), dpi=300)
ax = fig.add_subplot(111)
for i in range(1, self.out_chn):
y_name = 'ave_dice_class_' + str(i)
x, y = self._get_data(y_name)
y = uniform_filter1d(y, size=self.average_N)
ax.plot(x, y, label='train_dice_' + str(i))
ax.legend()
ax.set_ylim((0, 1))
ax.set_xlim(left=0)
ax.set_yticks([0, 0.2, 0.4, 0.6, 0.8, 0.9, 0.95, 1])
ax.set_ylabel('dice')
ax.set_xlabel('epochs')
# plt.xlim(left=0)
# plt.gca().yaxis.set_major_locator(plt.MultipleLocator(0.1))
ax.plot([0, 90000], [0.95, 0.95], 'k-', lw=1, dashes=[2, 2])
# ax.plot([0,90000], [0.97, 0.97], 'k-', lw=1, dashes=[2,2])
plt.rc('font', size=18)
frame = plt.gca()
frame.axes.get_yaxis().set_visible(True)
frame.axes.get_xaxis().set_visible(True)
# plt.show()
plt.savefig('all_train_dice.png')
plt.close()
def plot_all_train_dice(self):
fig = plt.figure(facecolor='w', figsize=(6, 6), dpi=300)
ax = fig.add_subplot(111)
for i in range(1, self.out_chn):
if self.new_arch:
y_name = self.task_name + '_out_segmentation_dice_' + str(i)
else:
y_name = 'out_' + self.task_name + '_segmentation_dice_' + str(
i)
x, y = self._get_data(y_name)
x, y = x[::skip_number], y[::skip_number]
y = uniform_filter1d(y, size=self.average_N)
ax.plot(x, y, label='train_dice_' + str(i))
ax.legend()
ax.set_ylim((0, 1))
ax.set_xlim(left=0)
ax.set_yticks([0, 0.2, 0.4, 0.6, 0.8, 0.9, 0.95, 1])
ax.set_ylabel('dice')
ax.set_xlabel('steps')
# plt.xlim(left=0)
# plt.gca().yaxis.set_major_locator(plt.MultipleLocator(0.1))
ax.plot([0, 90000], [0.95, 0.95], 'k-', lw=1, dashes=[2, 2])
# ax.plot([0,90000], [0.97, 0.97], 'k-', lw=1, dashes=[2,2])
plt.rc('font', size=18)
# plt.show()
frame = plt.gca()
frame.axes.get_yaxis().set_visible(True)
frame.axes.get_xaxis().set_visible(True)
plt.savefig('all_train_dice.png')
plt.close()
def sp_mvg_avg(v, N, edges='nearest'):
"""
Use scipy's uniform_filter1d to calculate a moving average, see the docs at
https://docs.scipy.org/doc/scipy/reference/generated/scipy.ndimage.uniform_filter1d.html
Handles NaNs by removing them before interpolation.
Parameters
----------
v : np.ndarray
data to average.
N : int
number of samples per average.
edges : str, optional
mode of uniform_filter1d (see docs). The default is 'nearest'.
Returns
-------
avg : np.ndarray
averaged data.
"""
m = np.isfinite(v)
avg = np.empty(v.shape)
avg[~m] = np.nan
avg[m] = uniform_filter1d(v[m], size=N, mode=edges)
return avg
def calc_est_fit(estimates, conv_n, tau):
"""Make estimate by fitting exponential convergence to estimates.
"""
n = len(estimates)
if n < conv_n:
return nan, inf
# iteration number, fit function to inverse this to get k->infinity
ks = np.arange(1, len(estimates) + 1)
# smooth data with a running mean
smoothed_estimates = uniform_filter1d(estimates, n // 2)
# ignore this amount of the initial estimates and fit later part only
ni = n // 2
try:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
# fit the inverse data with a line, weighting recent ests more
popt, pcov = np.polyfit(x=(1 / ks[ni:]),
y=smoothed_estimates[ni:],
w=ks[ni:],
deg=1,
cov=True)
# estimate of function at 1 / k = 0 and standard error
est, err = popt[-1], abs(pcov[-1, -1])**0.5
except (ValueError, RuntimeError):
est, err = nan, inf
return est, err
def plot_all_val_dice(self):
fig = plt.figure(facecolor='w', figsize=(6, 6), dpi=300)
ax = fig.add_subplot(111)
for i in range(1, self.out_chn):
y_name = 'val_' + self.task_name + '_out_segmentation_dice_' + str(
i)
x, y = self._get_va_data(y_name)
y = uniform_filter1d(y, size=self.average_N)
ax.plot(x, y, label='val_dice_' + str(i))
ax.legend()
ax.set_ylim((0, 1))
ax.set_xlim(left=0)
ax.set_yticks([0, 0.2, 0.4, 0.6, 0.8, 0.9, 0.95, 1])
ax.set_ylabel('dice')
ax.set_xlabel('steps')
# plt.xlim(left=0)
# plt.gca().yaxis.set_major_locator(plt.MultipleLocator(0.1))
ax.plot([0, x[-1]], [0.95, 0.95], 'k-', lw=1, dashes=[2, 2])
# ax.plot([0,90000], [0.97, 0.97], 'k-', lw=1, dashes=[2,2])
plt.rc('font', size=18)
frame = plt.gca()
frame.axes.get_yaxis().set_visible(True)
frame.axes.get_xaxis().set_visible(True)
# plt.show()
plt.savefig(self.va_log.split('.log')[0] + 'all_valid_dice.png')
print('save fig at',
self.va_log.split('.log')[0] + 'all_valid_dice.png')
plt.close()
def parse_labels(signal_files, fs=SAMPLING_FREQ, cutoff=CUTOFF_FREQ):
cycle_labels = []
for signal_file in tqdm(signal_files):
df = pd.read_csv(signal_file, header=None)
if len(df.columns) == 1:
df = pd.read_csv(signal_file, sep=';', header=None)
hsig = df.iloc[:, -2].values
vsig = df.iloc[:, -1].values
L = []
for signal in [hsig, vsig]:
X = fft(signal)
freqs = fftfreq(len(signal)) * SAMPLING_FREQ
X = np.abs(X[1:len(signal)//2])
freqs = freqs[1:len(signal)//2]
# moving average filter
Xf = uniform_filter1d(X, size=5)
L += [np.mean(Xf[ freqs <= cutoff ])]
cycle_labels.append(np.max(L))
cycle_labels = (np.array(cycle_labels) - np.min(cycle_labels)) / (np.max(cycle_labels) - np.min(cycle_labels))
return cycle_labels
def get_chan_imp_resp(self, sch_burst):
sch_burst_bl = resize(array(sch_burst), (int(len(sch_burst)/self.OSR),self.OSR))
correlation_bl = zeros(shape(sch_burst_bl), dtype=np.complex64)
for ii in xrange(0,self.OSR):
correlation_bl[:,ii]=correlate(sch_burst_bl[:,ii],self.sync_seq_msk,'same')
correlation_bl = correlation_bl/len(self.sync_seq_msk)
power_bl_mov_avg = uniform_filter1d(abs(correlation_bl)**2,self.L+1,mode='constant',axis=0)
print "correlation_bl.argmax()",argmax(abs(correlation_bl))
print "power_bl_mov_avg.argmax()",(power_bl_mov_avg).argmax()
print 'unravel_index(correlation_bl.argmax(), correlation_bl.shape)',unravel_index(argmax(abs(correlation_bl)), correlation_bl.shape)
print 'unravel_index(power_bl_mov_avg.argmax(), power_bl_mov_avg.shape)',unravel_index(power_bl_mov_avg.argmax(), power_bl_mov_avg.shape)
(r_corrmax, c_corrmax)=unravel_index(argmax(abs(correlation_bl)), correlation_bl.shape)
(r_powmax, c_powmax)=unravel_index(power_bl_mov_avg.argmax(), power_bl_mov_avg.shape)
# correlation = zeros(shape(sch_burst))
# correlation = correlate(sch_burst, self.sync_seq_msk_interp,'same')/len(self.sync_seq_msk)
# print "pozycja maksimum",argmax(abs(correlation))
# plot(abs(hstack(correlation_bl))*1000)
## hold(True)
## plot(abs(sch_burst)*500)
## print shape(range(0,len(sch_burst),self.OSR))
## print shape(correlation_bl[:,0])
# for ii in range(0,self.OSR):
# if ii == c_powmax:
# plot(range(ii,len(correlation_bl[:,0])*self.OSR,self.OSR),power_bl_mov_avg[:,ii]*5e6,'g.')
# else:
# plot(range(ii,len(correlation_bl[:,0])*self.OSR,self.OSR),power_bl_mov_avg[:,ii]*5e6,'r.')
# show()
# figure()
print 'r_powmax: ',r_powmax
def running_average_special(year_df, running=240):
"""
Method: running_average_special(data)
Purpose: Applies the 10 running day filter to the data
Require:
data: The mdata dictionary
Version: 05/2021, MJB: Documentation
"""
i = 1
longest = 0
indi = len(year_df)
depths = list(year_df.columns)
arr = []
for depth in depths:
arr.append(uniform_filter1d(year_df[str(depth)].dropna(), size=running))
dfdelta = pd.DataFrame(np.column_stack(arr), columns=depths, index=year_df.dropna().index)
'''
for depth in depths:
# Cambiado entre otros index del data al dropna
series1 = pd.DataFrame(uniform_filter1d(year_df[str(depth)].dropna(), size=running),
index=year_df[str(depth)].dropna().index, columns=[str(depth)]).reindex(year_df.index)
i += 1
if 'dfdelta' in locals():
dfdelta = pd.merge(dfdelta, series1, right_index=True, left_index=True)
else:
dfdelta = pd.DataFrame(series1)
'''
return dfdelta
def running_average(data, running=240):
"""
Method: apply_uniform_filter(data)
Purpose: Applies the 10 running day filter to the data
Require:
data: The mdata dictionary
Version: 05/2021, MJB: Documentation
"""
df, depths, _ = list_to_df(data)
i = 1
longest = 0
indi = 0 # Checks the longest time series of all to use it as the base for the plots
for u in range(0, len(data)):
if len(data[u]['time']) > longest:
longest = len(data[u]['time'])
indi = u
for depth in depths:
# Cambiado entre otros index del data al dropna
series1 = pd.DataFrame(uniform_filter1d(df[str(depth)].dropna(), size=running),
index=df[str(depth)].dropna().index, columns=[str(depth)]).reindex(data[indi]['time'])
i += 1
if 'dfdelta' in locals():
dfdelta = pd.merge(dfdelta, series1, right_index=True, left_index=True)
else:
dfdelta = pd.DataFrame(series1)
return dfdelta
def types_temp_draw4(all_recs):
all_recs=types_draw
# maxi=[]
# for rec in all_recs:
# maxi.append(max(rec[1]))
# print 'Max value of y-axis is %d' %(max(maxi))
# x_ticks=['LTE_ATTACH','LTE_DETACH','CSFB','Service_Req','TAU','Path_Switch','LTE_PAGING']
x_ticks=['TAU','CSFB','Detach','pSwitch']
with PdfPages('types_temp4.pdf') as pdf:
font_size='30'
fig = plt.figure()
ax=fig.add_subplot(111)
ax.plot(np.arange(0,288), gaussian_filter1d(all_recs[0],5),'r-',marker='D',markersize=8, markevery=10,linewidth=2)
ax.plot(np.arange(0,288), gaussian_filter1d(all_recs[1],5),'b--',marker='o',markersize=8, markevery=10,linewidth=2)
ax.plot(np.arange(0,288), gaussian_filter1d(all_recs[2],5),'g:', marker='s',markersize=8, markevery=10,linewidth=2)
ax.plot(np.arange(0,288), uniform_filter1d(all_recs[3],2),'-.',color='#666666',marker='^',markersize=8, markevery=10,linewidth=2)
plt.subplots_adjust(top=0.73, right=0.9, left=0.21, bottom=0.21)
plt.xticks(np.arange(0,289,step=48),[0,4,8,12,16,20,24],size=font_size,fontweight='bold')
# plt.yticks(size=font_size,fontweight='bold')
plt.yticks(np.arange(0,35001,step=5000),[0,5,10,15,20,25,30,35],size=font_size,fontweight='bold')
plt.yticks(size=font_size,fontweight='bold')
plt.xlabel('Time of Day (h)', fontsize=font_size,fontweight='bold')
plt.ylabel('# of Records (k)',fontsize=font_size,fontweight='bold')
# plt.ylim(0,100)
plt.legend(x_ticks, fontsize=30, loc="upper center", ncol=2,
bbox_to_anchor=(0.5, 1.6) ,columnspacing=0.01, handletextpad=0.1)
plt.xlim(0,289)
plt.grid()
plt.show()
pdf.savefig(fig)
def apply_uniform_filter(data):
"""
Method: apply_uniform_filter(data)
Purpose: Applies the 10 running day filter to the data
Require:
data: The mdata dictionary
Version: 05/2021, MJB: Documentation
"""
df, depths = temp_difference(data)
i = 1
longest = 0
indi = 0 # Checks the longest time series of all to use it as the base for the plots
for u in range(0, len(data)):
if len(data[u]['time']) > longest:
longest = len(data[u]['time'])
indi = u
for depth in depths[:-1]:
series1 = pd.DataFrame(uniform_filter1d(df[str(depth) + "-" + str(depths[i])].dropna(), size=240),
index=df[str(depth) + "-" + str(depths[i])].dropna().index, columns=[str(depth) + "-" + str(depths[i])]).reindex(data[indi]['time'])
#series1 = pd.DataFrame(uniform_filter1d(df[str(depth) + "-" + str(depths[i])], size=240),
# index=data[indi]['time'], columns=[str(depth) + "-" + str(depths[i])])
i += 1
if 'dfdelta' in locals():
dfdelta = pd.merge(dfdelta, series1, right_index=True, left_index=True)
else:
dfdelta = pd.DataFrame(series1)
return dfdelta
def detect(self, threshold, combine=30, pre_avg=100, pre_max=30,
post_avg=30, post_max=70, delay=0):
"""
Detects the onsets.
:param threshold: threshold for peak-picking
:param combine: only report 1 onset for N miliseconds
:param pre_avg: use N miliseconds past information for moving average
:param pre_max: use N miliseconds past information for moving maximum
:param post_avg: use N miliseconds future information for mov. average
:param post_max: use N miliseconds future information for mov. maximum
:param delay: report the onset N miliseconds delayed
In online mode, post_avg and post_max are set to 0.
Implements the peak-picking method described in:
"Evaluating the Online Capabilities of Onset Detection Methods"
Sebastian Böck, Florian Krebs and Markus Schedl
Proceedings of the 13th International Society for Music Information
Retrieval Conference (ISMIR), 2012
"""
# online mode?
if self.online:
post_max = 0
post_avg = 0
# convert timing information to frames
pre_avg = int(round(self.fps * pre_avg / 1000.))
pre_max = int(round(self.fps * pre_max / 1000.))
post_max = int(round(self.fps * post_max / 1000.))
post_avg = int(round(self.fps * post_avg / 1000.))
# convert to seconds
combine /= 1000.
delay /= 1000.
# init detections
self.detections = []
# moving maximum
max_length = pre_max + post_max + 1
max_origin = int(np.floor((pre_max - post_max) / 2))
mov_max = maximum_filter1d(self.activations, max_length,
mode='constant', origin=max_origin)
# moving average
avg_length = pre_avg + post_avg + 1
avg_origin = int(np.floor((pre_avg - post_avg) / 2))
mov_avg = uniform_filter1d(self.activations, avg_length,
mode='constant', origin=avg_origin)
# detections are activation equal to the maximum
detections = self.activations * (self.activations == mov_max)
# detections must be greater or equal than the mov. average + threshold
detections = detections * (detections >= mov_avg + threshold)
# convert detected onsets to a list of timestamps
last_onset = 0
for i in np.nonzero(detections)[0]:
onset = float(i) / float(self.fps) + delay
# only report an onset if the last N miliseconds none was reported
if onset > last_onset + combine:
self.detections.append(onset)
# save last reported onset
last_onset = onset
def filter(self, average_size):
(times, values) = zip(*self.values)
#times = [t / 1000 for t in range(0, average_size // 2)] + [t + average_size // 2 / 1000 for t in times]
#values = [values[0] for v in range(0, average_size // 2)] + list(values)
filtered_data = uniform_filter1d(values, size=average_size, mode='nearest')
newdata = list(zip(times, filtered_data))
return Signal(newdata)
def rolling_mean(data, windowsize, sample_rate):
'''calculates rolling mean
Function to calculate the rolling mean (also: moving average) over the passed data.
Parameters
----------
data : 1-dimensional numpy array or list
sequence containing data over which rolling mean is to be computed
windowsize : int or float
the window size to use, in seconds
calculated as windowsize * sample_rate
sample_rate : int or float
the sample rate of the data set
Returns
-------
out : 1-d numpy array
sequence containing computed rolling mean
Examples
--------
>>> data, _ = load_exampledata(example = 1)
>>> rmean = rolling_mean(data, windowsize=0.75, sample_rate=100)
>>> rmean[100:110]
array([514.49333333, 514.49333333, 514.49333333, 514.46666667,
514.45333333, 514.45333333, 514.45333333, 514.45333333,
514.48 , 514.52 ])
'''
rol_mean = uniform_filter1d(np.asarray(data, dtype='float'),
size=int(windowsize * sample_rate))
return rol_mean
def get_smoothed_running_minimum(timeseries, tau1=30, tau2=100):
result = minimum_filter1d(uniform_filter1d(timeseries,
tau1,
mode='nearest'),
tau2,
mode='reflect')
return result
def legacy_bandpass_fftw(image, lshort, llong, threshold=None):
"""Remove noise and background variation.
Convolve with a Gaussian to remove short-wavelength noise and subtract out
long-wavelength variations, retaining features of intermediate scale.
This implementation performs a Fourier transform using FFTW
(Fastest Fourier Transform in the West). Without FFTW and pyfftw, it
will raise an ImportError
In benchmarks using typical inputs, it was found to be slower than the
``bandpass`` function in this module.
Parameters
----------
image : ndarray
lshort : small-scale cutoff (noise)
llong : large-scale cutoff
for both lshort and llong:
give a tuple value for different sizes per dimension
give int value for same value for all dimensions
when 2*lshort >= llong, no noise filtering is applied
threshold : float or integer
By default, 1 for integer images and 1/256. for float images.
Returns
-------
result : array
the bandpassed image
See Also
--------
bandpass, legacy_bandpass
"""
if not FFTW_AVAILABLE:
raise ImportError("This implementation requires pyfftw.")
lshort = validate_tuple(lshort, image.ndim)
llong = validate_tuple(llong, image.ndim)
if np.any([x * 2 >= y for (x, y) in zip(lshort, llong)]):
raise ValueError("The smoothing length scale must be more" +
"than twice the noise length scale.")
if threshold is None:
if np.issubdtype(image.dtype, np.integer):
threshold = 1
else:
threshold = 1 / 256.
# Perform a rolling average (boxcar) with kernel size = 2*llong + 1
boxcar = np.asarray(image)
for (axis, size) in enumerate(llong):
boxcar = uniform_filter1d(boxcar,
size * 2 + 1,
axis,
mode='nearest',
cval=0)
# Perform a gaussian filter
gaussian = ifftn(fourier_gaussian(fftn(image), lshort)).real
result = gaussian - boxcar
return np.where(result > threshold, result, 0)
def extract_lines(pc, window_size=3, dot_min=0.95, **kwargs):
"""Extract a first approximation of all lines in 2D line string
Args:
pc (ndarray): 2D Line String
window_size (int, optional): Smoothing window for Line. Defaults to 3.
dot_min (float, optional): Minimum dot prodcut for joining lines. Defaults to 0.88.
Returns:
List: List of lines(dict)
"""
t1 = time.perf_counter()
pc_shift = np.roll(pc, -1, axis=0)
diff = pc_shift - pc
diff_vec, length = normalized(diff)
idx_max = np.argmax(length)
assert idx_max != 0 or idx_max != length.shape[0] - \
1, "LineString is not continuously connected"
t2 = time.perf_counter()
x = uniform_filter1d(diff[:, 0], size=window_size)
y = uniform_filter1d(diff[:, 1], size=window_size)
diff_smooth = np.column_stack((x, y))
t3 = time.perf_counter()
diff_smooth, length = normalized(diff_smooth)
diff_smooth_shift = np.roll(diff_smooth, -1, axis=0)
acos = np.einsum('ij, ij->i', diff_smooth, diff_smooth_shift)
t4 = time.perf_counter()
mask = acos > dot_min
np_diff = np.diff(np.hstack(([False], mask, [False])))
idx_pairs = np.where(np_diff)[0].reshape(-1, 2)
t5 = time.perf_counter()
logging.debug("IDX Pairs %s", (idx_pairs))
fit_lines = [fit_line(pc, idx) for idx in idx_pairs if idx[1] - idx[0] > 1]
t6 = time.perf_counter()
# ms1 = (t2-t1) * 1000
# ms2 = (t3-t2) * 1000
# ms3 = (t4-t3) * 1000
# ms4 = (t5-t4) * 1000
# ms5 = (t6-t5) * 1000
# print(ms1, ms2, ms3, ms4, ms5)
return fit_lines
def detrend_ma(data, window):
from scipy.ndimage.filters import uniform_filter1d
for i in range(data.shape[1]):
data[:, i] = data[:, i] - uniform_filter1d(data[:, i], size=window)
return data
def window_stdev(arr, radius):
from scipy.ndimage.filters import uniform_filter1d
#array = arr, radius = half width of window in bins
#windows mean
c1 = uniform_filter1d(arr,
2 * radius,
mode='constant',
origin=-radius,
axis=0)
#windowed square mean
c2 = uniform_filter1d(arr * arr,
2 * radius,
mode='constant',
origin=-radius,
axis=0)
#std = windowed square mean - square(windowed mean)
return ((c2 - c1 * c1)**.5)[:-2 * radius + 1]
def _running_mean(self, ):
"""
From https://stackoverflow.com/a/43200476/13186064
"""
return uniform_filter1d(
self._reward_array,
self.window_size,
mode='constant',
origin=-(self.window_size // 2))[:-(self.window_size - 1)]
def running_min(X, tau1, tau2):
###DEBUGGING IMPLEMENTATION###
# return minimum_filter1d(X,tau2,mode = 'nearest')
###PREVIOUS IMPLEMENTATION###
mode = 'nearest'
result = minimum_filter1d(uniform_filter1d(X, tau1, mode=mode),
tau2,
mode='reflect')
return result
def legacy_bandpass_fftw(image, lshort, llong, threshold=None):
"""Remove noise and background variation.
Convolve with a Gaussian to remove short-wavelength noise and subtract out
long-wavelength variations, retaining features of intermediate scale.
This implementation performs a Fourier transform using FFTW
(Fastest Fourier Transform in the West). Without FFTW and pyfftw, it
will raise an ImportError
In benchmarks using typical inputs, it was found to be slower than the
``bandpass`` function in this module.
Parameters
----------
image : ndarray
lshort : small-scale cutoff (noise)
llong : large-scale cutoff
for both lshort and llong:
give a tuple value for different sizes per dimension
give int value for same value for all dimensions
when 2*lshort >= llong, no noise filtering is applied
threshold : float or integer
By default, 1 for integer images and 1/256. for float images.
Returns
-------
result : array
the bandpassed image
See Also
--------
bandpass, legacy_bandpass
"""
if not FFTW_AVAILABLE:
raise ImportError("This implementation requires pyfftw.")
lshort = validate_tuple(lshort, image.ndim)
llong = validate_tuple(llong, image.ndim)
if np.any([x*2 >= y for (x, y) in zip(lshort, llong)]):
raise ValueError("The smoothing length scale must be more" +
"than twice the noise length scale.")
if threshold is None:
if np.issubdtype(image.dtype, np.integer):
threshold = 1
else:
threshold = 1/256.
# Perform a rolling average (boxcar) with kernel size = 2*llong + 1
boxcar = np.asarray(image)
for (axis, size) in enumerate(llong):
boxcar = uniform_filter1d(boxcar, size*2+1, axis, mode='nearest',
cval=0)
# Perform a gaussian filter
gaussian = ifftn(fourier_gaussian(fftn(image), lshort)).real
result = gaussian - boxcar
return np.where(result > threshold, result, 0)
def applyFunction(data,f):
if re.match("^log$",f):
return np.log(data);
elif re.match("^log10$",f):
return np.log10(data);
elif re.match("^log2$",f):
return np.log2(data);
elif re.match("^exp$",f):
return np.exp(data);
else:
m = re.match("^([<>=!]+)([-.eE0-9]+)$",f);
if m:
if m.group(1)=="<=":
return (data<=float(m.group(2))).astype(int);
elif m.group(1)==">=":
return (data>=float(m.group(2))).astype(int);
elif m.group(1)=="<":
return (data<float(m.group(2))).astype(int);
elif m.group(1)==">":
return (data>float(m.group(2))).astype(int);
elif m.group(1)=="!=":
return (data!=float(m.group(2))).astype(int);
elif m.group(1)=="==":
return (data==float(m.group(2))).astype(int);
m = re.match("^([\^\*\+\-\/])([-.eE0-9]+)$",f);
if m:
if m.group(1)=="^":
return data**float(m.group(2));
elif m.group(1)=="*":
return data*float(m.group(2));
elif m.group(1)=="+":
return data+float(m.group(2));
elif m.group(1)=="/":
return data/float(m.group(2));
elif m.group(1)=="-":
return data-float(m.group(2));
m = re.match("^default([-.eE0-9]*)$",f);
if m:
invalid = np.isnan( data )
data2 = data;
data2[ invalid ] = float(m.group(1));
return data2;
m = re.match("^smoothG([-.eE0-9]*)$",f);
if m:
return gaussian_filter(data, float(m.group(1)), mode='reflect', truncate=4.0)
m = re.match("^smoothU([-.eE0-9]*)$",f);
if m:
return uniform_filter1d(data, int(m.group(1)), mode='reflect')
m = re.match("^smoothZ([-.eE0-9]*)$",f);
if m:
return z_scoreize_online(data, int(m.group(1)))
m = re.match("^pow([-.eE0-9]*)$",f);
if m:
return np.power(float(m.group(1)),data)
raise Exception("Unknown function provided: %s"%f)
def compute_mask(self, frac=0.5, filtwidth=5):
"""
return a mask showing where noise spikes to frac over the local
background
"""
smoothed_noise = gaussian_filter1d(self.error, filtwidth)
mask = ((self.error >= (1 + frac) * smoothed_noise)
| (self.error <= 0)
| (self.error >= self.large_err)
| (self.spectrum == 0))
mask_filtered = uniform_filter1d(mask.astype(float),
max(3, filtwidth))
return mask_filtered > 0.5 / filtwidth
def local_average(f,n,size=None,origin=None,mode='reflect'):
"""
get the local, windowed function of the average, +/- n
Args:
f: what we want the stdev of
n: window size (in either direction)
mode: see uniform_filter1d
Returns:
array, same size as f, with the dat we want
"""
if (size is None):
size = 2*n
if (origin is None):
origin = 0
return uniform_filter1d(f, size=size, mode=mode, origin=origin)
def Du(x, y, Np, ndiv=1, axis=-1, mode='strip', cval=0.):
"""
Does the central derrivative after performing a uniform
filter (average of Np points).
Date needs to be equally spaced and Np odd.
"""
strip = False
if mode == 'strip':
strip = True
mode = 'reflect'
y = filters.uniform_filter1d(y, Np, axis=axis, mode=mode, cval=cval)
#if Np%2 == 0: x -= (x[1]-x[0])/2 # or x = filters.uniform_filter1d(x, Np, axis=axis, mode='nearest')
if strip:
x, y = _do_strip(x, y, Np, axis=axis)
Np = ndiv*2+1
x, D = Dn(x, y, Np, ndiv=ndiv, axis=axis, mode=mode, cval=cval)
if strip:
x, D = _do_strip(x, D, Np, axis=axis)
return x, D
def bandpass(image, lshort, llong, threshold=None):
"""Convolve with a Gaussian to remove short-wavelength noise,
and subtract out long-wavelength variations,
retaining features of intermediate scale.
Parmeters
---------
image : ndarray
lshort : small-scale cutoff (noise)
llong : large-scale cutoff
for both lshort and llong:
give a tuple value for different sizes per dimension
give int value for same value for all dimensions
when 2*lshort >= llong, no noise filtering is applied
threshold : float or integer
By default, 1 for integer images and 1/256. for float images.
Returns
-------
ndarray, the bandpassed image
"""
lshort = validate_tuple(lshort, image.ndim)
llong = validate_tuple(llong, image.ndim)
if np.any([x*2 >= y for (x, y) in zip(lshort, llong)]):
raise ValueError("The smoothing length scale must be more" +
"than twice the noise length scale.")
if threshold is None:
if np.issubdtype(image.dtype, np.integer):
threshold = 1
else:
threshold = 1/256.
settings = dict(mode='nearest', cval=0)
axes = range(image.ndim)
sizes = [x*2+1 for x in llong]
boxcar = np.asarray(image)
for (axis, size) in zip(axes, sizes):
boxcar = uniform_filter1d(boxcar, size, axis, **settings)
gaussian = ifftn(fourier_gaussian(fftn(image), lshort)).real
result = gaussian - boxcar
return np.where(result > threshold, result, 0)
def threshold_minimum(image, nbins=256, max_iter=10000):
"""Return threshold value based on minimum method.
The histogram of the input `image` is computed and smoothed until there are
only two maxima. Then the minimum in between is the threshold value.
Parameters
----------
image : (M, N) ndarray
Input image.
nbins : int, optional
Number of bins used to calculate histogram. This value is ignored for
integer arrays.
max_iter: int, optional
Maximum number of iterations to smooth the histogram.
Returns
-------
threshold : float
Upper threshold value. All pixels with an intensity higher than
this value are assumed to be foreground.
Raises
------
RuntimeError
If unable to find two local maxima in the histogram or if the
smoothing takes more than 1e4 iterations.
References
----------
.. [1] C. A. Glasbey, "An analysis of histogram-based thresholding
algorithms," CVGIP: Graphical Models and Image Processing,
vol. 55, pp. 532-537, 1993.
.. [2] Prewitt, JMS & Mendelsohn, ML (1966), "The analysis of cell
images", Annals of the New York Academy of Sciences 128: 1035-1053
DOI:10.1111/j.1749-6632.1965.tb11715.x
Examples
--------
>>> from skimage.data import camera
>>> image = camera()
>>> thresh = threshold_minimum(image)
>>> binary = image > thresh
"""
def find_local_maxima_idx(hist):
# We can't use scipy.signal.argrelmax
# as it fails on plateaus
maximum_idxs = list()
direction = 1
for i in range(hist.shape[0] - 1):
if direction > 0:
if hist[i + 1] < hist[i]:
direction = -1
maximum_idxs.append(i)
else:
if hist[i + 1] > hist[i]:
direction = 1
return maximum_idxs
hist, bin_centers = histogram(image.ravel(), nbins)
smooth_hist = np.copy(hist).astype(np.float64)
for counter in range(max_iter):
smooth_hist = ndif.uniform_filter1d(smooth_hist, 3)
maximum_idxs = find_local_maxima_idx(smooth_hist)
if len(maximum_idxs) < 3:
break
if len(maximum_idxs) != 2:
raise RuntimeError('Unable to find two maxima in histogram')
elif counter == max_iter - 1:
raise RuntimeError('Maximum iteration reached for histogram'
'smoothing')
# Find lowest point between the maxima
threshold_idx = np.argmin(smooth_hist[maximum_idxs[0]:maximum_idxs[1] + 1])
return bin_centers[maximum_idxs[0] + threshold_idx]
def _n_profiles_H_V(arrayH, arrayV, virtual_pixelsize,
zlabel=r'z',
titleH='Horiz', titleV='Vert',
nprofiles=5, filter_width=0,
remove1stOrderDPC=False,
saveFileSuf='',
saveFigFlag=True):
xxGrid, yyGrid = wpu.grid_coord(arrayH, virtual_pixelsize)
fit_coefs = [[], []]
data2saveH = None
data2saveV = None
labels_H = None
labels_V = None
plt.rcParams['lines.markersize'] = 4
plt.rcParams['lines.linewidth'] = 2
# Horizontal
if np.all(np.isfinite(arrayH)):
plt.figure(figsize=(12, 12*9/16))
xvec = xxGrid[0, :]
data2saveH = np.c_[xvec]
header = ['x [m]']
if filter_width != 0:
arrayH_filtered = uniform_filter1d(arrayH, filter_width, 0)
else:
arrayH_filtered = arrayH
ls_cycle, lc_jet = wpu.line_style_cycle(['-'], ['o', 's', 'd', '^'],
ncurves=nprofiles,
cmap_str='gist_rainbow_r')
lc = []
labels_H = []
for i, row in enumerate(np.linspace(filter_width//2,
np.shape(arrayV)[0]-filter_width//2-1,
nprofiles + 2, dtype=int)):
if i == 0 or i == nprofiles + 1:
continue
yvec = arrayH_filtered[row, :]
lc.append(next(lc_jet))
p01 = np.polyfit(xvec, yvec, 1)
fit_coefs[0].append(p01)
if remove1stOrderDPC:
yvec -= p01[0]*xvec + p01[1]
plt.plot(xvec*1e6, yvec, next(ls_cycle), color=lc[i-1],
label=str(row))
if not remove1stOrderDPC:
plt.plot(xvec*1e6, p01[0]*xvec + p01[1], '--',
color=lc[i-1], lw=3)
data2saveH = np.c_[data2saveH, yvec]
header.append(str(row))
labels_H.append(str(row))
if remove1stOrderDPC:
titleH = titleH + ', 2nd order removed'
plt.legend(title='Pixel Y', loc=0, fontsize=12)
plt.xlabel(r'x [$\mu m$]', fontsize=18)
plt.ylabel(zlabel, fontsize=18)
plt.title(titleH + ', Filter Width = {:d} pixels'.format(filter_width),
fontsize=20)
if saveFigFlag:
wpu.save_figs_with_idx(saveFileSuf + '_H')
plt.show(block=False)
header.append(zlabel + ', Filter Width = {:d} pixels'.format(filter_width))
wpu.save_csv_file(data2saveH,
wpu.get_unique_filename(saveFileSuf +
'_WF_profiles_H', 'csv'),
headerList=header)
plt.figure(figsize=(12, 12*9/16))
plt.imshow(arrayH, cmap='RdGy',
vmin=wpu.mean_plus_n_sigma(arrayH, -3),
vmax=wpu.mean_plus_n_sigma(arrayH, 3))
plt.xlabel('Pixel')
plt.ylabel('Pixel')
plt.title(titleH + ', Profiles Position')
currentAxis = plt.gca()
_, lc_jet = wpu.line_style_cycle(['-'], ['o', 's', 'd', '^'],
ncurves=nprofiles,
cmap_str='gist_rainbow_r')
for i, row in enumerate(np.linspace(filter_width//2,
np.shape(arrayV)[0]-filter_width//2-1,
nprofiles + 2, dtype=int)):
if i == 0 or i == nprofiles + 1:
continue
currentAxis.add_patch(Rectangle((-.5, row - filter_width//2 - .5),
np.shape(arrayH)[1], filter_width,
facecolor=lc[i-1], alpha=.5))
plt.axhline(row, color=lc[i-1])
if saveFigFlag:
wpu.save_figs_with_idx(saveFileSuf + '_H')
plt.show(block=True)
# Vertical
if np.all(np.isfinite(arrayV)):
plt.figure(figsize=(12, 12*9/16))
xvec = yyGrid[:, 0]
data2saveV = np.c_[xvec]
header = ['y [m]']
if filter_width != 0:
arrayV_filtered = uniform_filter1d(arrayV, filter_width, 1)
else:
arrayV_filtered = arrayV
ls_cycle, lc_jet = wpu.line_style_cycle(['-'], ['o', 's', 'd', '^'],
ncurves=nprofiles,
cmap_str='gist_rainbow_r')
lc = []
labels_V = []
for i, col in enumerate(np.linspace(filter_width//2,
np.shape(arrayH)[1]-filter_width//2-1,
nprofiles + 2, dtype=int)):
if i == 0 or i == nprofiles + 1:
continue
yvec = arrayV_filtered[:, col]
lc.append(next(lc_jet))
p10 = np.polyfit(xvec, yvec, 1)
fit_coefs[1].append(p10)
if remove1stOrderDPC:
yvec -= p10[0]*xvec + p10[1]
plt.plot(xvec*1e6, yvec, next(ls_cycle), color=lc[i-1],
label=str(col))
if not remove1stOrderDPC:
plt.plot(xvec*1e6, p10[0]*xvec + p10[1], '--',
color=lc[i-1], lw=3)
data2saveV = np.c_[data2saveV, yvec]
header.append(str(col))
labels_V.append(str(col))
if remove1stOrderDPC:
titleV = titleV + ', 2nd order removed'
plt.legend(title='Pixel X', loc=0, fontsize=12)
plt.xlabel(r'y [$\mu m$]', fontsize=18)
plt.ylabel(zlabel, fontsize=18)
plt.title(titleV + ', Filter Width = {:d} pixels'.format(filter_width),
fontsize=20)
if saveFigFlag:
wpu.save_figs_with_idx(saveFileSuf + '_Y')
plt.show(block=False)
header.append(zlabel + ', Filter Width = {:d} pixels'.format(filter_width))
wpu.save_csv_file(data2saveV,
wpu.get_unique_filename(saveFileSuf +
'_WF_profiles_V', 'csv'),
headerList=header)
plt.figure(figsize=(12, 12*9/16))
plt.imshow(arrayV, cmap='RdGy',
vmin=wpu.mean_plus_n_sigma(arrayV, -3),
vmax=wpu.mean_plus_n_sigma(arrayV, 3))
plt.xlabel('Pixel')
plt.ylabel('Pixel')
plt.title(titleV + ', Profiles Position')
currentAxis = plt.gca()
for i, col in enumerate(np.linspace(filter_width//2,
np.shape(arrayH)[1]-filter_width//2-1,
nprofiles + 2, dtype=int)):
if i == 0 or i == nprofiles + 1:
continue
currentAxis.add_patch(Rectangle((col - filter_width//2 - .5, -.5),
filter_width, np.shape(arrayV)[0],
facecolor=lc[i-1], alpha=.5))
plt.axvline(col, color=lc[i-1])
if saveFigFlag:
wpu.save_figs_with_idx(saveFileSuf + '_Y')
plt.show(block=True)
return data2saveH, data2saveV, labels_H, labels_V, fit_coefs
from sys import argv
from scipy.ndimage import filters
import re
if len(argv) < 2:
print('Usage: %s logfile' %argv[0])
exit(1)
loss = []
iter = []
additional_loss = {}
lines = open(argv[1],'r').read()
r = re.compile('Train net output .* (.*) = .* = (.*) loss\)')
for l in lines.split('\n'):
s = l.split()
losses = r.findall(l)
if losses:
if not losses[0][0] in additional_loss:
additional_loss[losses[0][0]] = []
additional_loss[losses[0][0]].append(float(losses[0][1]))
if len(s)>4 and s[-2] == '=' and s[-3] == 'loss':
loss.append(float(s[-1]))
iter.append(float(s[-4][:-1]))
plot(iter, loss, label='loss')
plot(iter, filters.uniform_filter1d(loss, 100), label='smooth loss')
for i in additional_loss:
plot(iter[:len(additional_loss[i])], additional_loss[i], label=i)
legend()
show()
results.append(onevsallresult)
if test is not None:
score = []
results = [list(res) for res in zip(*results)]
for ind, result in enumerate(results):
if result.count(1) == 1:
if target[ind] == labels[result.index(1)]:
score.append(1)
else:
score.append(0)
else:
score.append(0)
print "score : ", float(sum(score))/len(score)
# print "correct : ", float(len(target) - np.sum(np.abs(np.subtract(onevsall, onevsallresult))))/len(target)
# print "correct : ", len(target) - np.sum(np.abs(np.subtract(onevsall, onevsallresult))), "/", len(target)
onevsall(train, target, test)
# Pourquoi ca ne marche pas ?
# Resultat non lineairement separable ? Reseau pas assez 'deep' ? Entrainement pas assez grand ?
train = [uniform_filter1d(np.absolute(np.fft.fft(t)), 7) for t in train]
test = [uniform_filter1d(np.absolute(np.fft.fft(t)), 7) for t in test]
onevsall(train, target, test)
def detect(self, threshold, combine=0.03, pre_avg=0.15, pre_max=0.01,
post_avg=0, post_max=0.05, delay=0):
"""
Detects the onsets.
:param threshold: threshold for peak-picking
:param combine: only report 1 onset for N seconds
:param pre_avg: use N seconds past information for moving average
:param pre_max: use N seconds past information for moving maximum
:param post_avg: use N seconds future information for moving average
:param post_max: use N seconds future information for moving maximum
:param delay: report the onset N seconds delayed
In online mode, post_avg and post_max are set to 0.
Implements the peak-picking method described in:
"Evaluating the Online Capabilities of Onset Detection Methods"
Sebastian Böck, Florian Krebs and Markus Schedl
Proceedings of the 13th International Society for Music Information
Retrieval Conference (ISMIR), 2012
"""
# online mode?
if self.online:
post_max = 0
post_avg = 0
# convert timing information to frames
pre_avg = int(round(self.fps * pre_avg))
pre_max = int(round(self.fps * pre_max))
post_max = int(round(self.fps * post_max))
post_avg = int(round(self.fps * post_avg))
# convert to seconds
combine /= 1000.
delay /= 1000.
# init detections
self.detections = []
# moving maximum
max_length = pre_max + post_max + 1
max_origin = int(np.floor((pre_max - post_max) / 2))
mov_max = maximum_filter1d(self.activations, max_length,
mode='constant', origin=max_origin)
# moving average
avg_length = pre_avg + post_avg + 1
avg_origin = int(np.floor((pre_avg - post_avg) / 2))
mov_avg = uniform_filter1d(self.activations, avg_length,
mode='constant', origin=avg_origin)
# detections are activation equal to the moving maximum
detections = self.activations * (self.activations == mov_max)
# detections must be greater or equal than the mov. average + threshold
detections *= (detections >= mov_avg + threshold)
# convert detected onsets to a list of timestamps
detections = np.nonzero(detections)[0].astype(np.float) / self.fps
# shift if necessary
if delay != 0:
detections += delay
# always use the first detection and all others if none was reported
# within the last `combine` seconds
if detections.size > 1:
# filter all detections which occur within `combine` seconds
combined_detections = detections[1:][np.diff(detections) > combine]
# add them after the first detection
self.detections = np.append(detections[0], combined_detections)
else:
self.detections = detections
def threshold_minimum(image, nbins=256, bias='min', max_iter=10000):
"""Return threshold value based on minimum method.
The histogram of the input `image` is computed and smoothed until there are
only two maxima. Then the minimum in between is the threshold value.
Parameters
----------
image : (M, N) ndarray
Input image.
nbins : int, optional
Number of bins used to calculate histogram. This value is ignored for
integer arrays.
bias : {'min', 'mid', 'max'}, optional
'min', 'mid', 'max' return lowest, middle, or highest pixel value
with minimum histogram value.
max_iter: int, optional
Maximum number of iterations to smooth the histogram.
Returns
-------
threshold : float
Upper threshold value. All pixels with an intensity higher than
this value are assumed to be foreground.
Raises
------
RuntimeError
If unable to find two local maxima in the histogram or if the
smoothing takes more than 1e4 iterations.
References
----------
.. [1] Prewitt, JMS & Mendelsohn, ML (1966), "The analysis of cell images",
Annals of the New York Academy of Sciences 128: 1035-1053
Examples
--------
>>> from skimage.data import camera
>>> image = camera()
>>> thresh = threshold_minimum(image)
>>> binary = image > thresh
"""
def find_local_maxima(hist):
# We can't use scipy.signal.argrelmax
# as it fails on plateaus
maximums = list()
direction = 1
for i in range(hist.shape[0] - 1):
if direction > 0:
if hist[i + 1] < hist[i]:
direction = -1
maximums.append(i)
else:
if hist[i + 1] > hist[i]:
direction = 1
return maximums
if bias not in ('min', 'mid', 'max'):
raise ValueError("Unknown bias: {0}".format(bias))
hist, bin_centers = histogram(image.ravel(), nbins)
smooth_hist = np.copy(hist)
for counter in range(max_iter):
smooth_hist = ndif.uniform_filter1d(smooth_hist, 3)
maximums = find_local_maxima(smooth_hist)
if len(maximums) < 3:
break
if len(maximums) != 2:
raise RuntimeError('Unable to find two maxima in histogram')
elif counter == max_iter - 1:
raise RuntimeError('Maximum iteration reached for histogram'
'smoothing')
# Find lowest point between the maxima, biased to the low end (min)
minimum = smooth_hist[maximums[0]]
threshold = maximums[0]
for i in range(maximums[0], maximums[1]+1):
if smooth_hist[i] < minimum:
minimum = smooth_hist[i]
threshold = i
if bias == 'min':
return bin_centers[threshold]
else:
upper_bound = threshold
while smooth_hist[upper_bound] == smooth_hist[threshold]:
upper_bound += 1
upper_bound -= 1
if bias == 'max':
return bin_centers[upper_bound]
elif bias == 'mid':
return bin_centers[(threshold + upper_bound) // 2]
