def show(self):
stopresults = self.get_stopresults()
print 'show', stopresults
# print ' dir(vehicleman)',dir(vehicleman)
print ' len(stopresults)', len(stopresults)
if len(stopresults) > 0:
i_fig = 0
plt.close("all")
#i_fig +=1;fig = plt.figure(i_fig)
# self.plot_waiting_person(fig)
i_fig += 1
fig = plt.figure(i_fig)
self.plot_waiting_person_number(fig)
#i_fig +=1;fig = plt.figure(i_fig)
# self.plot_waiting_person_time(fig)
i_fig += 1
fig = plt.figure(i_fig)
self.plot_waiting_person_number_stop(fig)
i_fig += 1
fig = plt.figure(i_fig)
self.plot_flow_stop(fig)
#i_fig +=1;fig = plt.figure(i_fig)
# self.plot_flows_compare(fig)
#i_fig +=1;fig = plt.figure(i_fig)
# self.plot_flows_compare_stop(fig)
plt.show()
def test_plot_acf_kwargs():
# Just test that it runs.
fig = plt.figure()
ax = fig.add_subplot(111)
ar = np.r_[1., -0.9]
ma = np.r_[1., 0.9]
armaprocess = tsp.ArmaProcess(ar, ma)
rs = np.random.RandomState(1234)
acf = armaprocess.generate_sample(100, distrvs=rs.standard_normal)
buff = BytesIO()
plot_acf(acf, ax=ax)
fig.savefig(buff, format='rgba')
plt.close(fig)
buff_with_vlines = BytesIO()
fig_with_vlines = plt.figure()
ax = fig_with_vlines.add_subplot(111)
vlines_kwargs = {'linestyles': 'dashdot'}
plot_acf(acf, ax=ax, vlines_kwargs=vlines_kwargs)
fig_with_vlines.savefig(buff_with_vlines, format='rgba')
plt.close(fig_with_vlines)
buff.seek(0)
buff_with_vlines.seek(0)
plain = buff.read()
with_vlines = buff_with_vlines.read()
assert_(with_vlines != plain)
def show_plot(X, y, n_neighbors=10, h=0.2):
# Create color maps
cmap_light = ListedColormap(['#FFAAAA', '#AAFFAA', '#AAAAFF','#FFAAAA', '#AAFFAA', '#AAAAFF','#FFAAAA', '#AAFFAA', '#AAAAFF','#AAAAFF'])
cmap_bold = ListedColormap(['#FF0000', '#00FF00', '#0000FF','#FF0000','#FF0000','#FF0000','#FF0000','#FF0000','#FF0000','#FF0000',])
for weights in ['uniform', 'distance']:
# we create an instance of Neighbours Classifier and fit the data.
clf = neighbors.KNeighborsClassifier(n_neighbors, weights=weights)
clf.fit(X, y)
clf.n_neighbors = n_neighbors
# Plot the decision boundary. For that, we will assign a color to each
# point in the mesh [x_min, x_max]x[y_min, y_max].
x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
np.arange(y_min, y_max, h))
Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])
# Put the result into a color plot
Z = Z.reshape(xx.shape)
plt.figure()
plt.pcolormesh(xx, yy, Z, cmap=cmap_light)
# Plot also the training points
plt.scatter(X[:, 0], X[:, 1], c=y, cmap=cmap_bold)
plt.xlim(xx.min(), xx.max())
plt.ylim(yy.min(), yy.max())
plt.title("3-Class classification (k = %i, weights = '%s')"
% (n_neighbors, weights))
plt.show()
def show(self, rescale=True, ax=None):
"""Visualization of a design matrix
Parameters
----------
rescale: bool, optional
rescale columns magnitude for visualization or not
ax: axis handle, optional
Handle to axis onto which we will draw design matrix
Returns
-------
ax: axis handle
"""
import matplotlib.pyplot as plt
# normalize the values per column for better visualization
x = self.matrix.copy()
if rescale:
x = x / np.sqrt(np.sum(x ** 2, 0))
if ax is None:
plt.figure()
ax = plt.subplot(1, 1, 1)
ax.imshow(x, interpolation='Nearest', aspect='auto')
ax.set_label('conditions')
ax.set_ylabel('scan number')
if self.names is not None:
ax.set_xticks(range(len(self.names)))
ax.set_xticklabels(self.names, rotation=60, ha='right')
return ax
def plot_predict_is(self,h=5,**kwargs):
""" Plots forecasts with the estimated model against data
(Simulated prediction with data)
Parameters
----------
h : int (default : 5)
How many steps to forecast
Returns
----------
- Plot of the forecast against data
"""
figsize = kwargs.get('figsize',(10,7))
plt.figure(figsize=figsize)
date_index = self.index[-h:]
predictions = self.predict_is(h)
data = self.data[-h:]
t_params = self.transform_z()
plt.plot(date_index,np.abs(data-t_params[-1]),label='Data')
plt.plot(date_index,predictions,label='Predictions',c='black')
plt.title(self.data_name)
plt.legend(loc=2)
plt.show()
def scree_plot(pca_obj, fname=None):
'''
Scree plot for variance & cumulative variance by component from PCA.
Arguments:
- pca_obj: a fitted sklearn PCA instance
- fname: path to write plot to file
Output:
- scree plot
'''
components = pca_obj.n_components_
variance = pca.explained_variance_ratio_
plt.figure()
plt.plot(np.arange(1, components + 1), np.cumsum(variance), label='Cumulative Variance')
plt.plot(np.arange(1, components + 1), variance, label='Variance')
plt.xlim([0.8, components]); plt.ylim([0.0, 1.01])
plt.xlabel('No. Components', labelpad=11); plt.ylabel('Variance Explained', labelpad=11)
plt.legend(loc='best')
plt.tight_layout()
if fname is not None:
plt.savefig(fname)
plt.close()
else:
plt.show()
return
def draw_ranges_for_parameters(data, title='', save_path='./pictures/'):
parameters = data.columns.values.tolist()
# remove flight name parameter
for idx, parameter in enumerate(parameters):
if parameter == 'flight_name':
del parameters[idx]
flight_names = np.unique(data['flight_name'])
print len(flight_names)
for parameter in parameters:
plt.figure()
axis = plt.gca()
# ax.set_xticks(numpy.arange(0,1,0.1))
axis.set_yticks(flight_names)
axis.tick_params(labelright=True)
axis.set_ylim([94., 130.])
plt.grid()
plt.title(title)
plt.xlabel(parameter)
plt.ylabel('flight name')
colors = iter(cm.rainbow(np.linspace(0, 1,len(flight_names))))
for flight in flight_names:
temp = data[data.flight_name == flight][parameter]
plt.plot([np.min(temp), np.max(temp)], [flight, flight], c=next(colors), linewidth=2.0)
plt.savefig(save_path+title+'_'+parameter+'.jpg')
plt.close()
def scatter_time_vs_s(time, norm, point_labels, title):
plt.figure()
size = 100
for i, l in enumerate(sorted(norm.keys())):
if l is not "fbpca":
plt.scatter(time[l], norm[l], label=l, marker='o', c='b', s=size)
for label, x, y in zip(point_labels, list(time[l]), list(norm[l])):
plt.annotate(label, xy=(x, y), xytext=(0, -80),
textcoords='offset points', ha='right',
arrowprops=dict(arrowstyle="->",
connectionstyle="arc3"),
va='bottom', size=11, rotation=90)
else:
plt.scatter(time[l], norm[l], label=l, marker='^', c='red', s=size)
for label, x, y in zip(point_labels, list(time[l]), list(norm[l])):
plt.annotate(label, xy=(x, y), xytext=(0, 30),
textcoords='offset points', ha='right',
arrowprops=dict(arrowstyle="->",
connectionstyle="arc3"),
va='bottom', size=11, rotation=90)
plt.legend(loc="best")
plt.suptitle(title)
plt.ylabel("norm discrepancy")
plt.xlabel("running time [s]")
def plotResults(datasetName, sampleSizes, foldsSet, cvScalings, sampleMethods, fileNameSuffix):
"""
Plots the errors for a particular dataset on a bar graph.
"""
for k in range(len(sampleMethods)):
outfileName = outputDir + datasetName + sampleMethods[k] + fileNameSuffix + ".npz"
data = numpy.load(outfileName)
errors = data["arr_0"]
meanMeasures = numpy.mean(errors, 0)
for i in range(sampleSizes.shape[0]):
plt.figure(k*len(sampleMethods) + i)
plt.title("n="+str(sampleSizes[i]) + " " + sampleMethods[k])
for j in range(errors.shape[3]):
plt.plot(foldsSet, meanMeasures[i, :, j])
plt.xlabel("Folds")
plt.ylabel('Error')
labels = ["VFCV", "PenVF+"]
labels.extend(["VFP s=" + str(x) for x in cvScalings])
plt.legend(tuple(labels))
plt.show()
def statistics_charts(self):
if plt is None:
return
for chart in self.stats_charts:
if chart["type"] == "plot":
fig = plt.figure(figsize=(8, 2))
for xdata, ydata, label in chart["data"]:
plt.plot(xdata, ydata, "-", label=label)
plt.legend(loc="center left", bbox_to_anchor=(1, 0.5))
elif chart["type"] == "timeline":
fig = plt.figure(figsize=(16, 2))
for i, (starts, stops, label) in enumerate(chart["data"]):
plt.hlines([i] * len(starts), starts, stops, label=label)
plt.ylim(-1, len(chart["data"]))
elif chart["type"] == "bars":
fig = plt.figure(figsize=(16, 4))
plt.bar(range(len(chart["data"])), chart["data"])
elif chart["type"] == "boxplot":
fig = plt.figure(figsize=(16, 4))
plt.boxplot(chart["data"])
else:
raise Exception("Unknown chart")
png = serialize_fig(fig)
yield chart["name"], html_embed_img(png)
def plot_scenario(strategies, names, scenario_id=1):
probabilities = get_scenario(scenario_id)
plt.figure(figsize=(6, 4.5))
ax = plt.subplot(111)
ax.spines["top"].set_visible(False)
ax.spines["bottom"].set_visible(False)
ax.spines["right"].set_visible(False)
ax.spines["left"].set_visible(False)
ax.get_xaxis().tick_bottom()
ax.get_yaxis().tick_left()
plt.yticks(fontsize=14)
plt.xticks(fontsize=14)
plt.xlim((0, 1300))
# Remove the tick marks; they are unnecessary with the tick lines we just plotted.
plt.tick_params(axis="both", which="both", bottom="on", top="off",
labelbottom="on", left="off", right="off", labelleft="on")
for rank, (strategy, name) in enumerate(zip(strategies, names)):
plot_strategy(probabilities, strategy, name, rank)
plt.title("Bandits: " + str(probabilities), fontweight='bold')
plt.xlabel('Number of Trials', fontsize=14)
plt.ylabel('Cumulative Regret', fontsize=14)
plt.legend(names)
plt.show()
def plotISVar():
plt.figure()
plt.title('Variance minimization problem (call).\nVertical lines mark the minima.')
for K in [0.6, 0.8, 1.0, 1.2]:
theta = np.linspace(-0.6, 2)
var = [BS.exactCallVar(K*s0, theta) for theta in theta]
minth = theta[np.argmin(var)]
line, = plt.plot(theta, var, label=str(K))
plt.axvline(minth, color=line.get_color())
plt.xlabel(r'$\theta$')
plt.ylabel('call variance')
plt.legend(title=r'$K/s_0$', loc='upper left')
plt.autoscale(tight=True)
plt.figure()
plt.title('Variance minimization problem (put).\nVertical lines mark the minima.')
for K in [0.8, 1.0, 1.2, 1.4]:
theta = np.linspace(-2, 0.5)
var = [BS.exactPutVar(K*s0, theta) for theta in theta]
minth = theta[np.argmin(var)]
line, = plt.plot(theta, var, label=str(K))
plt.axvline(minth, color=line.get_color())
plt.xlabel(r'$\theta$')
plt.ylabel('put variance')
plt.legend(title=r'$K/s_0$', loc='upper left')
plt.autoscale(tight=True)
def plotTestData(tree):
plt.figure()
plt.axis([0,1,0,1])
plt.xlabel("X axis")
plt.ylabel("Y axis")
plt.title("Green: Class1, Red: Class2, Blue: Class3, Yellow: Class4")
for value in class1:
plt.plot(value[0],value[1],'go')
plt.hold(True)
for value in class2:
plt.plot(value[0],value[1],'ro')
plt.hold(True)
for value in class3:
plt.plot(value[0],value[1],'bo')
plt.hold(True)
for value in class4:
plt.plot(value[0],value[1],'yo')
plotRegion(tree)
for value in classPlot1:
plt.plot(value[0],value[1],'g.',ms=3.0)
plt.hold(True)
for value in classPlot2:
plt.plot(value[0],value[1],'r.', ms=3.0)
plt.hold(True)
for value in classPlot3:
plt.plot(value[0],value[1],'b.', ms=3.0)
plt.hold(True)
for value in classPlot4:
plt.plot(value[0],value[1],'y.', ms=3.0)
plt.grid(True)
plt.show()
def heatmap(vals, size=6, aspect=1):
"""
Plot a heatmap from matrix data
"""
plt.figure(figsize=(size, size))
plt.imshow(vals, cmap="gray", aspect=aspect, interpolation="none", vmin=0, vmax=1)
plt.axis("off")
def test_get_obs(self):
plt.figure()
ant_sigs = antennas.antennas_signal(self.ants, self.ant_models, self.sources, self.rad.timebase)
rad_sig_full = self.rad.sampled_signal(ant_sigs[0, :], 0)
obs_full = self.rad.get_full_obs(ant_sigs, self.utc_date, self.config)
ant_sigs_simp = antennas.antennas_simplified_signal(self.ants, self.ant_models, self.sources, self.rad.baseband_timebase, self.rad.int_freq)
obs_simp = self.rad.get_simplified_obs(ant_sigs_simp, self.utc_date, self.config)
freqs, spec_full_before_obs = spectrum.plotSpectrum(rad_sig_full, self.rad.ref_freq, label='full_before_obs_obj', c='blue')
freqs, spec_full = spectrum.plotSpectrum(obs_full.get_antenna(1), self.rad.ref_freq, label='full', c='cyan')
freqs, spec_simp = spectrum.plotSpectrum(obs_simp.get_antenna(1), self.rad.ref_freq, label='simp', c='red')
plt.legend()
self.assertTrue((spec_full_before_obs == spec_full).all(), True)
plt.figure()
plt.plot(freqs, (spec_simp-spec_full)/spec_full)
plt.show()
print len(obs_full.get_antenna(1)), obs_full.get_antenna(1).mean()
print len(obs_simp.get_antenna(1)), obs_simp.get_antenna(1).mean()
def _plot_histogram(self, data, number_of_devices=1,
preamp_timeout=1253):
if number_of_devices == 0:
return
data = np.array(data)
plt.figure(3)
plt.ioff()
plt.get_current_fig_manager().window.wm_geometry("800x550+700+25")
plt.clf()
if number_of_devices == 1:
plt.hist(data[0,:], bins=preamp_timeout, range=(1, preamp_timeout-1),
color='b')
elif number_of_devices == 2:
plt.hist(data[0,:], bins=preamp_timeout, range=(1, preamp_timeout-1),
color='r', label='JPM A')
plt.hist(data[1,:], bins=preamp_timeout, range=(1, preamp_timeout-1),
color='b', label='JPM B')
plt.legend()
elif number_of_devices > 2:
raise Exception('Histogram plotting for more than two ' +
'devices is not implemented.')
plt.xlabel('Timing Information [Preamp Time Counts]')
plt.ylabel('Counts')
plt.xlim(0, preamp_timeout)
plt.draw()
plt.pause(0.05)
def plot_data(kx,omega,F,F_R,F_L,K,O):
#plt.figure(4)
#plt.imshow(K,extent=[omega[0],omega[-1],kx[0],kx[-1]],\
# interpolation = "nearest", aspect = "auto")
#plt.xlabel('KX')
#plt.colorbar()
#plt.figure(5)
#plt.imshow(O,extent =[omega[0],omega[-1],kx[0],kx[-1]],interpolation="nearest", aspect="auto")
#plt.xlabel('omega')
#plt.colorbar()
plt.figure(6)
pylab.subplot(1,2,1)
plt.imshow(abs(F_R), extent= [omega[0],omega[-1],kx[0],kx[-1]], interpolation= "nearest", aspect = "auto")
plt.xlabel('abs FFT_R')
plt.colorbar()
plt.subplot(1,2,2)
plt.imshow(abs(F_L), extent= [omega[0],omega[-1],kx[0],kx[-1]], interpolation= "nearest", aspect = "auto")
plt.xlabel('abs FFT_L')
plt.colorbar()
plt.figure(7)
plt.subplot(2,1,1)
plt.imshow(abs(F_L+F_R),extent=[omega[0],omega[-1],kx[0],kx[-1]],interpolation= "nearest", aspect = "auto")
plt.xlabel('abs(F_L+F_R) reconstructed')
plt.colorbar()
pylab.subplot(2,1,2)
plt.imshow(abs(F),extent=[omega[0],omega[-1],kx[0],kx[-1]],interpolation ="nearest",aspect = "auto")
plt.xlabel('FFT of the original data')
plt.colorbar()
#plt.show()
return
def plot_wav_fft(wav_filename, desc=None):
plt.clf()
plt.figure(num=None, figsize=(6, 4))
sample_rate, X = scipy.io.wavfile.read(wav_filename)
spectrum = np.fft.fft(X)
freq = np.fft.fftfreq(len(X), 1.0 / sample_rate)
plt.subplot(211)
num_samples = 200.0
plt.xlim(0, num_samples / sample_rate)
plt.xlabel("time [s]")
plt.title(desc or wav_filename)
plt.plot(np.arange(num_samples) / sample_rate, X[:num_samples])
plt.grid(True)
plt.subplot(212)
plt.xlim(0, 5000)
plt.xlabel("frequency [Hz]")
plt.xticks(np.arange(5) * 1000)
if desc:
desc = desc.strip()
fft_desc = desc[0].lower() + desc[1:]
else:
fft_desc = wav_filename
plt.title("FFT of %s" % fft_desc)
plt.plot(freq, abs(spectrum), linewidth=5)
plt.grid(True)
plt.tight_layout()
rel_filename = os.path.split(wav_filename)[1]
plt.savefig("%s_wav_fft.png" % os.path.splitext(rel_filename)[0],
bbox_inches='tight')
def bandpass_filter(data, lowcut, highcut, fs, order,filter_type='butter_worth'):
""" the function performs bandpass filtering
Parameters
-------------
data : numpy-array
input signal
lowcut,highcut : integer
lower and higher cutoff freuencies in Hz
Fs : Integer
Samping freuency in Hz
order : Integer
Order of butterworth filter
Returns
--------
out : numpy-array
Filtered signal
"""
global once
if filter_type=='butter_worth':
b, a = butter_bandpass(lowcut, highcut, fs, order=order)
if once==0:
plt.figure(2)
plotFrequency(b,a)
once=1
y = filtfilt(b, a, data)
return y
def entries_histogram(turnstile_weather):
'''
Before we perform any analysis, it might be useful to take a
look at the data we're hoping to analyze. More specifically, lets
examine the hourly entries in our NYC subway data and determine what
distribution the data follows. This data is stored in a dataframe
called turnstile_weather under the ['ENTRIESn_hourly'] column.
Why don't you plot two histograms on the same axes, showing hourly
entries when raining vs. when not raining. Here's an example on how
to plot histograms with pandas and matplotlib:
turnstile_weather['column_to_graph'].hist()
Your histograph may look similar to the following graph:
http://i.imgur.com/9TrkKal.png
You can read a bit about using matplotlib and pandas to plot
histograms:
http://pandas.pydata.org/pandas-docs/stable/visualization.html#histograms
You can look at the information contained within the turnstile weather data at the link below:
https://www.dropbox.com/s/meyki2wl9xfa7yk/turnstile_data_master_with_weather.csv
'''
plt.figure()
(turnstile_weather[turnstile_weather.rain==0].ENTRIESn_hourly).hist(bins=175) # your code here to plot a historgram for hourly entries when it is not raining
(turnstile_weather[turnstile_weather.rain==1].ENTRIESn_hourly).hist(bins=175) # your code here to plot a historgram for hourly entries when it is raining
plt.ylim(ymax = 45000, ymin = 0)
plt.xlim(xmax = 6000, xmin = 0)
return plt
def visualize(u1, t1, u2, t2, U, omega):
plt.figure(1)
plt.plot(t1, u1, 'r--o')
t_fine = np.linspace(0, t1[-1], 1001) # мелкая сетка для точного решения
u_e = u_exact(t_fine, U, omega)
plt.hold('on')
plt.plot(t_fine, u_e, 'b-')
plt.legend([u'приближенное', u'точное'], loc='upper left')
plt.xlabel('$t$')
plt.ylabel('$u$')
tau = t1[1] - t1[0]
plt.title('$\\tau = $ %g' % tau)
umin = 1.2*u1.min();
umax = -umin
plt.axis([t1[0], t1[-1], umin, umax])
plt.savefig('tmp1.png'); plt.savefig('tmp1.pdf')
plt.figure(2)
plt.plot(t2, u2, 'r--o')
t_fine = np.linspace(0, t2[-1], 1001) # мелкая сетка для точного решения
u_e = u_exact(t_fine, U, omega)
plt.hold('on')
plt.plot(t_fine, u_e, 'b-')
plt.legend([u'приближенное', u'точное'], loc='upper left')
plt.xlabel('$t$')
plt.ylabel('$u$')
tau = t2[1] - t2[0]
plt.title('$\\tau = $ %g' % tau)
umin = 1.2 * u2.min();
umax = -umin
plt.axis([t2[0], t2[-1], umin, umax])
plt.savefig('tmp2.png');
plt.savefig('tmp2.pdf')
def mlr_show4( clf, RMv, yEv, disp = True, graph = True):
yEv_calc = clf.predict( RMv)
if len( np.shape(yEv)) == 2 and len( np.shape(yEv_calc)) == 1:
yEv_calc = np.mat( yEv_calc).T
r_sqr, RMSE, MAE, DAE = estimate_accuracy4( yEv, yEv_calc, disp = disp)
if graph:
plt.figure()
ms_sz = max(min( 4000 / yEv.shape[0], 8), 1)
plt.plot( yEv.tolist(), yEv_calc.tolist(), '.', ms = ms_sz)
ax = plt.gca()
lims = [
np.min([ax.get_xlim(), ax.get_ylim()]), # min of both axes
np.max([ax.get_xlim(), ax.get_ylim()]), # max of both axes
]
# now plot both limits against eachother
#ax.plot(lims, lims, 'k-', alpha=0.75, zorder=0)
ax.plot(lims, lims, '-', color = 'pink')
plt.xlabel('Experiment')
plt.ylabel('Prediction')
#plt.title( '$r^2$={0:.2e}, RMSE={1:.2e}, AAE={2:.2e}'.format( r_sqr, RMSE, aae))
plt.title( '$r^2$={0:.1e},$\sigma$={1:.1e},MAE={2:.1e},DAE={3:.1e}'.format( r_sqr, RMSE, MAE, DAE))
plt.show()
return r_sqr, RMSE, MAE, DAE
def regress_show4( yEv, yEv_calc, disp = True, graph = True, plt_title = None, ms_sz = None):
# if the output is a vector and the original is a metrix,
# the output is translated to a matrix.
r_sqr, RMSE, MAE, DAE = estimate_accuracy4( yEv, yEv_calc, disp = disp)
if graph:
#plt.scatter( yEv.tolist(), yEv_calc.tolist())
plt.figure()
if ms_sz is None:
ms_sz = max(min( 6000 / yEv.shape[0], 8), 3)
# plt.plot( yEv.tolist(), yEv_calc.tolist(), '.', ms = ms_sz) # Change ms
plt.scatter( yEv.tolist(), yEv_calc.tolist(), s = ms_sz)
ax = plt.gca()
lims = [
np.min([ax.get_xlim(), ax.get_ylim()]), # min of both axes
np.max([ax.get_xlim(), ax.get_ylim()]), # max of both axes
]
# now plot both limits against eachother
#ax.plot(lims, lims, 'k-', alpha=0.75, zorder=0)
ax.plot(lims, lims, '-', color = 'pink')
plt.xlabel('Experiment')
plt.ylabel('Prediction')
if plt_title is None:
plt.title( '$r^2$={0:.1e}, RMSE={1:.1e}, MAE={2:.1e}, MedAE={3:.1e}'.format( r_sqr, RMSE, MAE, DAE))
elif plt_title != "":
plt.title( plt_title)
# plt.show()
return r_sqr, RMSE, MAE, DAE
def make_fish(zoom=False):
plt.close(1)
plt.figure(1, figsize=(6, 4))
plt.plot(plot_limits['pitch'], plot_limits['rolldev'], '-g', lw=3)
plt.plot(plot_limits['pitch'], -plot_limits['rolldev'], '-g', lw=3)
plt.plot(pitch.midvals, roll.midvals, '.b', ms=1, alpha=0.7)
p, r = make_ellipse() # pitch, off nominal roll
plt.plot(p, r, '-c', lw=2)
gf = -0.08 # Fudge on pitch value for illustrative purposes
plt.plot(greta['pitch'] + gf, -greta['roll'], '.r', ms=1, alpha=0.7)
plt.plot(greta['pitch'][-1] + gf, -greta['roll'][-1], 'xr', ms=10, mew=2)
if zoom:
plt.xlim(46.3, 56.1)
plt.ylim(4.1, 7.3)
else:
plt.ylim(-22, 22)
plt.xlim(40, 180)
plt.xlabel('Sun pitch angle (deg)')
plt.ylabel('Sun off-nominal roll angle (deg)')
plt.title('Mission off-nominal roll vs. pitch (5 minute samples)')
plt.grid()
plt.tight_layout()
plt.savefig('fish{}.png'.format('_zoom' if zoom else ''))
def make_entity_plot(filename, title, fixed_noip, fixed_ip, dynamic_noip, dynamic_ip):
plt.figure(figsize=(12,5))
plt.title("Settings comparison - " + title)
plt.xlabel('Time (ms)', fontsize=12)
plt.xlim([0,62000])
x = 0
barwidth = 0.5
bargroupspacing = 1.5
fixed_noip_mean,fixed_noip_conf = conf_stats(fixed_noip)
fixed_ip_mean,fixed_ip_conf = conf_stats(fixed_ip)
dynamic_noip_mean,dynamic_noip_conf = conf_stats(dynamic_noip)
dynamic_ip_mean,dynamic_ip_conf = conf_stats(dynamic_ip)
values = [fixed_noip_mean,fixed_ip_mean,dynamic_noip_mean, dynamic_ip_mean]
errs = [fixed_noip_conf,fixed_ip_conf,dynamic_noip_conf, dynamic_ip_conf]
y_pos = numpy.arange(len(values))
plt.barh(y_pos, values, xerr=errs, align='center', color=['r', 'b', 'r', 'b'], ecolor='black', alpha=0.7)
plt.yticks(y_pos, ["Fixed | no I.P.", "Fixed | I.P.", "Dynamic | no I.P.", "Dynamic | I.P."])
plt.savefig(output_file(filename))
plt.clf()
def blue_sideband_thermal_tester():
#BSB sideband = +1
import matplotlib.pyplot as plt
dataobj = ReadData('2014Jun19',experiment = 'RabiFlopping' )
data = dataobj.get_data('1212_15')
sideband = 1
trap_freq = 2.57
nbar_init= .3 #default 20.
rabi = RabiFlop(data)
rabi.setData(data)
f_rabi = thermal_tester() #f_rabi is the same on same set of data
print("f_rabi from thermal tester is :"+str(f_rabi))
initial_guess = {'nbar':nbar_init, 'f_rabi':f_rabi, #same as that of thermal tester
'delta':0.0, 'delta_fluctuations':0.,
'trap_freq':trap_freq, 'sideband':sideband, 'nmax':1000,
'angle': 50./360.*2*np.pi,
'rabi_type':'thermal','eta': 0.04 #rabi.guess_eta()#0.05
}
fit_params = {}
#Put it in the fit params format
for key in initial_guess.keys():
fit_params[key] = (False, False, initial_guess[key]) # fixed most of the parameters
fit_params['nbar'] = (False, False, initial_guess['nbar'])
fit_params['angle'] = (True, False, initial_guess['angle'])
#fit_params['delta'] = (True, False, initial_guess['delta']) #we decided to fix delta
rabi.setUserParameters(fit_params)
x,y = rabi.fit()
plt.figure()
plt.plot(data[:,0], data[:,1],'o') # plotting raw data : excitation probability versus time
plt.plot(x,y) #plotting the fit
nbar = rabi.get_parameter_value('nbar') # rabi frequency in Hz
angle= rabi.get_parameter_value('angle')
print ('nbar: {}'.format(nbar))
print ('{} radians = {} degrees'.format(str(angle), str(angle/(2*np.pi)*360)))
def delta():
beta = 0.99
N = 1000
u = lambda c: np.sqrt(c)
W = np.linspace(0,1,N)
X, Y = np.meshgrid(W,W)
Wdiff = (X-Y).T
index = Wdiff <0
Wdiff[index] = 0
util_grid = u(Wdiff)
util_grid[index] = -10**10
Vprime = np.zeros((N,1))
delta = np.ones(1)
tol = 10**-9
it = 0
max_iter = 500
while (delta[-1] >= tol) and (it < max_iter):
V = Vprime
it += 1;
val = util_grid + beta*V.T
Vprime = np.amax(val, axis = 1)
Vprime = Vprime.reshape((N,1))
delta = np.append(delta,np.dot((Vprime-V).T,Vprime-V))
plt.figure()
plt.plot(delta[1:])
plt.ylabel(r'$\delta_k$')
plt.xlabel('iteration')
plt.savefig('convergence.pdf')
plt.clf()
def cv_show( yEv, yEv_calc, disp = True, graph = True, grid_std = None):
# if the output is a vector and the original is a metrix,
# the output is translated to a matrix.
if len( np.shape(yEv_calc)) == 1:
yEv_calc = np.mat( yEv_calc).T
if len( np.shape(yEv)) == 1:
yEv = np.mat( yEv).T
r_sqr, RMSE = jchem.estimate_accuracy( yEv, yEv_calc, disp = disp)
if graph:
#plt.scatter( yEv.tolist(), yEv_calc.tolist())
plt.figure()
ms_sz = max(min( 4000 / yEv.shape[0], 8), 1)
plt.plot( yEv.tolist(), yEv_calc.tolist(), '.', ms = ms_sz) # Change ms
ax = plt.gca()
lims = [
np.min([ax.get_xlim(), ax.get_ylim()]), # min of both axes
np.max([ax.get_xlim(), ax.get_ylim()]), # max of both axes
]
# now plot both limits against eachother
#ax.plot(lims, lims, 'k-', alpha=0.75, zorder=0)
ax.plot(lims, lims, '-', color = 'pink')
plt.xlabel('Experiment')
plt.ylabel('Prediction')
if grid_std:
plt.title( '($r^2$, std) = ({0:.2e}, {1:.2e}), RMSE = {2:.2e}'.format( r_sqr, grid_std, RMSE))
else:
plt.title( '$r^2$ = {0:.2e}, RMSE = {1:.2e}'.format( r_sqr, RMSE))
plt.show()
return r_sqr, RMSE
def thermal_tester():
import matplotlib.pyplot as plt
dataobj = ReadData('2014Jun19',experiment = 'RabiFlopping' )
car_data = dataobj.get_data('1219_57')
sideband = 0
trap_freq = 2.57
nbar_init= 0.1 #default 20.
carrier_rabi = RabiFlop(car_data)
initial_guess = {'nbar':nbar_init, 'f_rabi':carrier_rabi.guess_f_rabi(), #changing inital guess for rabi frequency to 1/2 max
'delta':0., 'delta_fluctuations':0.,
'trap_freq':trap_freq, 'sideband':sideband, 'nmax':1000,
'angle':10./360.*2*np.pi , 'rabi_type':'thermal'
,'eta': 0.05 }
fit_params = {}
#Put it in the fit params format
for key in initial_guess.keys():
fit_params[key] = (False, False, initial_guess[key]) # fix most of the parameters
fit_params['f_rabi'] = (True, False, initial_guess['f_rabi']) # fit for the rabi frequency
carrier_rabi.setUserParameters(fit_params)
x,y = carrier_rabi.fit()
plt.figure()
plt.plot(car_data[:,0], car_data[:,1],'o') # plotting raw data : excitation probability versus time
plt.plot(x,y) #plotting the fit
# Note: Only get_parameter_value returns the user_guess. Use get_parameter_info for autofit result.
f_rabi = carrier_rabi.get_parameter_info()['f_rabi'][2][2] #autofit result (rabi frequency in Hz)
return f_rabi
def plot_jacobian(A, name, cmap= plt.cm.coolwarm, normalize=True, precision=1e-6):
"""
Customized visualization of jacobian matrices for observing
sparsity patterns
"""
plt.figure()
fig, ax = plt.subplots()
if normalize is True:
plt.imshow(A, interpolation='none', cmap=cmap,
norm = mpl.colors.Normalize(vmin=-1.,vmax=1.))
else:
plt.imshow(A, interpolation='none', cmap=cmap)
plt.colorbar(format=ticker.FuncFormatter(fmt))
ax.spy(A, marker='.', markersize=0, precision=precision)
ax.spines['right'].set_visible(True)
ax.spines['bottom'].set_visible(True)
ax.xaxis.set_ticks_position('top')
ax.yaxis.set_ticks_position('left')
xlabels = np.linspace(0, A.shape[0], 5, True, dtype=int)
ylabels = np.linspace(0, A.shape[1], 5, True, dtype=int)
plt.xticks(xlabels)
plt.yticks(ylabels)
plt.savefig(name, bbox_inches='tight', pad_inches=0.05)
plt.close()
return
llcrnrlat = llcrnrlat, urcrnrlat = urcrnrlat,
resolution='h',
area_thresh=1000)# setting area_thresh doesn't plot lakes/coastlines smaller than threshold
x,y = m(lon,lat)
var_list = ["O3", "PMIJ"]
unit_list = ["ppb", "$ug/m^3$"]
############################################
# Averaged domain basemaps
############################################
#save maps into the pdf file (two maps in single page)
with PdfPages(base_dir+'4km_avg_basemap_' + '_'+ start.strftime("%Y%m%d") + '-' + end.strftime("%Y%m%d") + '.pdf') as pdf:
for i, sp in enumerate(var_list):
plt.style.use("dark_background")
fig = plt.figure(figsize=(14,10))
#plt.title(sp)
o3_max = 45
pm_max = 40
o3_bins = np.arange(0, o3_max, 5)
pm_bins = np.arange(0, pm_max, 5)
vmin = 0
if sp == "O3":
clevs = o3_bins
vmax = o3_max
else:
clevs = pm_bins
vmax = pm_max
# compute auto color-scale using maximum concentrations
down_scale = np.percentile(airpact[sp], 5)
up_scale = np.percentile(airpact[sp], 95)
print(e)
print("SENSITIVITY FIT FAILED FOR INDEX :{}".format(ii))
sens = 0.0
disc = 0.0
dec = 0.0
alert_sens['sens'].append(sens)
alert_sens['disc'].append(disc)
alert_sens['sin_dec'].append(np.sin(dec))
for k in alert_sens.keys():
alert_sens[k] = np.array(alert_sens[k])
utils.initialize_mpl_style()
sens_cols = [sns.xkcd_rgb['navy blue'], sns.xkcd_rgb['navy green'], sns.xkcd_rgb['deep magenta']]
ii=2
try:
fig = plt.figure(dpi=200)
fig.set_facecolor('white')
plt.plot(time_integrated['sinDec'], time_integrated['sens'] * 86400. * 10. * 365. * 1e6,
color = sns.xkcd_rgb['light grey'],
alpha = 0.7, ls='-', zorder=4)
plt.plot(time_integrated['sinDec'], time_integrated['disc'] * 86400. * 10. *365.* 1e6,
color = sns.xkcd_rgb['light grey'],
alpha = 0.7, ls='--', zorder=4)
plt.plot(ideal['sinDec'], ideal['sens'] * 86400. * 8.607 * 365. * 1e6,
color = sens_cols[ii], lw=3, label = 'P.S. sensitivity',
alpha = 0.7, ls='-', zorder=4)
plt.plot(ideal['sinDec'], ideal['disc'] * 86400. * 8.607 *365.* 1e6,
color = sens_cols[ii], lw=3, label = r'P.S. $3\sigma$ discovery' +'\n\t(50\% CL)',
alpha = 0.7, ls='--', zorder=4)
done = False
while env.z_g > env.z_g_down_limit and not done:
action = 0
env.render()
_, _, done, _ = env.step(action)
belief = env.belief
print('z_g:', env.z_g)
print('theta:', env.angle_str[env.theta])
print(belief)
print()
z_g_all.append(env.z_g)
theta_all.append(env.theta)
z_theta_dict[(env.action_str[action], env.z_g)] = env.angle_str[env.theta]
print(z_g_all)
print(theta_all)
print(z_theta_dict)
print()
for ii in range(belief.shape[0]):
if belief[ii] == 1:
assert (ii == env.z_t)
plt.figure(figsize=(12, 8))
plt.xlabel('Angle')
plt.ylabel('z_g')
plt.plot(theta_all, z_g_all)
plt.show()
env.close()
#
# ___
# The majority of the companies' stock index remained fairly constant after April, except for Goldman Sachs (**GS**) having a slight increase.
# In[42]:
bank_stocks.xs(key='Close', axis=1, level='Stock Info').iplot(title="Stock Price over Time in 2020")
# And we can look at the increase in Goldman Sachs' stock price by comparing it to a 15-Day Moving Average (Which is just an average of the stock prices for consecutive 15 day periods).
# In[56]:
plt.figure(figsize=(8,4))
GS['Close'].loc['2020-01-01':'2020-07-12'].rolling(window=15).mean().plot(label='15-Day Moving Average')
GS['Close'].loc['2020-01-01':'2020-07-12'].plot(label='GS Close')
plt.legend()
plt.title("Stock Index and Moving Average of Goldman Sachs")
plt.ylabel("Stock Price")
plt.show()
# Out of the 10 companies, it's had the highest increase in stock prices since the begining of April.
# # Analyzing Returns
#
# Now that we've analyzed total amounts of Stock Prices, we now want to analyze the change in prices, or in other words, the returns you would get if you invested in a certain company which are a good measure of how that company is performing.
#
# The $Return$ on any stock at time $t$ is defined as the percent change of the initial $Price_{0}$ plus any Dividend $D$
def adc2keV(adcPlot):
import numpy as np
import time
import heapq
import statsmodels.api as sm
import numpy as np
import matplotlib.pyplot as plt
MeV = []
index1 = 0
index2 = 0
index3 = 0
index4 = 0
adcVal = []
i = 0
for row in adcPlot:
# peakVal, peakTime = max(enumerate(row), key=operator.itemgetter(1)) #findPeaks(row)
#print('temp = ',temp)
#print(row)
peakTime = row.argmax()
pulseIntegral = integratePulse(row, peakTime)
#tailIntegral = integrateTail(row,peakTime)
#tailToTotalRatio += [tailIntegral/float(pulseIntegral)]
adcVal += [pulseIntegral]
if i % 100000 == 0:
print('m = ', i)
print('elapsed time = ', time.time() - tic, 's')
i = i + 1
adcVal = np.asarray(adcVal, dtype='float')
histNADC = np.histogram(adcVal, 100000)
d1 = histNADC[0]
e1 = histNADC[1]
f1 = e1[0:100000]
plt.figure(1)
plt.plot(f1, d1, 'b-', label='Cs Plot')
plt.xlabel('ADC Val')
plt.ylabel('Counts')
plt.legend(loc='upper right')
plt.title(plane)
#plt.plot(d,f)
plt.show()
#Cs = heapq.nlargest(1,range(125000,len(f1)),key=f1.__getitem__)
#Co1 = heapq.nlargest(1,range(250000,len(f1)),key=f1.__getitem__)
#Co2 = heapq.nlargest(1,range(315000,len(f1)),key=f1.__getitem__)
#f1 = np.around(f1)
#f1 = np.array(f1,dtype='int')
#print(f1)
index1 = (np.abs(f1 - 30000)).argmin()
index2 = (np.abs(f1 - 50000)).argmin()
index3 = (np.abs(f1 - 100000)).argmin()
#index4 = (np.abs(f1 - 400000)).argmin()
CsCompton = np.argmax(f1[index1:index2])
Cs = np.argmax(f1[index2:index3])
#for i in range(0,len(d1)):
y = np.array([477.7, 662])
x = np.array([f1[CsCompton], f1[Cs]])
#x = np.array(x)
#m, b = np.polyfit(x, y, 1)
#plt.plot(x, y, '.')
#plt.plot(x, m*x + b, '-')
#X = np.random.rand(100)
#Y = X + np.random.rand(100)*0.1
results = sm.OLS(y, sm.add_constant(x)).fit()
#print(results.summary())
slope, intercept = np.polyfit(x, y, 1)
abline_values = [slope * i + intercept for i in x]
plt.plot(x, y, 'ro')
plt.plot(x, abline_values, 'b')
plt.xlabel('ADC Val')
plt.ylabel('Energy [keV]')
plt.title('Best Fit Line')
plt.show()
# plt.show()
return slope, intercept
# analysis. # That is, before starting anything, this will make the analysis and # the challenge more difficult but it will simulate a situation of deployment. # By the time that we use the test dataset, we'll need to think # how to preprocess the incoming data, so that it can be consumed # the `.predict` of the trained model. # In[29]: # grade: loan grade df_train["grade"].hist(bins=30) # I imagine that A is better than B, and so on... # We can test this later. # On this first stage we'll make univariate analysis of the variables. # In[31]: # sub_grade: plt.figure(figsize=(14, 6)) df_train["sub_grade"].hist(bins=70) # In[33]: # emp_title: the job title that the borrower told when applying for the loan plt.figure(figsize=(12, 6)) df_train["emp_title"].value_counts() #%% # We have 143_694 different employee titles. # This number is too high. # This is a situation where we have too many different types # of categorical non-ordered possible values for a feature. # I worked with this type of data in the insurance industry. # (I should revisit the code that I used there to see the methods # that I wrote to automatically group the categories that are too sparce) # # We can also find a feature with too many categories
eig_image = cv.CreateMat(self.map_img.height, self.map_img.width, cv.CV_32FC1)
temp_image = cv.CreateMat(self.map_img.height, self.map_img.width, cv.CV_32FC1)
features_x_y_vector = cv.GoodFeaturesToTrack(self.map_img, eig_image, temp_image, self.map_img.width, 0.15, 10, useHarris=True)
print "get corners finished"
return features_x_y_vector
#===============================================================================
# If this code is executed instead of imported we will do this
#===============================================================================
if __name__ == "__main__":
grid = MapPossibleGoalsDetector("inria_full")
""" Preprocess Map"""
preprocessed_map = grid.preproc_map_img(grid.map_img)
np_img = np.asarray(preprocessed_map)
fig_4 =plt.figure(4)
#plt.subplot(2,1,1)
imgplot=plt.imshow(np_img, cmap="jet")
fig_4.colorbar(imgplot)
""" Plot Distance Image """
(dist_img_gray, dist_img) = grid.compute_distance_image(preprocessed_map)
np_img = np.asarray(dist_img)
fig_1 =plt.figure(1)
#plt.subplot(2,1,1)
imgplot=plt.imshow(np_img, cmap="jet")
fig_1.colorbar(imgplot)
def addStackedBarPlot(self,
colData=None,
yErrorData=None,
xlabel="XLabel",
xTickLabel="",
ylabel="YLabel",
legendPos="",
legendLabels=[],
hatching=False,
fillBar=True,
colorBar="w",
yrange=None):
# this met
self.columnData.append(colData)
self.xticks.append(xTickLabel)
self.yErrorData.append(yErrorData)
fig = plt.figure()
# axes = fig.axes.set_size("large")
ax = fig.add_subplot(111)
rects = list()
# set width to default
width = 0.35
# revise width of single stack if number of data rows is unequal zero
if len(self.columnData) != 0:
width = 1.0 / len(self.columnData)
ind = np.arange(len(self.columnData))
# ax.set_size("large")
for i in range(len(self.columnData)):
stackedData = self.columnData[i]
stackedYerror = self.yErrorData[i]
heightFromBottom = 0
for j in range(len(self.columnData[i])):
if hatching:
rect = ax.bar(i,
height=stackedData[j],
bottom=heightFromBottom,
width=width,
color=colorBar,
ecolor="black",
yerr=stackedYerror[j],
fill=fillBar,
hatch=self.hatches[j])
heightFromBottom += stackedData[j]
if i == 0:
rects.append(rect)
else:
rect = ax.bar(i,
height=stackedData[j],
bottom=heightFromBottom,
width=width,
color=colorBar,
ecolor="black",
yerr=stackedYerror[j],
fill=fillBar)
heightFromBottom += stackedData[j]
if i == 0:
rects.append(rect)
# rect = ax.bar(i, height=stackedData[j], width=width, color=self.colors[j], ecolor="black", yerr=stackedYerror[j])
# add to rects array only to first rect as it will be
# needed for later labeling
# else:
# rect = ax.bar(i, height=stackedData[j], bottom=stackedData[j-1], width=width, color=self.colors[j], ecolor="black", yerr=stackedYerror[j])
ax.set_ylabel(ylabel)
plt.xlabel(xlabel)
ax.set_xticklabels(self.xticks)
ax.set_xticks(ind + width / 2)
if yrange is not None:
plt.ylim(yrange)
if legendPos != "None":
ax.legend((rects), legendLabels, loc=legendPos)
def autolabel(rects):
# attach some text labels
for rect in rects:
height = rect.get_height()
ax.text(rect.get_x() + rect.get_width() / 2.,
1.05 * height,
'%d' % int(height),
ha='center',
va='bottom')
# training_Y.append(0)
print(len(training_X))
print(len(training_Y))
# Specify Gaussian Processes with fixed and optimized hyperparameters
gp_opt = GaussianProcessClassifier(RBF(length_scale=1.0))
gp_opt.fit(training_X,training_Y)
print("The trained hyperparameter are {}".format((gp_opt.kernel_.theta)))
# print("Log Marginal Likelihood (optimized): %.3f"
# % gp_opt.log_marginal_likelihood(gp_opt.kernel_.theta))
# print("The probability of occupancy is {}".format(p_occ))
fig = plt.figure()
ZZ = np.empty([30,30])
for idx1, row in enumerate(x):
for idx2,col in enumerate(y):
K = [row,col]
if K in training_X:
ZZ[idx1,idx2] = 0.0
else:
# print("({},{})".format(row,col))
# print("({},{})".format(idx1,idx2))
p_occ = gp_opt.predict_proba(np.reshape(K,(-1,2)))[:,1]
occ_ = gp_opt.predict(np.reshape(K,(-1,2)))
ZZ[idx1,idx2] = p_occ
plt.text(row,col,"{}".format(round(p_occ[0],3)), color='black',fontsize=5)
if occ_ == 1:
import math
import matplotlib.pyplot as plt
import numpy as np
def logistic (t):
return 1000./(1+999*math.exp(-2*t))
x = np.linspace(0,10,11)
y = [logistic(i) for i in x]
plt.scatter(x,y)
plt.plot(x, y)
def ode_FE(f, U_0, dt, T):
N_t = int(round(float(T)/dt))
u = zeros(N_t+1)
t = linspace(0, N_t*dt, len(u))
u[0] = U_0
for n in range(N_t):
u[n+1] = u[n] + dt*f(u[n], t[n])
return u, t
for dt, T in zip((0.5, 20), (60, 100)):
u, t = ode_FE(f=lambda u, t: 0.1*(1 - u/500.)*u, \
U_0=100, dt=dt, T=T)
plt.figure() # Make separate figures for each pass in the loop
plt.plot(t, u, 'b-')
plt.xlabel('t'); plt.ylabel('N(t)')
plt.savefig('tmp_%g.png' % dt); plt.savefig('tmp_%g.pdf' % dt)
predicted_y_test_all = model.predict_rep(
autoscaled_variables_test[:, numbers_of_X], numbers_of_X, numbers_of_y)
# Inverse analysis
predicted_x_test_all = model.predict_rep(
autoscaled_variables_test[:, numbers_of_y], numbers_of_y, numbers_of_X)
# Check results of forward analysis (regression)
print('Results of forward analysis (regression)')
plt.rcParams['font.size'] = 18
for Y_number in range(len(numbers_of_y)):
predicted_ytest = np.ndarray.flatten(predicted_y_test_all[:, Y_number])
predicted_ytest = predicted_ytest * variables_train[:, numbers_of_y[Y_number]].std(ddof=1) + \
variables_train[:, numbers_of_y[Y_number]].mean()
# yy-plot
plt.figure(figsize=figure.figaspect(1))
plt.scatter(variables_test[:, numbers_of_y[Y_number]], predicted_ytest)
y_max = np.max(
np.array([
np.array(variables_test[:, numbers_of_y[Y_number]]),
predicted_ytest
]))
y_min = np.min(
np.array([
np.array(variables_test[:, numbers_of_y[Y_number]]),
predicted_ytest
]))
plt.plot([y_min - 0.05 * (y_max - y_min), y_max + 0.05 * (y_max - y_min)],
[y_min - 0.05 * (y_max - y_min), y_max + 0.05 * (y_max - y_min)],
'k-')
plt.ylim(y_min - 0.05 * (y_max - y_min), y_max + 0.05 * (y_max - y_min))
def addBarPlot(self,
colData=None,
yErrorData=None,
xlabel="XLabel",
xTickLabel="",
ylabel="YLabel",
legendPos="",
hatching=False,
fillBar=True,
colors="Default",
yrange=None):
self.columnData.append(colData)
self.xticks.append(xTickLabel)
self.yErrorData.append(yErrorData)
fig = plt.figure()
# axes = fig.axes.set_size("large")
ax = fig.add_subplot(111)
rects = list()
width = 0.6
# if (len(self.columnData) != 0):
# width = 1.0 / len(self.columnData)
if colors == "Default":
colors = self.colors
ind = np.arange(len(self.columnData))
# ax.set_size("large")
for i in range(len(self.columnData)):
if hatching:
rect = ax.bar(i + (1 - width) / 2,
height=self.columnData[i],
width=width,
color=colors[i],
ecolor="black",
yerr=self.yErrorData[i],
fill=fillBar,
hatch=self.hatches[i])
rects.append(rect)
else:
rect = ax.bar(i + (1 - width) / 2,
height=self.columnData[i],
width=width,
color=colors[i],
ecolor="black",
yerr=self.yErrorData[i],
fill=fillBar)
rects.append(rect)
ax.set_ylabel(ylabel)
plt.xlabel(xlabel)
ax.set_xticklabels(self.xticks)
ax.set_xticks(ind + 0.5)
ax.yaxis.grid(linestyle='dotted')
ax.xaxis.grid(linestyle='None')
for t in self.ax.xaxis.get_major_ticks():
t.tick1On = False
t.tick2On = False
if yrange is not None:
plt.ylim(yrange)
if legendPos != "None":
ax.legend((rects), self.xticks, loc=legendPos)
def autolabel(rects):
# attach some text labels
for rect in rects:
height = rect.get_height()
ax.text(rect.get_x() + rect.get_width() / 2.,
1.05 * height,
'%d' % int(height),
ha='center',
va='bottom')
def plot_diffraction_vectors(
self,
xlim=1.0,
ylim=1.0,
unique_vectors=None,
distance_threshold=0.01,
method='distance_comparison',
min_samples=1,
image_to_plot_on=None,
image_cmap='gray',
plot_label_colors=False,
distance_threshold_all=0.005): # pragma: no cover
"""Plot the unique diffraction vectors.
Parameters
----------
xlim : float
The maximum x coordinate to be plotted.
ylim : float
The maximum y coordinate in reciprocal Angstroms to be plotted.
unique_vectors : DiffractionVectors, optional
The unique vectors to be plotted (optional). If not given, the
unique vectors will be found by get_unique_vectors.
distance_threshold : float, optional
The minimum distance in reciprocal Angstroms between diffraction
vectors for them to be considered unique diffraction vectors.
Will be passed to get_unique_vectors if no unique vectors are
given.
method : str
The method to use to determine unique vectors, if not given.
Valid methods are 'strict', 'distance_comparison' and 'DBSCAN'.
'strict' returns all vectors that are strictly unique and
corresponds to distance_threshold=0.
'distance_comparison' checks the distance between vectors to
determine if some should belong to the same unique vector,
and if so, the unique vector is iteratively updated to the
average value.
'DBSCAN' relies on the DBSCAN [1] clustering algorithm, and
uses the Eucledian distance metric.
min_samples : int, optional
The minimum number of not identical vectors within one cluster
for it to be considered a core sample, i.e. to not be considered
noise. Will be passed to get_unique_vectors if no unique vectors
are given. Only used if method=='DBSCAN'.
image_to_plot_on : BaseSignal, optional
If provided, the vectors will be plotted on top of this image.
The image must be calibrated in terms of offset and scale.
image_cmap : str, optional
The colormap to plot the image in.
plot_label_colors : bool, optional
If True (default is False), also the vectors contained within each
cluster will be plotted, with colors according to their
cluster membership. If True, the unique vectors will be
calculated by get_unique_vectors. Requires on method=='DBSCAN'.
distance_threshold_all : float, optional
The minimum distance, in calibrated units, between diffraction
vectors inside one cluster for them to be plotted. Only used if
plot_label_colors is True and requires method=='DBSCAN'.
Returns
-------
fig : matplotlib figure
The plot as a matplotlib figure.
"""
fig = plt.figure()
ax = fig.add_subplot(111)
offset, scale = 0., 1.
if image_to_plot_on is not None:
offset = image_to_plot_on.axes_manager[-1].offset
scale = image_to_plot_on.axes_manager[-1].scale
ax.imshow(image_to_plot_on, cmap=image_cmap)
else:
ax.set_xlim(-xlim, xlim)
ax.set_ylim(ylim, -ylim)
ax.set_aspect('equal')
if plot_label_colors is True and method == 'DBSCAN':
clusters = self.get_unique_vectors(distance_threshold,
method='DBSCAN',
min_samples=min_samples,
return_clusters=True)[1]
labs = clusters.labels_[clusters.core_sample_indices_]
# Get all vectors from the clustering not considered noise
cores = clusters.components_
if cores.size == 0:
warn('No clusters were found. Check parameters, or '
'use plot_label_colors=False.')
else:
peaks = DiffractionVectors(cores)
peaks.axes_manager.set_signal_dimension(1)
# Since this original number of vectors can be huge, we
# find a reduced number of vectors that should be plotted, by
# running a new clustering on all the vectors not considered
# noise, considering distance_threshold_all.
peaks = peaks.get_unique_vectors(distance_threshold_all,
min_samples=1,
return_clusters=False)
peaks_all_len = peaks.data.shape[0]
labels_to_plot = np.zeros(peaks_all_len)
peaks_to_plot = np.zeros((peaks_all_len, 2))
# Find the labels of each of the peaks to plot by referring back
# to the list of labels for the original vectors.
for n, peak in zip(np.arange(peaks_all_len), peaks):
index = distance_matrix([peak.data], cores).argmin()
peaks_to_plot[n] = cores[index]
labels_to_plot[n] = labs[index]
# Assign a color value to each label, and shuffle these so that
# adjacent clusters hopefully get distinct colors.
cmap_lab = get_cmap('gist_rainbow')
lab_values_shuffled = np.arange(np.max(labels_to_plot) + 1)
np.random.shuffle(lab_values_shuffled)
labels_steps = np.array(
list(
map(lambda n: lab_values_shuffled[int(n)],
labels_to_plot)))
labels_steps = labels_steps / (np.max(labels_to_plot) + 1)
# Plot all peaks
for lab, peak in zip(labels_steps, peaks_to_plot):
ax.plot((peak[0] - offset) / scale,
(peak[1] - offset) / scale,
'.',
color=cmap_lab(lab))
if unique_vectors is None:
unique_vectors = self.get_unique_vectors(distance_threshold,
method=method,
min_samples=min_samples)
# Plot the unique vectors
ax.plot((unique_vectors.data.T[0] - offset) / scale,
(unique_vectors.data.T[1] - offset) / scale, 'kx')
plt.tight_layout()
plt.axis('off')
return fig
alphanum = "abcdefghijklmnopqrstuvwxyz0123456789"
touchesspe = ["CTRL", "ENTER", "NOKEY", "SHIFT", "SPACE", "SUPPR"]
from legacy.prepare_data import generate_dataset_dataframe
df, df_cleaned = generate_dataset_dataframe(alphanum, touchesspe, False)
import pandas as pd
unique = df['label'].unique()
df_new = pd.DataFrame()
for v in unique:
mean = df[df['label'] == v].mean()
df_new[v] = mean
df_new
import seaborn as sn
import matplotlib.pyplot as plt
corrMatrix = df_new.corr()
sn.heatmap(corrMatrix,
xticklabels=corrMatrix.columns,
yticklabels=corrMatrix.columns)
plt.figure(1)
plt.show()
def plot_collapse(flat_list,
gamma_shape,
min_rep=10,
min_d=4,
ax=None,
str_leg=None,
extrapolate=False,
color='r',
show_subplots=True):
#Definitions
interp_points = 1000
if ax is None:
plt.figure()
else:
plt.sca(ax)
#Flattens list of reps
#flat_list = np.array([item for sublist in shape_list for item in sublist])
flat_list = np.array(flat_list)
#List of avalanche sizes
shape_size = np.zeros(len(flat_list))
for i in range(len(flat_list)):
shape_size[i] = flat_list[i].size
max_size = shape_size.max()
#Avalanche size count
shape_count, _ = np.histogram(shape_size, bins=np.arange(0, max_size + 2))
#Censors data by size
censor_d_keep = np.arange(0, max_size + 1) >= min_d
censor_rep_keep = shape_count >= min_rep
censor_index = np.where(
[a and b for a, b in zip(censor_d_keep, censor_rep_keep)])[0]
#Defines average size matrix
average_shape = np.zeros((censor_index.size, interp_points))
#Defines bottom interpolation range from data, to prevent extrapolation bias
#x_min = 1/censor_index[0]
if extrapolate is True:
x_min = 0
elif extrapolate is False:
x_min = 1 / censor_index[0]
else:
error('extrapolate is not binary.')
x_range = np.linspace(x_min, 1, num=interp_points)
#Averages shape for each duration and interpolates results
y_min = 100
for i in range(len(censor_index)):
#Calculates average shape
size_i = censor_index[i]
avg_shape_i_y = np.mean(flat_list[shape_size == size_i]) / np.power(
size_i, gamma_shape - 1)
avg_shape_i_x = np.arange(1, size_i + 1) / size_i
if np.min(avg_shape_i_y) < y_min:
y_min = np.min(avg_shape_i_y)
#Interpolates results
fx = InterpolatedUnivariateSpline(avg_shape_i_x, avg_shape_i_y)
average_shape[i, :] = fx(x_range)
#Plots transparent subplots
if show_subplots:
ax.plot(avg_shape_i_x, avg_shape_i_y, alpha=0.2, color=color)
#Plots interpolated average curve
if show_subplots:
color_collapse = 'k'
else:
color_collapse = color
plot_line, = ax.plot(x_range,
np.mean(average_shape, axis=0),
color=color_collapse,
linewidth=2,
label=str_leg)
ax.legend([plot_line], [str_leg])
plt.legend()
#Beautifies plot
ax.set_xlabel('Scaled time')
ax.set_ylabel('Scaled activity')
plt.xlim([0, 1])
import matplotlib.pyplot as plt
import numpy as np
x = np.arange(-1, 1, 0.1) # x=[-1.0, -0.9, ... 1.0]
y = np.exp(x) # y = e^x
y_diff_1 = 1 + x
y_diff_2 = y_diff_1 + (1/2) * x**2
y_diff_3 = y_diff_2 + (1/6) * x**3
# Figureの初期化
fig = plt.figure(figsize=(8, 8))
plt.plot(x, y, color="b", marker="o", label="e^x")
plt.plot(x, y_diff_1, color="g", ls="-.", label="1+x")
plt.plot(x, y_diff_2, color="r", ls="--", label="1+x+(1/2)x^2")
plt.plot(x, y_diff_3, color="k", ls=":", label="1+x+(1/2)x^2+(1/6)x^3")
# 軸ラベルを設定
plt.xlabel("x", fontsize=18)
plt.ylabel("y", fontsize=18)
# X軸の最大値、最小値を指定
plt.xlim(x.min(), x.max())
# タイトルを表示
plt.title('Maclaurin(e^x)', fontsize=25)
# 凡例を表示
plt.legend()
plt.show()
if done:
print("out of bounds")
env.render()
S = np.array(S)
#%% Plotting the policy in the state space.
x = np.linspace(0, 10, 50)
v = np.linspace(-20, 20, 60)
A = np.zeros((len(x), len(v)))
for i, xi in enumerate(x):
for j, vj in enumerate(v):
A[i, j] = select_action(FloatTensor([[vj, xi, env.referencepoint]
])).data.cpu().numpy()[0, 0]
SUP = str.maketrans("0123456789", "⁰¹²³⁴⁵⁶⁷⁸⁹")
plt.figure(3)
plt.title('Policy')
plt.xlabel("Velocity (in ms2)".translate(SUP))
plt.ylabel("Position (in meters)")
plt.contourf(v, x, A, levels=[0.1, 1])
plt.scatter(S[:, 0], S[:, 1], c='r')
plt.plot(S[0, 0], S[0, 1], c='k', marker='*')
plt.scatter(S[-1, 0], S[-1, 1], c='k', marker='s')
plt.figure(4)
plt.title("Convergence Graph")
plt.xlabel("Episodes")
plt.ylabel("Position (in meters)")
plt.plot([0, 500], [env.referencepoint] * 2)
plt.plot(S[:, 1])
plt.legend(['Reference point', 'Stability'])
print(labels)
df["Clus_km"] = labels
df.head(5)
df.groupby('Clus_km').mean()
area = np.pi * ( X[:, 1])**2
plt.scatter(X[:, 0], X[:, 3], s=area, c=labels.astype(np.float), alpha=0.5)
plt.xlabel('Age', fontsize=18)
plt.ylabel('Income', fontsize=16)
plt.show()
from mpl_toolkits.mplot3d import Axes3D
fig = plt.figure(1, figsize=(8, 6))
plt.clf()
ax = Axes3D(fig, rect=[0, 0, .95, 1], elev=48, azim=134)
plt.cla()
# plt.ylabel('Age', fontsize=18)
# plt.xlabel('Income', fontsize=16)
# plt.zlabel('Education', fontsize=16)
ax.set_xlabel('Education')
ax.set_ylabel('Age')
ax.set_zlabel('Income')
ax.scatter(X[:, 1], X[:, 0], X[:, 3], c= labels.astype(np.float))
def calculate_integrated_sliding2(self, integration_factor, output_images):
file_temp = h5py.File(self.results_folder + "/mask.hdf5", 'r')
dset = file_temp["Mask"]
mask = dset[...]
file_temp.close()
new_clean_data_list = []
for index, element in enumerate(self.clean_data_list):
if index == len(self.clean_data_list) - 1:
break
new_clean_data_list.append(
[self.clean_data_list[index], self.clean_data_list[index + 1]])
n = len(new_clean_data_list) - (len(new_clean_data_list) %
integration_factor)
new_clean_data_list = new_clean_data_list[:n]
groups_list = [
new_clean_data_list[i:i + integration_factor]
for i in range(0, len(new_clean_data_list), integration_factor)
]
integrated_sliding2_results_folder = self.results_folder + "/integrated_sliding2_factor_%d" % (
integration_factor)
if not os.path.exists(integrated_sliding2_results_folder):
os.makedirs(integrated_sliding2_results_folder)
integrated_sliding2_len = len(groups_list) - 1
mag_mean = np.zeros(integrated_sliding2_len)
phase_mean = np.zeros(integrated_sliding2_len)
phase_mean_sum = np.zeros(integrated_sliding2_len)
phase_std_dev = np.zeros(integrated_sliding2_len)
date_values = []
vmax = np.pi * self.lambda_d
vmin = -1 * vmax
for enum, group in enumerate(groups_list):
if enum == len(groups_list) - 1:
break
for index, element in enumerate(groups_list[enum]):
data0 = self.sar_collection.find_one({'take': str(element[0])})
file_temp0 = h5py.File(data0['route'], 'r')
dset0 = file_temp0["Complex_image"]
data1 = self.sar_collection.find_one({'take': str(element[1])})
file_temp1 = h5py.File(data1['route'], 'r')
dset1 = file_temp1["Complex_image"]
if index == 0:
Imagen = np.sqrt(dset0[...] * np.conj(dset1[...]))
date_begin = data0['date']
time_begin = data0['time']
else:
Imagen += np.sqrt(dset0[...] * np.conj(dset1[...]))
file_temp0.close()
file_temp1.close()
if index == len(group) - 1:
date = data1['date']
time = data1['time']
Imagen /= integration_factor
phase = np.angle(Imagen)
magnitude = np.absolute(Imagen)
masked_angle = np.ma.masked_where(mask == 0, phase)
masked_magnitude = np.ma.masked_where(mask == 0, magnitude)
masked_plot = np.ma.masked_where(mask == 1, magnitude)
mag_mean[enum] = masked_magnitude.mean()
phase_mean[enum] = masked_angle.mean() * self.lambda_d
phase_mean_sum[enum] = np.sum(phase_mean)
phase_std_dev[enum] = np.std(masked_angle) * self.lambda_d
date_values.append(
datetime.strptime(''.join((date_begin, time_begin)), ''.join(
(date_format, time_format))))
if output_images:
fig = plt.figure(1)
plt.title(
"Integration factor: {integration_factor} \n From {date_begin} {time_begin} \n to {date} {time} (UTC)"
.format(integration_factor=integration_factor,
date_begin=date_begin,
time_begin=time_begin,
date=date,
time=time),
fontsize=11)
plt.ylabel('Range (m)', fontsize=12)
plt.xlabel('Cross-range (m)', fontsize=12)
if enum == 0:
im = plt.imshow(
self.lambda_d * phase,
cmap='jet',
aspect='auto',
extent=[self.xi, self.xf, self.yi, self.yf],
vmin=vmin,
vmax=vmax)
cbar = plt.colorbar(im,
orientation='vertical',
format='%.2f')
cbar.ax.set_title('Displacement \n (mm)', fontsize=10)
aux = int(
(self.yf - (self.xf * np.tan(48.0 * np.pi / 180.0))) *
Imagen.shape[0] / (self.yf - self.yi))
mask_aux = np.zeros(Imagen.shape)
count = 0
nposx = Imagen.shape[1]
nposy = Imagen.shape[0]
for k in range(nposy):
if k >= (aux + 1):
mask_aux[k, 0:count] = 1
mask_aux[k, nposx - count - 1:nposx - 1] = 1
count = count + 1
masked_values = np.ma.masked_where(mask_aux == 0, mask_aux)
plt.imshow(masked_values,
cmap='binary',
aspect='auto',
extent=[self.xi, self.xf, self.yi, self.yf])
plt.imshow(masked_plot,
cmap='binary',
aspect='auto',
extent=[self.xi, self.xf, self.yi, self.yf])
if enum > 0:
im = plt.imshow(
self.lambda_d * phase,
cmap='jet',
aspect='auto',
extent=[self.xi, self.xf, self.yi, self.yf],
vmin=vmin,
vmax=vmax)
plt.imshow(masked_values,
cmap='binary',
aspect='auto',
extent=[self.xi, self.xf, self.yi, self.yf])
plt.imshow(masked_plot,
cmap='binary',
aspect='auto',
extent=[self.xi, self.xf, self.yi, self.yf])
plt.savefig(integrated_sliding2_results_folder +
"/groups_%d_%d.png" % (enum, enum + 1))
fig = plt.figure(1)
fig.clear()
fig = plt.figure(figsize=(10.0, 6.0))
plt.subplot(221)
plt.title(r'$\overline{\Delta r}\/\/vs\/\/time$', fontsize=16)
plt.ylabel(r'$\overline{\Delta r}\/\/(mm)$', fontsize=16)
plt.xlabel(r'$time$', fontsize=16)
plt.plot(date_values, phase_mean)
ax = plt.gca()
ax.set_ylim(
[-(np.amax(phase_mean) * 2.0), (np.amax(phase_mean) * 2.0)])
ax.xaxis.set_major_formatter(DateFormatter('%d/%m %H:%M'))
labels = ax.get_xticklabels()
plt.setp(labels, rotation=30, fontsize=10)
plt.subplot(222)
plt.title(r'$\overline{\Delta r_{acc}}\/\/(mm)\/\/vs\/\/time$',
fontsize=16)
plt.ylabel(r'$\overline{\Delta r}\/\/(acc)$', fontsize=16)
plt.xlabel(r'$time$', fontsize=16)
plt.plot(date_values, phase_mean_sum)
ax = plt.gca()
ax.xaxis.set_major_formatter(DateFormatter('%d/%m %H:%M'))
labels = ax.get_xticklabels()
plt.setp(labels, rotation=30, fontsize=10)
sub = plt.subplot(223)
plt.title(r'$\sigma_{\Delta r}\/\/vs\/\/time$', fontsize=16)
plt.ylabel(r'$\sigma_{\Delta r}\/\/(mm)$', fontsize=16)
plt.xlabel(r'$time$', fontsize=16)
plt.plot(date_values, phase_std_dev)
ax = plt.gca()
ax.set_ylim([0.0, np.amax(phase_std_dev) * 1.2])
ax.xaxis.set_major_formatter(DateFormatter('%d/%m %H:%M'))
labels = ax.get_xticklabels()
plt.setp(labels, rotation=30, fontsize=10)
plt.subplot(224)
plt.title(r'$\overline{mag}\/\/vs\/\/time$', fontsize=16)
plt.ylabel(r'$\overline{mag}$', fontsize=16)
plt.xlabel(r'$time$', fontsize=16)
#plt.plot(date_values, mag_mean * 2 * 1e6)
plt.plot(date_values, mag_mean)
ax = plt.gca()
ax.xaxis.set_major_formatter(DateFormatter('%d/%m %H:%M'))
#ax.set_ylim([0.0 , np.amax(mag_mean * 2 * 1e6) * 1.2])
ax.set_ylim([0.0, np.amax(mag_mean) * 1.2])
labels = ax.get_xticklabels()
plt.setp(labels, rotation=30, fontsize=10)
plt.tight_layout()
plt.savefig(integrated_sliding2_results_folder +
"/statistical_report.png")
print "Done!"
def run_analysis(data, newFig=True, label='Data', color='k'):
#Sets up figure
if newFig is True:
fig = plt.figure(figsize=(18, 12))
gs = fig.add_gridspec(2, 2)
ax_pS = fig.add_subplot(gs[0, 0])
ax_pD = fig.add_subplot(gs[0, 1])
ax_avgS = fig.add_subplot(gs[1, 0])
ax_shape = fig.add_subplot(gs[1, 1])
else:
fig = plt.gcf()
if len(fig.get_axes()) != 4:
ValueError('Current figure does not have a 2x2 layout.')
ax_pS, ax_pD, ax_avgS, ax_shape = fig.get_axes()
#Analyzes avalanches
avalanches = get_avalanches(data)
S_list = [avalanches[i]['S'] for i in avalanches.keys()]
D_list = [avalanches[i]['D'] for i in avalanches.keys()]
shape_list = [avalanches[i]['shape'] for i in avalanches.keys()]
#Calculates S_avg
S_avg = np.zeros((np.max(D_list), 3))
for i in range(np.max(D_list)):
S_avg[i, 0] = i + 1
S_D = [
avalanches[j]['S'] for j in avalanches.keys()
if avalanches[j]['D'] == i + 1
]
S_avg[i, 1] = np.mean(S_D)
S_avg[i, 2] = np.std(S_D)
#Plots p(S)
fit_pS = powerlaw.Fit(S_list, xmin=1)
str_label = label + r': $\alpha$ = {:0.3f}'.format(fit_pS.power_law.alpha)
fit_pS.plot_pdf(ax=ax_pS, color=color, **{'label': str_label})
fit_pS.power_law.plot_pdf(ax=ax_pS, color='k', linestyle='--')
#Plots p(D)
fit_pD = powerlaw.Fit(D_list, xmin=1)
str_label = label + r': $\beta$ = {:0.3f}'.format(fit_pD.power_law.alpha)
fit_pD.plot_pdf(ax=ax_pD, color=color, **{'label': str_label})
fit_pD.power_law.plot_pdf(ax=ax_pD, color='k', linestyle='--')
#Plots <S>(D)
fit_gamma, _, _ = fit_powerlaw(S_avg[:, 0],
S_avg[:, 1],
S_avg[:, 2],
loglog=True)
str_label = label + r': $\gamma$ = {:0.3f}'.format(fit_gamma)
ax_avgS.plot(S_avg[:, 0], S_avg[:, 1], label=str_label, color=color)
ax_avgS.plot(S_avg[:, 0],
np.power(S_avg[:, 0], fit_gamma),
color='k',
linestyle='--')
#Fits and plots the average avalanche shape
fit_gamma_shape = fit_collapse(shape_list, 4, 20, extrapolate=True)
str_leg = label + r': $\gamma_s$ = {:0.2f}'.format(fit_gamma_shape)
plot_collapse(shape_list,
fit_gamma_shape,
4,
20,
ax_shape,
str_leg,
True,
color,
show_subplots=False)
print('== Exponents for {:s} =='.format(label))
print('alpha = {:0.3f}'.format(fit_pS.power_law.alpha))
print('beta = {:0.3f}'.format(fit_pD.power_law.alpha))
print('gamma_scaling = {:0.3f}'.format(
(fit_pD.power_law.alpha - 1) / (fit_pS.power_law.alpha - 1)))
print('gamma = {:0.3f}'.format(fit_gamma))
print('gamma_shape = {:0.3f}'.format(fit_gamma_shape))
#Beautifies plots
plt.sca(ax_pS)
plt.legend(loc='upper right')
plt.xlabel('S')
plt.ylabel('p(S)')
plt.sca(ax_pD)
plt.legend(loc='upper right')
plt.xlabel('D')
plt.ylabel('p(D)')
plt.sca(ax_avgS)
plt.legend(loc='upper left')
plt.xlabel('D')
plt.ylabel(r'$\langle S \rangle$ (D)')
plt.xlim([1, 1e3])
plt.ylim([1, 1e5])
ax_avgS.set_xscale('log')
ax_avgS.set_yscale('log')
def clean_data_report(self, threshold):
self.clean_data_list = []
file_temp = h5py.File(self.results_folder + "/mask.hdf5", 'r')
dset = file_temp["Mask"]
mask = dset[...]
file_temp.close()
clean_data_set = set()
for take in range(1, self.ntakes):
data = self.sar_collection.find_one({'take': str(take)})
file_temp = h5py.File(data['route'], 'r')
dset = file_temp["Complex_image"]
Imagen_master = dset[...]
file_temp.close()
data = self.sar_collection.find_one({'take': str(take + 1)})
date = (data['date'])
time = (data['time'])
file_temp = h5py.File(data['route'], 'r')
dset = file_temp["Complex_image"]
Imagen_slave = dset[...]
file_temp.close()
phase = np.angle(Imagen_master * np.conj(Imagen_slave))
masked_angle = np.ma.masked_where(mask == 0, phase)
phase_std_dev = np.std(masked_angle) * self.lambda_d
if phase_std_dev < threshold:
#clean_data_list.append([take, take +1])
clean_data_set.add(take)
clean_data_set.add(take + 1)
self.clean_data_list = list(clean_data_set)
clean_results_folder = self.results_folder + "/clean_data"
if not os.path.exists(clean_results_folder):
os.makedirs(clean_results_folder)
report_len = len(self.clean_data_list) - 1
mag_mean = np.zeros(report_len)
phase_mean = np.zeros(report_len)
phase_mean_sum = np.zeros(report_len)
phase_std_dev = np.zeros(report_len)
date_values = []
fig = plt.figure(1)
vmax = np.pi * self.lambda_d
vmin = -1 * vmax
for index, element in enumerate(self.clean_data_list):
if index == len(self.clean_data_list) - 1:
break
master = self.clean_data_list[index]
slave = self.clean_data_list[index + 1]
print "Processing take %d and %d." % (master, slave)
data = self.sar_collection.find_one({'take': str(master)})
file_temp = h5py.File(data['route'], 'r')
dset = file_temp["Complex_image"]
Imagen_master = dset[...]
file_temp.close()
data = self.sar_collection.find_one({'take': str(slave)})
date = (data['date'])
time = (data['time'])
file_temp = h5py.File(data['route'], 'r')
dset = file_temp["Complex_image"]
Imagen_slave = dset[...]
file_temp.close()
phase = np.angle(Imagen_master * np.conj(Imagen_slave))
magnitude = np.absolute(Imagen_master)
masked_angle = np.ma.masked_where(mask == 0, phase)
masked_magnitude = np.ma.masked_where(mask == 0, magnitude)
mag_mean[index] = masked_magnitude.mean()
phase_mean[index] = masked_angle.mean() * self.lambda_d
phase_mean_sum[index] = np.sum(phase_mean)
phase_std_dev[index] = np.std(masked_angle) * self.lambda_d
date_values.append(
datetime.strptime(''.join((date, time)), ''.join(
(date_format, time_format))))
fig.clear()
fig = plt.figure(figsize=(15.0, 8.0))
plt.subplot(221)
plt.title(r'$\overline{\Delta r}\/\/vs\/\/time$', fontsize=16)
plt.ylabel(r'$\overline{\Delta r}\/\/(mm)$', fontsize=16)
plt.xlabel(r'$time$', fontsize=16)
plt.plot(date_values, phase_mean)
ax = plt.gca()
ax.set_ylim(
[-(np.amax(phase_mean) * 2.0), (np.amax(phase_mean) * 2.0)])
ax.xaxis.set_major_formatter(DateFormatter('%d/%m %H:%M'))
labels = ax.get_xticklabels()
plt.setp(labels, rotation=30, fontsize=10)
plt.subplot(222)
plt.title(r'$\overline{\Delta r}\/\/(acc)\/\/vs\/\/time$', fontsize=16)
plt.ylabel(r'$\overline{\Delta r}\/\/(acc)$', fontsize=16)
plt.xlabel(r'$time$', fontsize=16)
plt.plot(date_values, phase_mean_sum)
ax = plt.gca()
ax.xaxis.set_major_formatter(DateFormatter('%d/%m %H:%M'))
labels = ax.get_xticklabels()
plt.setp(labels, rotation=30, fontsize=10)
sub = plt.subplot(223)
plt.title(r'$\sigma_{\Delta r}\/\/vs\/\/time$', fontsize=16)
plt.ylabel(r'$\sigma_{\Delta r}\/\/(mm)$', fontsize=16)
plt.xlabel(r'$time$', fontsize=16)
plt.plot(date_values, phase_std_dev)
ax = plt.gca()
ax.set_ylim([0.0, np.amax(phase_std_dev) * 1.2])
ax.xaxis.set_major_formatter(DateFormatter('%d/%m %H:%M'))
labels = ax.get_xticklabels()
plt.setp(labels, rotation=30, fontsize=10)
plt.subplot(224)
plt.title(r'$\overline{mag}\/\/vs\/\/time\/\/(normalized)$',
fontsize=16)
plt.ylabel(r'$\overline{mag}$', fontsize=16)
plt.xlabel(r'$time$', fontsize=16)
plt.plot(date_values, mag_mean)
ax = plt.gca()
ax.xaxis.set_major_formatter(DateFormatter('%d/%m %H:%M'))
labels = ax.get_xticklabels()
plt.setp(labels, rotation=30, fontsize=10)
plt.tight_layout()
plt.savefig(clean_results_folder + "/statistical_report.png")
print "Done!"
t_Z_ON.append(df2[0][data])
f_Z_ON.append(df2[1][data])
for data in range(len(df3[0])):
t_Z_OFF_F.append(df3[0][data])
f_Z_OFF_F.append(df3[1][data])
#Realizamos el analisis de deltas
[time1, delta_data, mean_delta, factor] = Analisys1(f_Z_ON, window, mean_factor)
#Realizamos la seleccion
[time2, frequency_data] = Analisys2(f_Z_ON, mean_delta, window, factor)
#pdb.set_trace()
fig1 = plt.figure()
ax1 = fig1.add_subplot(111)
ax1.set_ylim(min(delta_data), max(delta_data) + 0.2)
# ax1.set_xlim(min(time), max(time))
plt.xticks(np.arange(min(time1), max(time1), step=60), fontsize=8)
plt.yticks(np.arange(min(delta_data), max(delta_data) + 0.2, step=0.1), fontsize=8)
ax1.set_xlabel("time(s)", fontsize=6)
ax1.set_ylabel("Delta Frequency (Hz)", fontsize=6)
ax1.grid()
# plt.xticks(np.arange(min(time), max(time), step=60), fontsize=8)
# plt.yticks(np.arange(min(f_data), min(f_data), step=0.15), fontsize=8)
ax1.plot(time1, delta_data)
fig2 = plt.figure()
def tm_algorithm(self, s21, Rnk, take, date, time, results_folder,
results_collection):
I = np.zeros([self.npos], dtype=np.complex64)
s21_arr = np.zeros([self.nx, self.nr], dtype=np.complex64)
nc0 = int(self.nfre / 2.0) #first chunk of the frequency: f0,fc
nc1 = int((self.nfre + 1) / 2.0) #first chunk of the frequency: fc,ff
s21_arr[:, 0:nc1] = s21[:, nc0:self.nfre] #invert array order
s21_arr[:, self.nr - nc0:self.nr] = s21[:, 0:nc0]
Fn0 = self.nr * ifft(s21_arr, n=self.nr)
for k in range(0, self.nx):
Fn = np.interp(Rnk[k, :] - R0, self.rn, np.real(
Fn0[k, :])) + 1j * np.interp(Rnk[k, :] - R0, self.rn,
np.imag(Fn0[k, :]))
Fn *= np.exp(4j * np.pi * (self.fre_min / c0) * (Rnk[k, :] - R0))
I += Fn
I /= (self.nfre * self.nx)
I = np.reshape(I, (self.nposy, self.nposx))
I = np.flipud(I)
data_results_folder = results_folder + "/data"
images_results_folder = results_folder + "/images"
if not os.path.exists(data_results_folder):
os.makedirs(data_results_folder)
if not os.path.exists(images_results_folder):
os.makedirs(images_results_folder)
f = h5py.File(data_results_folder + "/image%d.hdf5" % take, 'w')
dset = f.create_dataset("Complex_image", (self.nposy, self.nposx),
dtype=np.complex64)
dset.attrs['dx'] = self.dx
dset.attrs['dy'] = self.dy
dset.attrs['date'] = date
dset.attrs['time'] = time
dset.attrs['ntoma'] = take
dset.attrs['xi'] = self.xi
dset.attrs['xf'] = self.xf
dset.attrs['yi'] = self.yi
dset.attrs['yf'] = self.yf
dset.attrs['fi'] = self.fre_min
dset.attrs['ff'] = self.fre_max
dset.attrs['beam_angle'] = self.beam_angle
dset[...] = I
f.close()
post = {
"type": "data",
"take": str(take),
"date": date,
"time": time,
"route": data_results_folder + "/image%d.hdf5" % take
}
results_collection.insert(post)
post = None
I = 10 * np.log10(np.absolute(I))
fig = plt.figure(1)
if take == 1:
self.vmin = np.amin(I) + 28
self.vmax = np.amax(I)
aux = int((self.yf - (self.xf * np.tan(48.0 * np.pi / 180.0))) *
I.shape[0] / (self.yf - self.yi))
mask = np.zeros(I.shape)
count = 0
for k in range(self.nposy):
#if k >= (int(0.5 * self.nposx / np.tan(32.0 * np.pi /180.0)) + 1):
if k >= (aux + 1):
mask[k, 0:count] = 1
mask[k, self.nposx - count - 1:self.nposx - 1] = 1
count = count + 1
self.masked_values = np.ma.masked_where(mask == 0, mask)
im = plt.imshow(I,
cmap='jet',
aspect='auto',
extent=[self.xi, self.xf, self.yi, self.yf],
vmin=self.vmin,
vmax=self.vmax)
plt.imshow(self.masked_values,
cmap='Greys',
aspect='auto',
extent=[self.xi, self.xf, self.yi, self.yf],
vmin=self.vmin,
vmax=self.vmax,
interpolation='none')
cbar = plt.colorbar(im, orientation='vertical')
plt.ylabel('Range (m)', fontsize=14)
plt.xlabel('Cross-range (m)', fontsize=14)
plt.savefig(images_results_folder + '/image%d.png' % take)
fig.clear()
def calculate_integrated_sliding(self, integration_factor):
file_temp = h5py.File(self.results_folder + "/mask.hdf5", 'r')
dset = file_temp["Mask"]
mask = dset[...]
file_temp.close()
n = len(self.clean_data_list) - (len(self.clean_data_list) %
integration_factor)
self.clean_data_list = self.clean_data_list[:n]
groups_list = [
self.clean_data_list[i:i + integration_factor]
for i in range(0, len(self.clean_data_list), integration_factor)
]
integrated_sliding_results_folder = self.results_folder + "/integrated_sliding_factor_%d" % (
integration_factor)
if not os.path.exists(integrated_sliding_results_folder):
os.makedirs(integrated_sliding_results_folder)
integrated_sliding_len = len(groups_list) - 1
mag_mean = np.zeros(integrated_sliding_len)
phase_mean = np.zeros(integrated_sliding_len)
phase_mean_sum = np.zeros(integrated_sliding_len)
phase_std_dev = np.zeros(integrated_sliding_len)
date_values = []
fig = plt.figure(1)
vmax = np.pi * self.lambda_d
vmin = -1 * vmax
for enum, group in enumerate(groups_list):
if enum == len(groups_list) - 1:
break
for index, element in enumerate(groups_list[enum]):
data = self.sar_collection.find_one({'take': str(element)})
file_temp = h5py.File(data['route'], 'r')
dset = file_temp["Complex_image"]
if index == 0:
Imagen_master = dset[...]
file_temp.close()
continue
Imagen_master += dset[...]
file_temp.close()
Imagen_master /= integration_factor
for index, element in enumerate(groups_list[enum + 1]):
data = self.sar_collection.find_one({'take': str(element)})
file_temp = h5py.File(data['route'], 'r')
dset = file_temp["Complex_image"]
if index == 0:
Imagen_slave = dset[...]
file_temp.close()
continue
Imagen_slave += dset[...]
file_temp.close()
if index == len(group) - 1:
date = (data['date'])
time = (data['time'])
Imagen_slave /= integration_factor
phase = np.angle(Imagen_master * np.conj(Imagen_slave))
magnitude = np.absolute(Imagen_master)
masked_angle = np.ma.masked_where(mask == 0, phase)
masked_magnitude = np.ma.masked_where(mask == 0, magnitude)
mag_mean[enum] = masked_magnitude.mean()
phase_mean[enum] = masked_angle.mean() * self.lambda_d
phase_mean_sum[enum] = np.sum(phase_mean)
phase_std_dev[enum] = np.std(masked_angle) * self.lambda_d
date_values.append(
datetime.strptime(''.join((date, time)), ''.join(
(date_format, time_format))))
#fig.suptitle("Image %d and %d" %(take, take + 1))
plt.ylabel('Range (m)', fontsize=14)
plt.xlabel('Cross-range (m)', fontsize=14)
if enum == 0:
im = plt.imshow(self.lambda_d * phase,
cmap='jet',
aspect='auto',
extent=[self.xi, self.xf, self.yi, self.yf],
vmin=vmin,
vmax=vmax)
cbar = plt.colorbar(im, orientation='vertical', format='%.2f')
im = plt.imshow(self.lambda_d * phase,
cmap='jet',
aspect='auto',
extent=[self.xi, self.xf, self.yi, self.yf],
vmin=vmin,
vmax=vmax)
plt.savefig(integrated_sliding_results_folder +
"/groups_%d_%d_takes%s_%s.png" %
(enum, enum + 1, tuple(groups_list[enum]),
tuple(groups_list[enum + 1])))
fig.clear()
fig = plt.figure(figsize=(15.0, 8.0))
plt.subplot(221)
plt.title(r'$\overline{\Delta r}\/\/vs\/\/time$', fontsize=16)
plt.ylabel(r'$\overline{\Delta r}\/\/(mm)$', fontsize=16)
plt.xlabel(r'$time$', fontsize=16)
plt.plot(date_values, phase_mean)
ax = plt.gca()
ax.set_ylim(
[-(np.amax(phase_mean) * 2.0), (np.amax(phase_mean) * 2.0)])
ax.xaxis.set_major_formatter(DateFormatter('%d/%m %H:%M'))
labels = ax.get_xticklabels()
plt.setp(labels, rotation=30, fontsize=10)
plt.subplot(222)
plt.title(r'$\overline{\Delta r}\/\/(acc)\/\/vs\/\/time$', fontsize=16)
plt.ylabel(r'$\overline{\Delta r}\/\/(acc)$', fontsize=16)
plt.xlabel(r'$time$', fontsize=16)
plt.plot(date_values, phase_mean_sum)
ax = plt.gca()
ax.xaxis.set_major_formatter(DateFormatter('%d/%m %H:%M'))
labels = ax.get_xticklabels()
plt.setp(labels, rotation=30, fontsize=10)
sub = plt.subplot(223)
plt.title(r'$\sigma_{\Delta r}\/\/vs\/\/time$', fontsize=16)
plt.ylabel(r'$\sigma_{\Delta r}\/\/(mm)$', fontsize=16)
plt.xlabel(r'$time$', fontsize=16)
plt.plot(date_values, phase_std_dev)
ax = plt.gca()
ax.set_ylim([0.0, np.amax(phase_std_dev) * 1.2])
ax.xaxis.set_major_formatter(DateFormatter('%d/%m %H:%M'))
labels = ax.get_xticklabels()
plt.setp(labels, rotation=30, fontsize=10)
plt.subplot(224)
plt.title(r'$\overline{mag}\/\/vs\/\/time\/\/(normalized)$',
fontsize=16)
plt.ylabel(r'$\overline{mag}$', fontsize=16)
plt.xlabel(r'$time$', fontsize=16)
plt.plot(date_values, mag_mean * 2)
ax = plt.gca()
ax.set_ylim([0.0, np.amax(mag_mean * 2) * 1.2])
ax.xaxis.set_major_formatter(DateFormatter('%d/%m %H:%M'))
labels = ax.get_xticklabels()
plt.setp(labels, rotation=30, fontsize=10)
plt.tight_layout()
plt.savefig(integrated_sliding_results_folder +
"/statistical_report.png")
print "Done!"
def plot(self,
min_freq,
output="VEL",
location="*",
channel="*",
time=None,
starttime=None,
endtime=None,
axes=None,
unwrap_phase=False,
plot_degrees=False,
show=True,
outfile=None):
"""
Show bode plot of instrument response of all (or a subset of) the
station's channels.
:type min_freq: float
:param min_freq: Lowest frequency to plot.
:type output: str
:param output: Output units. One of:
``"DISP"``
displacement, output unit is meters
``"VEL"``
velocity, output unit is meters/second
``"ACC"``
acceleration, output unit is meters/second**2
:type location: str
:param location: Only plot matching channels. Accepts UNIX style
patterns and wildcards (e.g. ``"BH*"``, ``"BH?"``, ``"*Z"``,
``"[LB]HZ"``; see :func:`~fnmatch.fnmatch`)
:type channel: str
:param channel: Only plot matching channels. Accepts UNIX style
patterns and wildcards (e.g. ``"BH*"``, ``"BH?"``, ``"*Z"``,
``"[LB]HZ"``; see :func:`~fnmatch.fnmatch`)
:param time: Only show channels active at given point in time.
:type starttime: :class:`~obspy.core.utcdatetime.UTCDateTime`
:param starttime: Only show channels active at or after given point in
time (i.e. channels ending before given time will not be shown).
:type endtime: :class:`~obspy.core.utcdatetime.UTCDateTime`
:param endtime: Only show channels active before or at given point in
time (i.e. channels starting after given time will not be shown).
:type axes: list of 2 :class:`matplotlib.axes.Axes`
:param axes: List/tuple of two axes instances to plot the
amplitude/phase spectrum into. If not specified, a new figure is
opened.
:type unwrap_phase: bool
:param unwrap_phase: Set optional phase unwrapping using NumPy.
:type plot_degrees: bool
:param plot_degrees: if ``True`` plot bode in degrees
:type show: bool
:param show: Whether to show the figure after plotting or not. Can be
used to do further customization of the plot before showing it.
:type outfile: str
:param outfile: Output file path to directly save the resulting image
(e.g. ``"/tmp/image.png"``). Overrides the ``show`` option, image
will not be displayed interactively. The given path/file name is
also used to automatically determine the output format. Supported
file formats depend on your matplotlib backend. Most backends
support png, pdf, ps, eps and svg. Defaults to ``None``.
.. rubric:: Basic Usage
>>> from obspy import read_inventory
>>> sta = read_inventory()[0][0]
>>> sta.plot(0.001, output="VEL", channel="*Z") # doctest: +SKIP
.. plot::
from obspy import read_inventory
sta = read_inventory()[0][0]
sta.plot(0.001, output="VEL", channel="*Z")
"""
import matplotlib.pyplot as plt
if axes is not None:
ax1, ax2 = axes
fig = ax1.figure
else:
fig = plt.figure()
ax1 = fig.add_subplot(211)
ax2 = fig.add_subplot(212, sharex=ax1)
matching = self.select(location=location,
channel=channel,
time=time,
starttime=starttime,
endtime=endtime)
for cha in matching.channels:
try:
cha.plot(min_freq=min_freq,
output=output,
axes=(ax1, ax2),
label=".".join(
(self.code, cha.location_code, cha.code)),
unwrap_phase=unwrap_phase,
plot_degrees=plot_degrees,
show=False,
outfile=None)
except ZeroSamplingRate:
msg = ("Skipping plot of channel with zero "
"sampling rate:\n%s")
warnings.warn(msg % str(cha), UserWarning)
except ObsPyException as e:
msg = "Skipping plot of channel (%s):\n%s"
warnings.warn(msg % (str(e), str(cha)), UserWarning)
# final adjustments to plot if we created the figure in here
if not axes:
from obspy.core.inventory.response import _adjust_bode_plot_figure
_adjust_bode_plot_figure(fig,
plot_degrees=plot_degrees,
show=False)
if outfile:
fig.savefig(outfile)
else:
if show:
plt.show()
return fig
def calculate_sliding(self, threshold, output_images):
self.threshold = threshold
print "Processing data..."
correlation = self.calculate_correlation()
aux = int((self.yf - (self.xf * np.tan(48.0 * np.pi / 180.0))) *
correlation.shape[0] / (self.yf - self.yi))
mask_aux = np.zeros(correlation.shape)
count = 0
nposx = correlation.shape[1]
nposy = correlation.shape[0]
for k in range(nposy):
if k >= (aux + 1):
mask_aux[k, 0:count] = 1
mask_aux[k, nposx - count - 1:nposx - 1] = 1
count = count + 1
masked_values = np.ma.masked_where(mask_aux == 0, mask_aux)
fig = plt.figure(1)
plt.title("Complex correlation magnitude", fontsize=11)
im = plt.imshow(np.absolute(correlation),
cmap='jet',
aspect='auto',
extent=[self.xi, self.xf, self.yi, self.yf],
vmin=0,
vmax=1)
plt.imshow(masked_values,
cmap='Greys',
aspect='auto',
extent=[self.xi, self.xf, self.yi, self.yf])
plt.colorbar(im, orientation='vertical', format='%.1f')
plt.savefig(self.results_folder + "/complex_correlation_mag.png")
fig.clear()
mask = np.absolute(correlation)
mask[mask < threshold] = 0
mask[mask >= threshold] = 1
vmax = np.pi * self.lambda_d
vmin = -1 * vmax
mag_mean = np.zeros(self.ntakes - 1)
phase_mean = np.zeros(self.ntakes - 1)
phase_mean_sum = np.zeros(self.ntakes - 1)
phase_std_dev = np.zeros(self.ntakes - 1)
date_values = []
fig = plt.figure(1)
file_temp = h5py.File(self.results_folder + "/mask.hdf5", 'w')
dset = file_temp.create_dataset("Mask", (mask.shape), dtype=np.uint8)
dset[...] = mask
file_temp.close()
for take in range(1, self.ntakes):
print "Processing take %d and %d." % (take, take + 1)
data = self.sar_collection.find_one({'take': str(take)})
file_temp = h5py.File(data['route'], 'r')
dset = file_temp["Complex_image"]
Imagen_master = dset[...]
file_temp.close()
data = self.sar_collection.find_one({'take': str(take + 1)})
date = (data['date'])
time = (data['time'])
file_temp = h5py.File(data['route'], 'r')
dset = file_temp["Complex_image"]
Imagen_slave = dset[...]
file_temp.close()
phase = np.angle(Imagen_master * np.conj(Imagen_slave))
magnitude = np.absolute(Imagen_master)
masked_angle = np.ma.masked_where(mask == 0, phase)
masked_magnitude = np.ma.masked_where(mask == 0, magnitude)
mag_mean[take - 1] = masked_magnitude.mean()
phase_mean[take - 1] = masked_angle.mean() * self.lambda_d
phase_mean_sum[take - 1] = np.sum(phase_mean)
phase_std_dev[take - 1] = np.std(masked_angle) * self.lambda_d
date_values.append(
datetime.strptime(''.join((date, time)), ''.join(
(date_format, time_format))))
if output_images:
fig.suptitle("Image %d and %d" % (take, take + 1))
plt.ylabel('Range (m)', fontsize=14)
plt.xlabel('Cross-range (m)', fontsize=14)
if take == 1:
im = plt.imshow(
self.lambda_d * phase,
cmap='jet',
aspect='auto',
extent=[self.xi, self.xf, self.yi, self.yf],
vmin=vmin,
vmax=vmax)
cbar = plt.colorbar(im,
orientation='vertical',
format='%.2f')
im = plt.imshow(self.lambda_d * phase,
cmap='jet',
aspect='auto',
extent=[self.xi, self.xf, self.yi, self.yf],
vmin=vmin,
vmax=vmax)
plt.savefig(self.results_folder + "/take%d_%d.png" %
(take, (take + 1)))
fig.clear()
fig = plt.figure(figsize=(10.0, 6.0))
plt.subplot(221)
plt.title(r'$\overline{\Delta r}\/\/vs\/\/time$', fontsize=16)
plt.ylabel(r'$\overline{\Delta r}\/\/(mm)$', fontsize=16)
plt.xlabel(r'$time$', fontsize=16)
plt.plot(date_values, phase_mean)
ax = plt.gca()
ax.set_ylim(
[-(np.amax(phase_mean) * 2.0), (np.amax(phase_mean) * 2.0)])
ax.xaxis.set_major_formatter(DateFormatter('%d/%m %H:%M'))
labels = ax.get_xticklabels()
plt.setp(labels, rotation=30, fontsize=10)
plt.subplot(222)
plt.title(r'$\overline{\Delta r_{acc}}\/\/(mm)\/\/vs\/\/time$',
fontsize=16)
plt.ylabel(r'$\overline{\Delta r_{acc}}$', fontsize=16)
plt.xlabel(r'$time$', fontsize=16)
plt.plot(date_values, phase_mean_sum)
ax = plt.gca()
ax.xaxis.set_major_formatter(DateFormatter('%d/%m %H:%M'))
labels = ax.get_xticklabels()
plt.setp(labels, rotation=30, fontsize=10)
sub = plt.subplot(223)
plt.title(r'$\sigma_{\Delta r}\/\/vs\/\/time$', fontsize=16)
plt.ylabel(r'$\sigma_{\Delta r}\/\/(mm)$', fontsize=16)
plt.xlabel(r'$time$', fontsize=16)
plt.plot(date_values, phase_std_dev)
ax = plt.gca()
ax.set_ylim([0.0, np.amax(phase_std_dev) * 1.2])
ax.xaxis.set_major_formatter(DateFormatter('%d/%m %H:%M'))
labels = ax.get_xticklabels()
plt.setp(labels, rotation=30, fontsize=10)
plt.subplot(224)
plt.title(r'$\overline{mag}\/\/vs\/\/time$', fontsize=16)
plt.ylabel(r'$\overline{mag}$', fontsize=16)
plt.xlabel(r'$time$', fontsize=16)
plt.plot(date_values, mag_mean)
ax = plt.gca()
ax.set_ylim([0.0, np.amax(mag_mean) * 1.2])
ax.xaxis.set_major_formatter(DateFormatter('%d/%m %H:%M'))
labels = ax.get_xticklabels()
plt.setp(labels, rotation=30, fontsize=10)
plt.tight_layout()
plt.savefig(self.results_folder + "/statistical_report.png")
print "Done!"
Lz = np.array(Lz.tolist())
Jz = np.array(Jz.tolist())
ylist = np.sqrt(Jr[np.where(np.logical_not(np.isnan(Jr)))])
xlist = Lz[np.where(np.logical_not(np.isnan(Jr)))]/ ( 8. * 220.)
zlist = Jz[np.where(np.logical_not(np.isnan(Jr)))]
age = age[np.where(np.logical_not(np.isnan(Jr)))]
STylist = np.sqrt(STJr[np.where(np.logical_not(np.isnan(STJr)))])
STxlist = STLz[np.where(np.logical_not(np.isnan(STJr)))]/ ( 8. * 220.)
STzlist = STJz[np.where(np.logical_not(np.isnan(STJr)))]
STage = STage[np.where(np.logical_not(np.isnan(STJr)))]
agelist = [4, 7, 10]
fig = plt.figure(figsize=(7.1,3.3))
axarr = fig.subplots(1,3, subplot_kw={'projection': '3d'})
for i in range(len(axarr)):
thisage = agelist[i]
with warnings.catch_warnings(record=True):
warnings.simplefilter("ignore")
xlistles = xlist[np.where(np.logical_and(age > thisage-2, age < thisage))]
ylistles = ylist[np.where(np.logical_and(age > thisage-2, age < thisage))]
zlistles = zlist[np.where(np.logical_and(age > thisage-2, age < thisage))]
ageles = age[np.where(np.logical_and(age > thisage-2, age < thisage))]
STxlistles = STxlist[np.where(np.logical_and(STage > thisage-2, STage < thisage))]
STylistles = STylist[np.where(np.logical_and(STage > thisage-2, STage < thisage))]
STzlistles = STzlist[np.where(np.logical_and(STage > thisage-2, STage < thisage))]
STageles = STage[np.where(np.logical_and(STage > thisage-2, STage < thisage))]
def visualize(data_list, output_folder):
import sklearn.manifold
from matplotlib import pyplot as plt
M = 1
has_stage = 'stage_id_gt' in data_list['test_fs'][0]
for nplot in range(10):
Xall = []
yall = []
ysall = []
sall = []
call = []
for i in range(nplot * M, (nplot + 1) * M):
data = data_list['test_fs'][i]
flag = data['flag_s'].numpy()
T = flag.sum()
X = data['h'][0, :T]
y = data['y_full'].numpy()[0, :T]
ys = data['y_s'].numpy()[0, :T]
Xall.append(X)
yall.append(y + (i - nplot * M) * 40)
ysall.append(ys)
if has_stage:
sall.append(data['stage_id'].numpy()[0, :T])
call.append(data['in_stage_class_id'].numpy()[0, :T])
if M > 1:
X = np.concatenate(Xall, axis=0)
y = np.concatenate(yall, axis=0)
ys = np.concatenate(ysall, axis=0)
if has_stage:
s = np.concatenate(sall, axis=0)
c = np.concatenate(call, axis=0)
else:
X = Xall[0]
y = yall[0]
ys = ysall[0]
if has_stage:
s = np.concatenate(sall, axis=0)
c = np.concatenate(call, axis=0)
if has_stage:
color = s * 10 + c
else:
color = y
maxidx = data['y_gt'].numpy().max()
labeled = ys < maxidx
unlabeled = ys == maxidx
tsne = sklearn.manifold.TSNE(n_components=2, verbose=1, random_state=1234)
pca = sklearn.decomposition.PCA(n_components=2)
Z = tsne.fit_transform(X)
fig = plt.figure(figsize=(8, 5))
cmap = 'rainbow'
plt.scatter(
Z[labeled][:, 0],
Z[labeled][:, 1],
c=color[labeled],
cmap=cmap,
alpha=1.0,
s=50,
linewidths=2)
plt.scatter(
Z[unlabeled][:, 0],
Z[unlabeled][:, 1],
c=color[unlabeled],
cmap=cmap,
alpha=0.3,
s=15,
linewidths=2)
plt.xticks([])
plt.yticks([])
for j in range(X.shape[0]):
if 'stage_id_gt' in data_list['test_fs'][0]:
cls_txt = '{:d}'.format(
data_list['test_fs'][nplot]['in_stage_class_id'][0, j])
plt.annotate(cls_txt, (Z[j, 0], Z[j, 1]), fontsize=8)
else:
plt.annotate('{:d}'.format(y[j]), (Z[j, 0], Z[j, 1]), fontsize=8)
plt.savefig(os.path.join(output_folder, 'tsne-{:03d}.pdf'.format(nplot)))
zoom_range=1.,
horizontal_flip=True,
vertical_flip=True)
# we store loss and acc data to be able to plot the data
history = model.fit_generator(datagen.flow(train, encoded, batch_size = batch_size),
steps_per_epoch = int(np.ceil(train.shape[0] / float(batch_size))),
epochs = epochs,
validation_data = (test, encoded_test),
workers = 4)
model.evaluate(test, encoded_test)
# we can plot the history of losses and accuracy as we stored it
# Loss Curves
plt.figure(figsize=[8,6])
plt.plot(history.history['loss'],'r',linewidth=3.0)
plt.plot(history.history['val_loss'],'b',linewidth=3.0)
plt.legend(['Training loss', 'Validation Loss'],fontsize=18)
plt.xlabel('Epochs ',fontsize=16)
plt.ylabel('Loss',fontsize=16)
plt.title('Loss Curves',fontsize=16)
# Accuracy Curves
plt.figure(figsize=[8,6])
plt.plot(history.history['acc'],'r',linewidth=3.0)
plt.plot(history.history['val_acc'],'b',linewidth=3.0)
plt.legend(['Training Accuracy', 'Validation Accuracy'],fontsize=18)
plt.xlabel('Epochs ',fontsize=16)
plt.ylabel('Accuracy',fontsize=16)
plt.title('Accuracy Curves',fontsize=16)
data["Order Priority cat"] = data["Order Priority"].cat.codes
data["Category"] = data["Category"].astype('category')
data["Category cat"] = data["Category"].cat.codes
data["Sub-Category"] = data["Sub-Category"].astype('category')
data["Sub-Category cat"] = data["Sub-Category"].cat.codes
data["Days_to_Ship_Product"] = data["Days_to_Ship_Product"].astype('str')
data["Product shipping Days"] = [str(i).split(" ")[0] if " " in i else "not days" for i in
data["Days_to_Ship_Product"]]
#print(data) # now the data is cleaned with new features created.
#final df preprocessed and cleaned # removing sales column as it is correlated to price
finaldf = data.drop(columns=["Sales", "Ship Mode","Segment","Market","Region","Order Priority", "Product ID","Product Name","Category","Sub-Category","Days_to_Ship_Product"])
#print(finaldf)
finaldf.isna().sum() # to check which column has null values.
#Using Pearson Correlation - to check features correlation
plt.figure(figsize=(12,10))
cor = data.corr()
sns.heatmap(cor, annot=True, cmap=plt.cm.Reds)
plt.show()
#Correlation with output variable
cor_target = abs(cor["Quantity"])
relevant_features = cor_target[cor_target>0.5] # this is just a cross check to see if there are highly correlated variables and to remove them and have only 1 of the two
#print(relevant_features)
#fitting linear reg model for features selected
#X = pd.DataFrame(np.c_[finaldf['Discount'], finaldf['Shipping Cost'],finaldf['Price'], finaldf['Ship Mode cat'],finaldf['Segment cat'],finaldf['Market cat'],finaldf['Order month'],finaldf['Region cat'], finaldf['Order Priority cat'], finaldf['Category cat'],finaldf['Sub-Category cat'],finaldf['Product shipping Days']],
#columns=['Discount','Shipping Cost','Price','Ship Mode cat','Segment cat','Market cat','Order month','Region cat', 'Order Priority cat','Category cat','Sub-Category cat','Product shipping Days'])
#target = np.array(data["Quantity"])
#print(target)
#modeling #XGboost
cleandf = finaldf.iloc[:,3:15]