def networkx_plot_measure(self, measure: str, draw_node_names = True, draw_graphweights: bool = True, draw_edges: bool = False):
""" Plots all specimens of the main dict. A measure is selected for the edge weights.
Graphvis tries to get the optimal arrangement based on weights, hard to do for 2d space.
Use dot, neato and fdp seem to maximize distances and just create a circular arrangement.
:param measure: distance measure for the edges
:return: None
"""
import networkx as nx
from networkx.drawing.nx_agraph import graphviz_layout
import matplotlib.pyplot as plt
dist_matrix_dict = utils.dict_of_dicts_matrix_measure(data_dict=self.data_dict, measure=measure)
print("dist_matrix_dict", dist_matrix_dict)
G = nx.from_dict_of_dicts(d=dist_matrix_dict)
G.graph['edges']={'arrowsize':'4.0'}
# neato, fdp seem to mazimize --> circle shape
# dot works best, however, 2d representation is difficult
pos = graphviz_layout(G, prog="fdp")
if draw_edges:
nx.draw_networkx(G, pos=pos)
else:
nx.draw_networkx_nodes(G, pos=pos)
if draw_graphweights:
nx.draw_networkx_edge_labels(G, pos=pos)
if draw_node_names:
nx.draw_networkx_labels(G, pos)
plt.tight_layout()
# adjust the [x, y] values for fitting all the text
axes = plt.gca()
#axes.set_xlim([-60, 800])
# use these to turn off axis for plotting
#plt.xticks([])
#plt.yticks([])
plt.show()
def write_pc_hist(df, p, args):
# Pivot, and then drop the "cycles" level of the multi-index.
df = pd.pivot_table(df, index=["pc"], columns="operation")
df = df.fillna(0)
df = df.droplevel(level=0, axis="columns")
# Use the colors key order above to stack the bars,
# but first we have to pick stalls that are actually IN the CSV (not all are printed)
cols = [k for k in colors.keys() if k in df.columns]
df = df[cols]
# Remove all PCs that were specified using the without flag
filts = {int(pc, 16) for pc in args.without}
fi = [pc for pc in df.index if int(pc, 16) not in filts]
removed = [pc for pc in df.index if int(pc, 16) in filts]
df = df.loc[fi]
print(f"Removed PCs: {removed}")
height = df.shape[0] * (labelsize + 4) / 72
ax = df.plot.barh(stacked=True, figsize=(11, height), color=colors)
ax.set_ylabel("Program Counter")
ax.set_xlabel(f"Cycles * 10^{math.floor(math.log10(ax.get_xlim()[1]))}")
ax.set_title(f"HammerBlade Program Counter Cycles Histogram")
ax.tick_params(labelsize=labelsize)
fig = ax.get_figure()
plt.gca().invert_yaxis()
plt.legend(loc="upper left")
plt.tight_layout()
fig.savefig(p / "pc_hist.pdf")
plt.close(fig)
def single_plot_residuals(targets,
forecasts,
order,
tam=[8, 8],
save=False,
file=None):
fig, ax = plt.subplots(nrows=1, ncols=3, figsize=tam)
res = residuals(targets, forecasts, order)
ax[0].set_title("Residuals", size='large')
ax[0].set_ylabel("Model", size='large')
ax[0].set_xlabel(' ')
ax[0].plot(res)
ax[1].set_title("Residuals Autocorrelation", size='large')
ax[1].set_ylabel('ACS')
ax[1].set_xlabel('Lag')
ax[1].acorr(res)
ax[2].set_title("Residuals Histogram", size='large')
ax[2].set_ylabel('Freq')
ax[2].set_xlabel('Bins')
ax[2].hist(res)
plt.tight_layout()
Util.show_and_save_image(fig, file, save)
def plot_classification_report(cr,
title='Classification report ',
with_avg_total=False,
cmap=plt.cm.Blues):
lines = cr.split('\n')
classes = []
plotMat = []
for line in lines[2:(len(lines) - 3)]:
#print(line)
t = line.split()
if len(t):
classes.append(t[0])
v = [float(x) for x in t[1:len(t) - 1]]
print(v)
plotMat.append(v)
if with_avg_total:
aveTotal = lines[len(lines) - 1].split()
classes.append('avg/total')
vAveTotal = [float(x) for x in t[1:len(aveTotal) - 1]]
plotMat.append(vAveTotal)
plt.imshow(plotMat, interpolation='nearest', cmap=cmap)
plt.title(title)
plt.colorbar()
x_tick_marks = np.arange(3)
y_tick_marks = np.arange(len(classes))
plt.xticks(x_tick_marks, ['precision', 'recall', 'f1-score', 'support'],
rotation=45)
plt.yticks(y_tick_marks, classes)
plt.tight_layout()
plt.ylabel('Classes')
plt.xlabel('Measures')
def linear_regression(data, power, models_to_plot):
# initialize predictors:
predictors = ['x']
if power >= 2:
predictors.extend(['x_%d' % i for i in range(2, power + 1)])
# Fit the model
linreg = LinearRegression(normalize=True)
linreg.fit(data[predictors], data['y'])
y_pred = linreg.predict(data[predictors])
# Check if a plot is to be made for the entered power
if power in models_to_plot:
plt.subplot(models_to_plot[power])
plt.tight_layout()
plt.plot(data['x'], y_pred)
plt.plot(data['x'], data['y'], '.')
plt.title('Plot for power: %d' % power)
# Return the result in pre-defined_format
rss = sum((y_pred - data['y'])**2)
ret = [rss]
ret.extend([linreg.intercept_])
ret.extend(linreg.coef_)
return ret
def plot_confusion_matrix(confusion_matrix,
class_labels,
normalize=False,
title='Confusion Matrix',
cmap=plt.cm.Blues):
""" Code courtesy of Abinav Sagar: https://towardsdatascience.com/convolutional-neural-network-for-breast-cancer-classification-52f1213dcc9 """
if normalize:
confusion_matrix = confusion_matrix.astype(
'float') / confusion_matrix.sum(axis=1)[:, np.newaxis]
print("Normalized confusion matrix")
else:
print('Confusion matrix, without normalization')
print(confusion_matrix)
plt.imshow(confusion_matrix, interpolation='nearest', cmap=cmap)
plt.title(title)
plt.colorbar()
tick_marks = np.arange(len(class_labels))
plt.xticks(tick_marks, class_labels, rotation=55)
plt.yticks(tick_marks, class_labels)
fmt = '.2f' if normalize else 'd'
thresh = confusion_matrix.max() / 2.
for i, j in itertools.product(range(confusion_matrix.shape[0]),
range(confusion_matrix.shape[1])):
plt.text(j,
i,
format(confusion_matrix[i, j], fmt),
horizontalalignment="center",
color="white" if confusion_matrix[i, j] > thresh else "black")
plt.ylabel('True label')
plt.xlabel('Predicted label')
plt.tight_layout()
def _vis_proposals(im, dets, thresh=0.5):
"""Draw detected bounding boxes."""
inds = np.where(dets[:, -1] >= thresh)[0]
if len(inds) == 0:
return
class_name = 'obj'
im = im[:, :, (2, 1, 0)]
fig, ax = plt.subplots(figsize=(12, 12))
ax.imshow(im, aspect='equal')
for i in inds:
bbox = dets[i, :4]
score = dets[i, -1]
ax.add_patch(
plt.Rectangle((bbox[0], bbox[1]),
bbox[2] - bbox[0],
bbox[3] - bbox[1],
fill=False,
edgecolor='red',
linewidth=3.5))
ax.text(bbox[0],
bbox[1] - 2,
'{:s} {:.3f}'.format(class_name, score),
bbox=dict(facecolor='blue', alpha=0.5),
fontsize=14,
color='white')
ax.set_title(('{} detections with '
'p({} | box) >= {:.1f}').format(class_name, class_name,
thresh),
fontsize=14)
plt.axis('off')
plt.tight_layout()
plt.draw()
def plot_confusion_matrix(cm,
classes,
normalize=False,
title='Confusion matrix',
cmap=plt.cm.Blues,
filename='viz\\confusion_matrix.png'):
plt.figure()
plt.imshow(cm, interpolation='nearest', cmap=cmap)
plt.title(title)
plt.colorbar()
tick_marks = np.arange(len(classes))
plt.xticks(tick_marks, classes, rotation=45)
plt.yticks(tick_marks, classes)
if normalize: cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
thresh = cm.max() / 2.
for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
plt.text(j,
i,
cm[i, j],
horizontalalignment="center",
color="white" if cm[i, j] > thresh else "black")
plt.tight_layout()
plt.ylabel('True label')
plt.xlabel('Predicted label')
plt.savefig(filename)
def pftsExploreOrderAndPartitions(data, save=False, file=None):
fig, axes = plt.subplots(nrows=4, ncols=1, figsize=[6, 8])
data_fs1 = Grid.GridPartitioner(data=data, npart=10).sets
mi = []
ma = []
axes[0].set_title('Point Forecasts by Order')
axes[2].set_title('Interval Forecasts by Order')
for order in np.arange(1, 6):
fts = pwfts.ProbabilisticWeightedFTS("")
fts.shortname = "n = " + str(order)
fts.train(data, sets=data_fs1.sets, order=order)
point_forecasts = fts.forecast(data)
interval_forecasts = fts.forecast_interval(data)
lower = [kk[0] for kk in interval_forecasts]
upper = [kk[1] for kk in interval_forecasts]
mi.append(min(lower) * 0.95)
ma.append(max(upper) * 1.05)
for k in np.arange(0, order):
point_forecasts.insert(0, None)
lower.insert(0, None)
upper.insert(0, None)
axes[0].plot(point_forecasts, label=fts.shortname)
axes[2].plot(lower, label=fts.shortname)
axes[2].plot(upper)
axes[1].set_title('Point Forecasts by Number of Partitions')
axes[3].set_title('Interval Forecasts by Number of Partitions')
for partitions in np.arange(5, 11):
data_fs = Grid.GridPartitioner(data=data, npart=partitions).sets
fts = pwfts.ProbabilisticWeightedFTS("")
fts.shortname = "q = " + str(partitions)
fts.train(data, sets=data_fs.sets, order=1)
point_forecasts = fts.forecast(data)
interval_forecasts = fts.forecast_interval(data)
lower = [kk[0] for kk in interval_forecasts]
upper = [kk[1] for kk in interval_forecasts]
mi.append(min(lower) * 0.95)
ma.append(max(upper) * 1.05)
point_forecasts.insert(0, None)
lower.insert(0, None)
upper.insert(0, None)
axes[1].plot(point_forecasts, label=fts.shortname)
axes[3].plot(lower, label=fts.shortname)
axes[3].plot(upper)
for ax in axes:
ax.set_ylabel('F(T)')
ax.set_xlabel('T')
ax.plot(data, label="Original", color="black", linewidth=1.5)
handles, labels = ax.get_legend_handles_labels()
ax.legend(handles, labels, loc=2, bbox_to_anchor=(1, 1))
ax.set_ylim([min(mi), max(ma)])
ax.set_xlim([0, len(data)])
plt.tight_layout()
cUtil.show_and_save_image(fig, file, save)
def plot_sets(data, sets, titles, tam=[12, 10], save=False, file=None):
num = len(sets)
#fig = plt.figure(figsize=tam)
maxx = max(data)
minx = min(data)
#h = 1/num
#print(h)
fig, axes = plt.subplots(nrows=num, ncols=1, figsize=tam)
for k in np.arange(0, num):
ticks = []
x = []
ax = axes[k]
ax.set_title(titles[k])
ax.set_ylim([0, 1.1])
for key in sets[k].keys():
s = sets[k][key]
if s.mf == Membership.trimf:
ax.plot(s.parameters, [0, 1, 0])
elif s.mf == Membership.gaussmf:
tmpx = [kk for kk in np.arange(s.lower, s.upper)]
tmpy = [s.membership(kk) for kk in np.arange(s.lower, s.upper)]
ax.plot(tmpx, tmpy)
elif s.mf == Membership.trapmf:
ax.plot(s.parameters, [0, 1, 1, 0])
ticks.append(str(round(s.centroid, 0)) + '\n' + s.name)
x.append(s.centroid)
ax.xaxis.set_ticklabels(ticks)
ax.xaxis.set_ticks(x)
plt.tight_layout()
Util.show_and_save_image(fig, file, save)
def plot_df(df, y_label='', sliced=False, years=None, country=None):
"""Takes a DataFrame and plots each Country Name against each year, if
nothing is passed for sliced and years. If sliced=True and years is given
an integer value, then only the years given will be plotted for each Country
Name.
Params:
df, a pandas DataFrame
y_label, a string to give the y-axis a label
slice, a boolean, when True expects argument passed in for years & country
years, a list of years passed in to be plotted
country, a string
Returns:
a plot of Country Names against years with the respective percentages
of land designation for the given CSV data file
"""
df_year = df.loc[years]
labels = list(df_year.index)
ax = df_year.plot(kind='area', fontsize=10)
ax.set_xlabel("Country Name", fontsize=10)
ylabel = "% Land Area Designated for " + y_label
ax.set_ylabel(str(ylabel), fontsize=10)
ax.set_xticklabels(labels)
df_year.plot(kind='area', rot='vertical', legend=False, ax=ax) #or: xticks=df_year[labels],
plt.tight_layout()
plt.show()
def plot_confusion_matrix(cm,
classes,
normalize=False,
title='Confusion matrix',
cmap=plt.cm.Blues):
if normalize:
cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
print("Normalized confusion matrix")
else:
print("Confusion Matrix, without normalization")
print(cm)
#imshow displays data as an image on a 2d master
plt.imshow(cm, interpolation='nearest', cmap=cmap)
plt.title(title)
plt.colorbar()
#returns evenly spaced values with a given inteerval
tick_marks = np.arange(len(classes))
plt.xticks(tick_marks, classes, rotation=45)
plt.yticks(tick_marks, classes)
fmt = '.2f' if normalize else 'd'
thresh = cm.max() / 2.
for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
plt.text(j,
i,
format(cm[i, j], fmt),
horizontalalignment='center',
color='white' if cm[i, j] > thresh else "black")
plt.tight_layout()
plt.ylabel('True Label')
plt.xlabel('Predicted Label')
def plotImages(images_arr):
fig, axes = plt.subplots(1, 5, figsize=(20,20))
axes = axes.flatten()
for img, ax in zip( images_arr, axes):
ax.imshow(img)
plt.tight_layout()
plt.show()
def plot_sets(partitioner,
start=0,
end=10,
step=1,
tam=[5, 5],
colors=None,
save=False,
file=None,
axes=None,
data=None,
window_size=1,
only_lines=False,
legend=True):
range = np.arange(start, end, step)
ticks = []
if axes is None:
fig, axes = plt.subplots(nrows=1, ncols=1, figsize=tam)
for ct, key in enumerate(partitioner.ordered_sets):
fset = partitioner.sets[key]
if not only_lines:
for t in range:
tdisp = t - (t % window_size)
fset.membership(0, tdisp)
param = fset.perturbated_parameters[str(tdisp)]
if fset.mf == Membership.trimf:
if t == start:
line = axes.plot([t, t + 1, t], param, label=fset.name)
fset.metadata['color'] = line[0].get_color()
else:
axes.plot([t, t + 1, t],
param,
c=fset.metadata['color'])
ticks.extend(["t+" + str(t), ""])
else:
tmp = []
for t in range:
tdisp = t - (t % window_size)
fset.membership(0, tdisp)
param = fset.perturbated_parameters[str(tdisp)]
tmp.append(np.polyval(param, tdisp))
axes.plot(range, tmp, ls="--", c="blue")
axes.set_ylabel("Universe of Discourse")
axes.set_xlabel("Time")
plt.xticks([k for k in range], ticks, rotation='vertical')
if legend:
handles0, labels0 = axes.get_legend_handles_labels()
lgd = axes.legend(handles0, labels0, loc=2, bbox_to_anchor=(1, 1))
if data is not None:
axes.plot(np.arange(start, start + len(data), 1), data, c="black")
if file is not None:
plt.tight_layout()
Util.show_and_save_image(fig, file, save)
def plotConfusionMatrix(lbllist, predlist, classes, type):
confusionMatrix = confusion_matrix(lbllist, predlist)
# print(confusionMatrix)
plt.imshow(confusionMatrix, interpolation="nearest", cmap=plt.cm.Blues)
if type == 'train':
plt.title("Confusion matrix training")
elif type == 'test':
plt.title("Confusion matrix testing")
plt.colorbar()
tick_marks = np.arange(len(classes))
plt.xticks(tick_marks, classes, rotation=45)
plt.yticks(tick_marks, classes)
fmt = "d"
thresh = confusionMatrix.max() / 2.
for i, j in itertools.product(range(confusionMatrix.shape[0]),
range(confusionMatrix.shape[1])):
plt.text(j,
i,
format(confusionMatrix[i, j], fmt),
horizontalalignment="center",
color="white" if confusionMatrix[i, j] > thresh else "black")
plt.tight_layout()
plt.ylabel("True label")
plt.xlabel("Predicted label")
# plt.show()
if type == 'train':
plt.savefig(LOG_PATH + 'Confusion matrix training.png')
elif type == 'test':
plt.savefig(LOG_PATH + 'Confusion matrix testing.png')
plt.close()
def plot_confusion_matrix(cm, classes,
normalize=False,
title='Confusion matrix',
cmap=plt.cm.Blues):
"""
This function prints and plots the confusion matrix.
Normalization can be applied by setting `normalize=True`.
"""
plt.imshow(cm, interpolation='nearest', cmap=cmap)
plt.title(title)
plt.colorbar()
tick_marks = np.arange(len(classes))
plt.xticks(tick_marks, classes, rotation=45)
plt.yticks(tick_marks, classes)
if normalize:
cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
print("Normalized confusion matrix")
else:
print('Confusion matrix, without normalization')
print(cm)
thresh = cm.max() / 2.
for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
plt.text(j, i, cm[i, j],
horizontalalignment="center",
color="white" if cm[i, j] > thresh else "black")
plt.tight_layout()
plt.ylabel('True label')
plt.xlabel('Predicted label')
def plot_bars_means(q_mean, qj_mean_v1, qj_var_v1, pj_mean_v1, pj_var_v1, qs_hat_std_ols, qs_hat_avg_ols, simu_kind=""):
x_pos = 1.5 * np.arange(8)
print(qs_hat_avg_ols)
print(qs_hat_std_ols)
width=0.25
bar_width=0.25
fig, ax = plt.subplots()
bar_ols=ax.bar(x_pos, np.array(qs_hat_avg_ols), yerr=np.array(qs_hat_std_ols), width=bar_width,align='center', alpha=0.5, ecolor='black', capsize=5)
bar_qj=ax.bar(x_pos+width+width, np.array(qj_mean_v1), yerr=np.array(qj_var_v1),width=bar_width, align='center', alpha=0.5, ecolor='black', capsize=5)
bar_pj=ax.bar(x_pos+width, np.array(pj_mean_v1), yerr=np.array(pj_var_v1), width=bar_width,align='center', alpha=0.5, ecolor='black', capsize=5)
bar_q=ax.bar(x_pos+width+width+width, np.array(q_mean), width=bar_width, align='center', alpha=0.5, ecolor='black', capsize=5)
ax.set_title('Compering mean and variance between the following simulation : '+ simu_kind)
ax.set_xticks(x_pos)
names=PARTY_NAMES
rev_names = [name[::-1] for name in list(names)]
long_names=[]
dict = {"פה": "כחול לבן", "ג": "יהדות התורה", "שס": "שס", "מחל": "ליכוד", "ל": "ישראל ביתנו", "טב": "ימינה",
"אמת": "עבודה גשר מרץ", "ודעם": "הרשימה המשותפת"}
for name in rev_names:
long_names.append(dict[name[::-1]][::-1])
ax.set_xticklabels(long_names,rotation=45)
ax.set_ylabel('vote precentange ')
ax.yaxis.grid(True)
dummy_1 = Rectangle((0, 0), 1, 1, fc="w", fill=False, edgecolor='none', linewidth=0)
dummy_2 = Rectangle((0, 0), 1, 1, fc="w", fill=False, edgecolor='none', linewidth=0)
ax.legend((bar_qj[0], bar_pj[0],bar_ols[0], bar_q), ("qj_hat","pj","qj_ols_hat", "q"))
# Save the figure and show
plt.tight_layout()
plt.savefig('bar_plot_with_error_bars.png')
plt.show()
def plot_residuals(targets, models, tam=[8, 8], save=False, file=None):
fig, axes = plt.subplots(nrows=len(models), ncols=3, figsize=tam)
for c, mfts in enumerate(models, start=0):
if len(models) > 1:
ax = axes[c]
else:
ax = axes
forecasts = mfts.forecast(targets)
res = residuals(targets, forecasts, mfts.order)
mu = np.mean(res)
sig = np.std(res)
if c == 0: ax[0].set_title("Residuals", size='large')
ax[0].set_ylabel(mfts.shortname, size='large')
ax[0].set_xlabel(' ')
ax[0].plot(res)
if c == 0: ax[1].set_title("Residuals Autocorrelation", size='large')
ax[1].set_ylabel('ACS')
ax[1].set_xlabel('Lag')
ax[1].acorr(res)
if c == 0: ax[2].set_title("Residuals Histogram", size='large')
ax[2].set_ylabel('Freq')
ax[2].set_xlabel('Bins')
ax[2].hist(res)
plt.tight_layout()
Util.show_and_save_image(fig, file, save)
def visualizeRGB(win,
imSize=None,
hidDims=None,
ordered=False,
saveFile=None,
normalize=False):
w = win - np.min(win)
w /= np.max(w)
numVis, numHid = w.shape
numVis /= 3
if imSize is None:
imSize = (int(np.sqrt(numVis)), int(np.sqrt(numVis)))
assert (imSize[0] * imSize[1] == numVis)
if hidDims is None:
tmp = min(20, int(np.ceil(np.sqrt(numHid))))
hidDims = (tmp, tmp)
margin = 2
img = np.zeros((hidDims[0] * (imSize[0] + margin),
hidDims[1] * (imSize[1] + margin), 3))
if ordered:
valList = []
for h in range(numHid):
wtmp = w[:, h] - np.mean(w[:, h])
val = wtmp.dot(wtmp)
valList.append(val)
order = np.argsort(valList)[::-1]
for h in range(min(hidDims[0] * hidDims[1], numHid)):
i = h / hidDims[1]
j = h % hidDims[1]
if ordered:
hshow = order[h]
else:
hshow = h
for co in range(3):
tmp = np.reshape(w[(numVis * co):(numVis * (co + 1)), hshow],
imSize)
if normalize:
tmp -= tmp.min()
tmp /= tmp.max()
img[(i * (imSize[0] + margin)):(i * (imSize[0] + margin) +
imSize[0]),
(j * (imSize[1] + margin)):(j * (imSize[1] + margin) +
imSize[1]), co] = tmp
plt.axis('off')
if saveFile is not None:
plt.tight_layout()
plt.savefig('./figures/' + saveFile + ".svg",
bbox_inches='tight',
dpi=2000)
else:
plt.imshow(img)
plt.show()
def visualize(win,
rgb=False,
imSize=None,
hidDims=None,
ordered=False,
saveFile=None,
normalize=False):
if rgb:
visualizeRGB(win,
imSize=imSize,
hidDims=hidDims,
saveFile=saveFile,
normalize=normalize,
ordered=ordered)
return
w = win - np.min(win)
w /= np.max(w)
numVis, numHid = w.shape
if imSize is None:
imSize = (int(np.sqrt(numVis)), int(np.sqrt(numVis)))
assert (imSize[0] * imSize[1] == numVis)
if hidDims is None:
tmp = min(20, int(np.ceil(np.sqrt(numHid))))
hidDims = (tmp, tmp)
if ordered:
valList = []
for h in range(numHid):
wtmp = w[:, h] - np.mean(w[:, h])
val = wtmp.dot(wtmp)
valList.append(val)
order = np.argsort(valList)[::-1]
margin = 1
img = np.zeros(
(hidDims[0] * (imSize[0] + margin), hidDims[1] * (imSize[1] + margin)))
for h in range(min(hidDims[0] * hidDims[1], numHid)):
i = h / hidDims[1]
j = h % hidDims[1]
if ordered:
hshow = order[h]
else:
hshow = h
content = (np.reshape(w[:, hshow], imSize)) # - np.mean(w[:,hshow]))
img[(i * (imSize[0] + margin)):(i * (imSize[0] + margin) + imSize[0]),
(j * (imSize[1] + margin)):(j * (imSize[1] + margin) +
imSize[1])] = content
plt.figure()
plt.axis('off')
plt.imshow(img, cmap=plt.cm.Greys_r, interpolation="nearest")
if saveFile is not None:
plt.tight_layout()
plt.savefig('./figures/' + saveFile + ".svg",
bbox_inches='tight',
dpi=2000)
plt.show()
def write_bb_hist(df, p, args):
# Pivot, and then drop the "cycles" level of the multi-index.
df = pd.pivot_table(df, index=["pc"], columns="operation")
df = df.fillna(0)
df = df.droplevel(level=0, axis="columns")
# Group together floating point and regular instructionos
tot_instrs = df.Instruction + df["FPU Instruction"]
# Group together PCs that have the same number of executions
bb_ranges = (tot_instrs != tot_instrs.shift()).cumsum()
df = df.groupby(bb_ranges).sum()
# Get the new grouped index
bb_tups = [(bb_ranges[bb_ranges == x].index.min(),
bb_ranges[bb_ranges == x].index.max()) for x in df.index]
bb_ranges = [range(int(x, 16), int(y, 16) + 1) for (x, y) in bb_tups]
# Filter BBs that include PCs that were specified with --without
filts = {int(pc, 16) for pc in args.without}
df.index = [x + "-" + y for (x, y) in bb_tups]
fi = [
x + "-" + y for (x, y) in bb_tups
if (all(e not in range(int(x, 16),
int(y, 16) + 1) for e in filts))
]
removed = [
x + "-" + y for (x, y) in bb_tups
if (any(e in range(int(x, 16),
int(y, 16) + 1) for e in filts))
]
# TODO: Print filtered BBs on graph?
print(f"Removed Basic Blocks: {removed}")
df = df.loc[fi]
ipc = (df.Instruction + df["FPU Instruction"]) / df.sum(axis=1)
pct = 100.0 * df.sum(axis=1) / df.sum(axis=1).sum()
idx = df.index.to_series()
idx = idx.combine(pct, lambda i, pct: f"{i} ({pct:.0f}%".rjust(10))
idx = idx.combine(ipc, (lambda i, ipc: f"{i} @ {ipc:1.3f})"))
df.index = idx
# Use the colors key order above to stack the bars,
# but first we have to pick stalls that are actually IN the CSV (not all are printed)
cols = [k for k in colors.keys() if k in df.columns]
height = df.shape[0] * (labelsize + 4) / 72
ax = df[cols].plot.barh(stacked=True, figsize=(11, height), color=colors)
ax.set_ylabel("Basic Block Range (% Cycles @ IPC)")
ax.set_xlabel(f"Cycles * 10^{math.floor(math.log10(ax.get_xlim()[1]))}")
ax.set_title(f"HammerBlade Basic Block Cycles Histogram")
ax.tick_params(labelsize=labelsize)
fig = ax.get_figure()
plt.gca().invert_yaxis()
plt.legend(loc="upper right")
plt.tight_layout()
fig.savefig(p / "bb_hist.pdf")
plt.close(fig)
def log_transform(data, columns, plot=True, figsize=(12, 6)):
'''
Nomralizes the dataset to be as close to the gaussian distribution.
Parameter:
-----------------------------------------
data: DataFrame, Series.
Data to Log transform.
columns: List, Series
Columns to be transformed to normality using log transformation
plot: bool, default True
Plots a before and after log transformation plot
Returns:
Log-transformed dataframe
'''
if data is None:
raise ValueError(
"data: Expecting a DataFrame/ numpy2d array, got 'None'")
if columns is None:
raise ValueError(
"columns: Expecting at least a column in the list of columns but got 'None'"
)
df = data.copy()
for col in columns:
df[col] = np.log1p(df[col])
if plot:
for col in columns:
_ = plt.figure(figsize=figsize)
plt.subplot(1, 2, 1)
sns.distplot(data[col],
color="m",
label="Skewness : %.2f" % (df[col].skew()))
plt.title('Distribution of ' + col + " before Log transformation")
plt.legend(loc='best')
plt.subplot(1, 2, 2)
sns.distplot(df[col],
color="m",
label="Skewness : %.2f" % (df[col].skew()))
plt.title('Distribution of ' + col + " after Log transformation")
plt.legend(loc='best')
plt.tight_layout(2)
plt.show()
return df
def plotModelVarImp(self, model, X, y):
imp = pd.DataFrame(model.feature_importances_,
columns=['Importance'],
index=X.columns)
mpl.figure(figsize=(60, 80))
imp = imp.sort_values(['Importance'], ascending=False)
imp[:20].plot(kind='barh')
print(model.score(X, y))
mpl.tight_layout()
mpl.savefig('importance.png', dpi=150)
def imgOut(epoch, generator):
rand_nums = np.random.randint(0, x_test_hr.shape[0], size=1)
image_batch_hr = denormalize(x_test_hr[rand_nums])
image_batch_lr = x_test_lr[rand_nums]
gen_img = generator.predict(image_batch_lr)
generated_image = denormalize(gen_img)
image_batch_lr = denormalize(image_batch_lr)
image_batch_bc = upsample(image_batch_lr, 4)
plt.figure(figsize=(15, 5))
dim = (1, 4) # 4 wide
# lowres
plt.subplot(dim[0], dim[1], 1)
plt.title('Low Res')
plt.imshow(image_batch_lr[0], interpolation='nearest')
plt.axis('off')
# naive resize
plt.subplot(dim[0], dim[1], 2)
plt.title('Naive Resize')
plt.imshow(image_batch_bc[0], interpolation='nearest')
plt.axis('off')
# generated
plt.subplot(dim[0], dim[1], 3)
plt.title('Generated SR')
plt.imshow(generated_image[0], interpolation='nearest')
plt.axis('off')
# truth
plt.subplot(dim[0], dim[1], 4)
plt.title('Truth')
plt.imshow(image_batch_hr[0], interpolation='nearest')
plt.axis('off')
# save file
plt.tight_layout()
plt.savefig(imgSav % epoch)
plt.close('all')
# learning gif images
me_dir = os.path.join('C:\\', 'Users', 'cruze', 'Documents', 'CS767',
'finalProj--')
me_path = os.path.join(me_dir, 'lowresme.jpg')
me = Image.open(me_path)
me = np.expand_dims(me, axis=0)
me_lr = np.array(me)
me_lr_norm = normalize(me_lr)
gen_img = generator.predict(me_lr_norm)
generated_image = denormalize(gen_img)
plt.imshow(generated_image[0])
plt.savefig(imgMeSav % epoch)
plt.close('all')
def plot_importances():
model = TaskTrain().output().load()
df_train = TaskPreprocess().output().load()
df_importance = pd.Series(model.feature_importances_, index=df_train.iloc[:,:-1].columns)
import matplotlib.pyplot as plt
df_importance.sort_values(ascending=False).plot.bar()
plt.xlabel('feature')
plt.ylabel('importance')
plt.title('')
plt.tight_layout()
plt.savefig('plot.png')
def PlotSMOTEPCA(self, X, y, X_res, y_res):
# Two subplots, unpack the axes array immediately
X = np.array(X)
y = np.array(y)
pca = PCA(n_components=2)
X_vis = pca.fit_transform(X)
X_vis_res = pca.fit_transform(X_res)
f, (ax1, ax2) = plt.subplots(1, 2)
c0 = ax1.scatter(X_vis[y == 0, 0],
X_vis[y == 0, 1],
label="Class #0 Neutral comments",
alpha=0.5,
color='gray')
c1 = ax1.scatter(X_vis[y == 1, 0],
X_vis[y == 1, 1],
label="Class #1 Emotional comments",
alpha=0.5,
color='red')
ax1.set_title('Original set')
ax2.scatter(X_vis_res[y_res == 0, 0],
X_vis_res[y_res == 0, 1],
label="Class #0 Neutral comments",
alpha=0.5,
color='gray')
ax2.scatter(X_vis_res[y_res == 1, 0],
X_vis_res[y_res == 1, 1],
label="Class #1 Emotional comments",
alpha=0.5,
color='red')
ax2.set_title('SMOTE')
# make nice plotting
for ax in (ax1, ax2):
ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)
ax.get_xaxis().tick_bottom()
ax.get_yaxis().tick_left()
ax.spines['left'].set_position(('outward', 10))
ax.spines['bottom'].set_position(('outward', 10))
ax.set_xlim([-6, 8])
ax.set_ylim([-6, 6])
f.legend((c0, c1), ('Class #0 Neutral', 'Class #1 Emotional'),
loc='lower center',
ncol=2,
labelspacing=0.)
plt.tight_layout(pad=3)
plt.show()
def the_word(type):
words = ''
df = data[data[type] == 1]['comment_text']
for i in df:
i = i.lower()
words = words + i + ' '
wordcloud_vis = WordCloud(collocations=False, width=800,
height=600).generate(words)
plt.figure(figsize=(10, 8), facecolor='k')
plt.imshow(wordcloud_vis)
plt.axis("off")
plt.tight_layout(pad=0)
plt.show()
def plot_food_sample_predictions(testX,
predictions_array,
testY,
number_of_classes=3,
num_rows=10,
num_cols=4,
width=None,
height=None,
is_random=True):
"""
this method plots a sample of predictions from the test dataset and highlight correct and wrong predictions
testX: this is the test dataset
image_index: index of the image that we will plot from the test dataset
predictions_array: it is the array that contains all the predictions of the test dataset as output of model.predict(testX)
true_binary_labels: these are the true labels array expressed as INTEGER values. It does not work with hot-encoding and string labels.
"""
num_images = num_rows * num_cols
true_binary_labels = np.argmax(testY, axis=1)
if (num_images > testX.shape[0]):
raise Exception(
"num_rows*num_cols is", (num_rows * num_cols),
"must be smaller than number of images in the Test Dataset",
testX.shape[0])
if width is None:
width = 6 * num_cols
if height is None:
height = 2 * num_rows
plt.figure(figsize=(width, height))
plt.style.use(['seaborn-bright'])
image_index = -1
for i in range(num_images):
if (is_random == True):
image_index = randint(0, testX.shape[0] - 1)
else:
image_index = image_index + 1
#print(image_index)
#---------------
plt.subplot(num_rows, 2 * num_cols, 2 * i + 1)
plot_test_image(testX, image_index, predictions_array[image_index],
true_binary_labels)
#---------------
plt.subplot(num_rows, 2 * num_cols, 2 * i + 2)
plot_value_array(image_index, predictions_array[image_index],
true_binary_labels, number_of_classes)
plt.tight_layout()
plt.show()
def plot_confusion_matrix(cm, labels, title='Confusion matrix', cmap=plt.cm.Blues, save=False):
plt.figure()
plt.imshow(cm, interpolation='nearest', cmap=cmap)
plt.title(title)
plt.colorbar()
tick_marks = np.arange(len(labels))
plt.xticks(tick_marks, labels, rotation=45)
plt.yticks(tick_marks, labels)
plt.tight_layout()
plt.ylabel('True label')
plt.xlabel('Predicted label')
plt.show()
if save:
plt.savefig(save)
def PlotSMOTE(self, X, y, X_res, y_res):
# Two subplots, unpack the axes array immediately
X = np.array(X)
y = np.array(y)
for i in range(0, (len(X[0, :]) - 1)):
fig, (ax1, ax2) = plt.subplots(1, 2)
c0 = ax1.scatter(X[y == 0, i],
X[y == 0, i + 1],
label="Class #0 Neutral comments",
alpha=0.5,
color='gray')
c1 = ax1.scatter(X[y == 1, i],
X[y == 1, i + 1],
label="Class #1 Emotional comments",
alpha=0.5,
color='red')
ax1.set_title('Original set')
ax2.scatter(X_res[y_res == 0, i],
X_res[y_res == 0, i + 1],
label="Class #0 Neutral comments",
alpha=0.5,
color='gray')
ax2.scatter(X_res[y_res == 1, i],
X_res[y_res == 1, i + 1],
label="Class #1 Emotional comments",
alpha=0.5,
color='red')
ax2.set_title('SMOTE')
# make nice plotting
for ax in (ax1, ax2):
ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)
ax.get_xaxis().tick_bottom()
ax.get_yaxis().tick_left()
ax.spines['left'].set_position(('outward', 10))
ax.spines['bottom'].set_position(('outward', 10))
ax.set_xlim([-6, 8])
ax.set_ylim([-6, 6])
fig.legend((c0, c1), ('Class #0 Neutral', 'Class #1 Emotional'),
loc='lower center',
ncol=2,
labelspacing=0.)
plt.tight_layout(pad=3)
fig.savefig(self.figuresOut + '/' + str(i) + '.png')
plt.close(fig)
def plot_learning_curve(X, y, maxdepth, plt):
# create cv training and test scores for various training set sizes
train_sizes, train_scores, test_scores = learning_curve(
DecisionTreeClassifier(max_depth=maxdepth, random_state=42),
X, # feature matrix
y, # target vector
cv=5, # number of folds in cross-validation
scoring="f1", # metric
#scoring="neg_mean_squared_error", # metric
n_jobs=-1, # use all computer cores,
train_sizes=np.linspace(0.01, 1.0,
30), # 30 different sizes of the training set
)
# create means and standart deviations of training set scores
train_mean = np.mean(train_scores, axis=1)
train_std = np.std(train_scores, axis=1)
# create means and standart deviations of test set scores
test_mean = np.mean(test_scores, axis=1)
test_std = np.std(test_scores, axis=1)
# draw lines
plt.plot(train_sizes,
train_mean,
"--",
color="#111111",
label="Training score")
plt.plot(train_sizes,
test_mean,
color="#111111",
label="Cross-validation score")
# draw bands
plt.fill_between(train_sizes,
train_mean - train_std,
train_mean + train_std,
color="#DDDDDD")
plt.fill_between(train_sizes,
test_mean - test_std,
test_mean + test_std,
color="#f4d0d7")
# create plot
plt.title("Learning curve for Decision Tree")
plt.xlabel("Training set size", fontsize=18)
plt.ylabel("F1 score", fontsize=18)
plt.legend(loc="best")
plt.tight_layout()
return
def plot_confusion_matrix(cm, classes,
normalize=False, title='Confusion matrix',
cmap=plt.cm.Blues, filename='viz\\confusion_matrix.png'):
plt.figure()
plt.imshow(cm, interpolation='nearest', cmap=cmap)
plt.title(title)
plt.colorbar()
tick_marks = np.arange(len(classes))
plt.xticks(tick_marks, classes, rotation=45)
plt.yticks(tick_marks, classes)
if normalize: cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
thresh = cm.max() / 2.
for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
plt.text(j, i, cm[i, j], horizontalalignment="center", color="white" if cm[i, j] > thresh else "black")
plt.tight_layout()
plt.ylabel('True label')
plt.xlabel('Predicted label')
plt.savefig(filename)