def showErrorBars(population, sizes, numTrials):
xVals = []
sizeMeans, sizeSDs = [], []
for sampleSize in sizes:
xVals.append(sampleSize)
trialMeans = []
for t in range(numTrials):
sample = random.sample(population, sampleSize)
popMean, sampleMean, popSD, sampleSD =\
getMeansAndSDs(population, sample)
trialMeans.append(sampleMean)
sizeMeans.append(sum(trialMeans) / len(trialMeans))
sizeSDs.append(numpy.std(trialMeans))
print(sizeSDs)
numpy.errorbar(xVals,
sizeMeans,
yerr=1.96 * numpy.array(sizeSDs),
fmt='o',
label='95% Confidence Interval')
numpy.title('Mean Temperature (' + str(numTrials) + ' trials)')
numpy.xlabel('Sample Size')
numpy.ylabel('Mean')
numpy.axhline(y=popMean, color='r', label='Population Mean')
numpy.xlim(0, sizes[-1] + 10)
numpy.legend()
def simAll(drunkKinds, walkLengths, numTrials):
styleChoice = styleIterator(('m-', 'b--', 'g-.'))
for dClass in drunkKinds:
curStyle = styleChoice.nextStyle()
print('Starting simulation of', dClass.__name__)
means = simDrunk(numTrials, dClass, walkLengths)
numpy.plot(walkLengths, means, curStyle, label=dClass.__name__)
numpy.title('Mean Distance from Origin (' + str(numTrials) + ' trials)')
numpy.xlabel('Number of Steps')
numpy.ylabel('Distance from Origin')
numpy.legend(loc='best')
def resize_and_crop_image(image, target_height=224, target_width=224):
if len(image.shape) == 2:
image = np.title(image[:, :, None], 3)
elif len(image.shape) == 4:
image = image[:, :, :, 0]
image = skimage.img_as_float(image).astype(np.float32)
height, width, rgb = image.shape
if width == height:
resized_image = cv2.resize(image, (target_height, target_width))
elif height < width:
resized_image = cv2.resize(
image, (int(width * float(target_height) / height), target_width))
cropping_length = int((resized_image.shape[1] - target_height) / 2)
resized_image = resized_image[:,
cropping_length:resized_image.shape[1] -
cropping_length]
else:
resized_image = cv2.resize(
image, (target_height, int(height * float(target_width) / width)))
cropping_length = int((resized_image.shape[0] - target_width) / 2)
resized_image = resized_image[cropping_length:resized_image.shape[0] -
cropping_length, :]
return cv2.resize(resized_image, (target_height, target_width))
def traceWalk(fieldKinds, numSteps):
styleChoice = styleIterator(('b+', 'r^', 'ko'))
for fClass in fieldKinds:
d = UsualDrunk()
f = fClass()
f.addDrunk(d, Location(0, 0))
locs = []
for s in range(numSteps):
f.moveDrunk(d)
locs.append(f.getLoc(d))
xVals, yVals = [], []
for loc in locs:
xVals.append(loc.getX())
yVals.append(loc.getY())
curStyle = styleChoice.nextStyle()
numpy.plot(xVals, yVals, curStyle, label=fClass.__name__)
numpy.title('Spots Visited on Walk (' + str(numSteps) + ' steps)')
numpy.xlabel('Steps East/West of Origin')
numpy.ylabel('Steps North/South of Origin')
numpy.legend(loc='best')
def plotLocs(drunkKinds, numSteps, numTrials):
styleChoice = styleIterator(('k+', 'r^', 'mo'))
for dClass in drunkKinds:
locs = getFinalLocs(numSteps, numTrials, dClass)
xVals, yVals = [], []
for loc in locs:
xVals.append(loc.getX())
yVals.append(loc.getY())
xVals = numpy.array(xVals)
yVals = numpy.array(yVals)
meanX = sum(abs(xVals)) / len(xVals)
meanY = sum(abs(yVals)) / len(yVals)
curStyle = styleChoice.nextStyle()
numpy.plot(xVals, yVals, curStyle,
label = dClass.__name__ +\
' mean abs dist = <'
+ str(meanX) + ', ' + str(meanY) + '>')
numpy.title('Location at End of Walks (' + str(numSteps) + ' steps)')
numpy.ylim(-1000, 1000)
numpy.xlim(-1000, 1000)
numpy.xlabel('Steps East/West of Origin')
numpy.ylabel('Steps North/South of Origin')
numpy.legend(loc='lower center')
def classfy0(inX, dataSet, labels, k):
dataSetSize = dataSet.shape[0]
diffMat = np.title(inX, (dataSetSize, 1)) - dataSet
sqDiffMat = diffMat**2
sqDistances = sqDiffMat.sum(axis=1)
distances = sqDistances**0.5
sorteDistIndicts = distances.argsort()
classCount = {}
for i in range(k):
voteILabel = label[sorteDistIndicts[i]]
classCount[voteILabel] = classCount.get(voteILabel, 0) + 1
sortedClassCount = sorted(classCount.iteritems(),
key=operator.itemgetter,
reverse=True)
return sortedClassCount[0][0]
def pngs_equal(a, b):
if files_equal(a, b):
print a, 'and', b, 'are perfectly equal'
return True
im_a = numpy.imread(a) * 255.0
im_b = numpy.imread(b) * 255.0
if im_a.shape != im_b.shape:
print a, 'and', b, 'have different size:', im_a.shape, im_b.shape
return False
diff = im_b - im_a
alpha = im_a.shape[-1] == 4
if alpha:
diff_alpha = diff[:, :, 3]
equal = True
print a, 'and', b, 'are different, analyzing whether it is just the undefined colors...'
print 'Average difference (255=white): (R, G, B, A)'
print numpy.mean(numpy.mean(diff, 0), 0)
print 'Average difference with premultiplied alpha (255=white): (R, G, B, A)'
diff = diff[:, :, 0:3]
if alpha:
diff *= numpy.imread(a)[:, :, 3:4]
res = numpy.mean(numpy.mean(diff, 0), 0)
print res
if numpy.mean(res) > 0.01:
# dithering should make this value nearly zero...
equal = False
print 'Maximum abs difference with premultiplied alpha (255=white): (R, G, B, A)'
res = numpy.amax(numpy.amax(abs(diff), 0), 0)
print res
if max(abs(res)) > 1.1:
# this error will be visible
# - smaller errors are hidden by the weak alpha
# (but we should pay attention not to accumulate such errors at each load/save cycle...)
equal = False
if not equal:
print 'Not equal enough!'
if alpha:
numpy.figure(1)
numpy.title('Alpha')
numpy.imshow(im_b[:, :, 3], interpolation='nearest')
numpy.colorbar()
numpy.figure(2)
numpy.title('Green Error (multiplied with alpha)')
numpy.imshow(diff[:, :, 1], interpolation='nearest')
numpy.colorbar()
if alpha:
numpy.figure(3)
numpy.title('Alpha Error')
numpy.imshow(diff_alpha, interpolation='nearest')
numpy.colorbar()
numpy.show()
return equal
def pngs_equal(a, b):
if files_equal(a, b):
print a, 'and', b, 'are perfectly equal'
return True
im_a = numpy.imread(a)*255.0
im_b = numpy.imread(b)*255.0
if im_a.shape != im_b.shape:
print a, 'and', b, 'have different size:', im_a.shape, im_b.shape
return False
diff = im_b - im_a
alpha = im_a.shape[-1] == 4
if alpha:
diff_alpha = diff[:, :, 3]
equal = True
print a, 'and', b, 'are different, analyzing whether it is just the undefined colors...'
print 'Average difference (255=white): (R, G, B, A)'
print numpy.mean(numpy.mean(diff, 0), 0)
print 'Average difference with premultiplied alpha (255=white): (R, G, B, A)'
diff = diff[:, :, 0:3]
if alpha:
diff *= numpy.imread(a)[:, :, 3:4]
res = numpy.mean(numpy.mean(diff, 0), 0)
print res
if numpy.mean(res) > 0.01:
# dithering should make this value nearly zero...
equal = False
print 'Maximum abs difference with premultiplied alpha (255=white): (R, G, B, A)'
res = numpy.amax(numpy.amax(abs(diff), 0), 0)
print res
if max(abs(res)) > 1.1:
# this error will be visible
# - smaller errors are hidden by the weak alpha
# (but we should pay attention not to accumulate such errors at each load/save cycle...)
equal = False
if not equal:
print 'Not equal enough!'
if alpha:
numpy.figure(1)
numpy.title('Alpha')
numpy.imshow(im_b[:, :, 3], interpolation='nearest')
numpy.colorbar()
numpy.figure(2)
numpy.title('Green Error (multiplied with alpha)')
numpy.imshow(diff[:, :, 1], interpolation='nearest')
numpy.colorbar()
if alpha:
numpy.figure(3)
numpy.title('Alpha Error')
numpy.imshow(diff_alpha, interpolation='nearest')
numpy.colorbar()
numpy.show()
return equal
def _repr_html_(self):
from identity import PartitionName
active_parts = set()
# Find out which of the name parts are being used, particularly
# time, space, grain
for p in self.all:
active_parts |= set(p.name.partital_dict.keys())
cols = ['Id', 'Vid', 'Name', 'VName']
for np, _, _ in PartitionName._name_parts:
if np in active_parts:
cols.append(np.title())
rows = [
"<tr>" + ''.join(['<th>{}</th>'.format(c) for c in cols]) + "</tr>"
]
for p in self.all:
cols = []
d = p.name.partital_dict
cols.append(p.identity.id_)
cols.append(p.identity.vid)
cols.append(p.identity.sname)
cols.append(p.identity.vname)
for np, _, _ in PartitionName._name_parts:
if np not in active_parts:
continue
cols.append(d[np] if np in d else '')
rows.append("<tr>" +
''.join(['<td>{}</td>'.format(c)
for c in cols]) + "</tr>")
return "<table>\n" + "\n".join(rows) + "\n</table>"
def createDataSet():
#四组二维特征
group = np.array([[1, 101], [5, 89], [108, 5], [115, 8]])
#四组特征的标签
labels = ['爱情片', '爱情片', '动作片', '动作片']
return group, labels
dataSize = dataSet.shape[0]
diffMat = np.title(inX, (dataSetSize, 1)) - dataSet()
sqDiffMat = diffMat**2
#sum（）所有元素相加，sum（0）列相加，sum（1）行相加
sqDistances = sqDiffMat.sum(axis=1)
#开方，计算出距离
distances = sqDistances**0.5
sortedDistIndices = distances.argsort()
classCount = {}
for i in range(k):
#取出前k个元素的类别range（5）等价于range（0,5） end：计数到end结束，但不包括end
voteIlabel = labels[sortedDistIndices[i]]
#sortedDistIndices[i] 索引值[2 3 1 0]
classCount[voteIlabel] = classCount.get(voteIlabel, 0) + 1
sortedClassCount = sorted(classCount.items(),
key=operator.itemgetter(1),
reverse=True)
return sortedClassCount[0][0] #取第一个
def env_input_recover(a): \
return np.title((a + 1.0) * 128.0, (3, 1, 1))
else:
def sawttooth_wave(size, T):
t = np.linspace(-1, 1, size)
y = np.title(t, T)
x = np.linspace(0, 2 * np.pi * T, size * T, endpoint=False)
return x, y
def read_images(src):
from scipy import misc
filepath = src
im = misc.imread(filepath)
import scipy.misc as mc
return mc.imresize(im, (ROWS, COLS))
files = []
y_all = []
for fish in SIGNATURE_CLASSES:
fish_files = get_images(fish)
files.extend(fish_files)
y_fish = np.title(fish, len(fish_title))
y_all.extend(y_fish)
print("{0} photo of {1}".format(len(fish_files),fish))
y_all = np.array(y_all)
print(len(files))
print(len(y_all))
x_all = np.ndarray(len(files), ROWS, COLS, CHANNELS), dtype=np.uint8)
for i, im in enumerate(files):
x_all[i] = read_image(TRAIN_DIR+im)
if x%1000 == 0:
print('processed {} of {}'.format(i, len(files)))
np.vsplit(
matriz, num_divisiones
) # Separar una matriz en matrices respecto a la posición de la dimensión 2 en el numero de divisiónes especificado (Si las divisiones no son exactas da error)
np.vsplit(
matriz, [pos1, pos2, pos3]
) # Separar una matriz en matrices respecto a la posición de la dimensión 2 en las posiciones de particion especificadas
np.dsplit(
matriz, num_divisiones
) # Separar una matriz en matrices respecto a la posición de la dimensión 3 en el numero de divisiónes especificado (Si las divisiones no son exactas da error)
np.dsplit(
matriz, [pos1, pos2, pos3]
) # Separar una matriz en matrices respecto a la posición de la dimensión 3 en las posiciones de particion especificadas
# Matrices y vectores por repetición (Mosaico)
np.title(
matriz, (ver_rep, hor_rep)
) # Repetir matriz en los ejes vertical y horizontal las veces especificadas
np.repeat(
matriz, [rep1, rep2, repN], dimensión
) # Repetir elementos de una matriz las veces especificadas respecto a la posición de la dimensión especificada
np.repeat(
matriz, repeticiones, dimensión
) # Contraer matriz a una dimensión repitiendo los elementos las veces especificadas
# Agregar y eliminar elementos
np.delete(
matriz, [pos1, pos2, posN], dimensión
) # Borrar filas o columnas especificadas respecto a la posición de la dimensión especificada de la matriz
np.delete(
matriz, [pos1, pos2, posN]
) # Contraer matriz a una dimensión borrando los elementos especificados de la matriz
def gaussian(x, mu, sigma):
factor1 = (1.0 / (sigma * ((2 * numpy.pi)**0.5)))
factor2 = numpy.e**-(((x - mu)**2) / (2 * sigma**2))
return factor1 * factor2
xVals, yVals = [], []
mu, sigma = 0, 2.5
x = -10
while x <= 10:
xVals.append(x)
yVals.append(gaussian(x, mu, sigma))
x += 0.05
numpy.plot(xVals, yVals)
numpy.title('Normal Distribution, mu = ' + str(mu)\
+ ', sigma = ' + str(sigma))
numpy.show()
# import scipy.integrate
# def checkEmpirical(numTrials):
# for t in range(numTrials):
# mu = random.randint(-10, 10)
# sigma = random.randint(1, 10)
# print('For mu =', mu, 'and sigma =', sigma)
# for numStd in (1, 1.96, 3):
# area = scipy.integrate.quad(gaussian,
# mu-numStd*sigma,
# mu+numStd*sigma,
# (mu, sigma))[0]
# print(' Fraction within', numStd,
# #plt.title('Accuracy for different Momentum for CNN')
# plt.ylabel('Error')
# #plt.ylabel('Accuracy')
# plt.xlabel('Momentum')
# plt.show()
# #For Batch Size
# plt.xticks([1 ,2, 3, 4, 5],['1','10','100','500','1000'])
# plt.plot([1 ,2, 3, 4, 5],[1.88155 ,1.25564 ,0.43665 ,1.11511 ,3.02981 ], 'bo', label='Training')
# plt.plot([1 ,2, 3, 4, 5],[1.88067 ,1.71583 ,0.63323 ,1.07845 ,2.84858 ], 'go', label='Validation')
# plt.axis([0, 5.5, 0.0, 3.2])
# plt.legend(loc=0)
# plt.title('Cross-Entropy for different batch sizes for CNN')
# #plt.title('Accuracy for different batch sizes for CNN')
# plt.ylabel('Error')
# #plt.ylabel('Accuracy')
# plt.xlabel('Batch Size')
# plt.show()
#For 3.3
plt.xticks([1, 2, 3], ['[2 16]', '[15 16]', '[30 16]'])
plt.plot([1, 2, 3], [0.28542, 0.74748, 0.28542], 'bo', label='Training')
plt.plot([1, 2, 3], [0.27924, 0.70883, 0.27924], 'go', label='Validation')
plt.axis([0.5, 3.5, 0, 0.8])
plt.legend(loc=0)
#plt.title('Cross-Entropy for different number of filters in the first layer of CNN')
plt.title('Accuracy for different number of filters in the first layer of CNN')
#plt.ylabel('Error')
plt.ylabel('Accuracy')
plt.xlabel('Number of Units')
plt.show()
def makeHist(data, title, xlabel, ylabel, bins=20):
numpy.hist(data, bins=bins)
numpy.title(title)
numpy.xlabel(xlabel)
numpy.ylabel(ylabel)
# mini-batch training
epochs = training.train(model, X_train, y_train, X_test, y_test, batch_size, num_epochs, acc_threshold)
# mini-batch testing
acc, bayes_acc = training.test(model, X_test, y_test, batch_size)
df.set_value(experiment_name, 'test_acc', acc)
df.set_value(experiment_name, 'bayes_test_acc', bayes_acc)
# uncertainty prediction
test_mean_std_bayesian = {x:[] for x in range(10)}
test_mean_std_deterministic = {x:[] for x in range(10)}
test_entropy_bayesian = {x:[] for x in range(10)}
test_entropy_deterministic = {x:[] for x in range(10)}
for i in range(len(X_test_all)):
bayesian_probs = model.probabilities(np.title(X_test_all[i], batch_size).reshape([-1]+n_in))
bayesian_entropy = model.entropy_bayesian(np.title(X_terst_all[i], batch_size).reshape([-1]+n_in))
predictve_mean = np.mean(bayesian_probs, axis=0)
predictve_std = np.std(bayesian_probs, axis=0)
# plotting
if plot:
for k in sorted(test_mean_std_bayesian.keys()):
sns.plt.figure()
sns.plt.hist(test_entropy_bayesian[k], label="Bayesian Entropy v1 for "+ str(k))
sns.plt.legend()
sns.plt.show()
# anomaly_detection
def anomaly_detection(anomaly_score_dict, name, df):
X=[]
def plotDiffs(sampleSizes, diffs, title, label, color='b'):
numpy.plot(sampleSizes, diffs, label=label, color=color)
numpy.xlabel('Sample Size')
numpy.ylabel('% Difference in SD')
numpy.title(title)
numpy.legend()
x_test = x_test.astype('float32') / 255.
x_test = x_test.reshape(x_test.shape + (1, ))
vae.fit(x=x_train,
y=None,
shuffle=True,
epochs=10,
batch_size=batch_size,
validation_data=(x_test, None))
import matplotlib.pyplot as plt
from scipy.stats import norm
n = 15
digit_size = 28
figure = np.zeros((digit_size * n, digit_size * n))
grid_x = norm.ppf(np.linspace(0.05, 0.95, n))
grid_y = norm.ppf(np.linspace(0.05, 0.95, n))
for i, yi in enumerate(grid_x):
for j, xi in enumerate(grid_y):
z_sample = np.array([[xi, yi]])
z_sample = np.title(z_sample, batch_size).reshape(batch_size, 2)
x_decoded = decoder.predict(z_sample, batch_size=batch_size)
digit = x_decoded[0].reshape(digit_size, digit_size)
figure[i * digit_size:(i + 1) * digit_size,
j * digit_size:(j + 1) * digit_size] = digit
plt.figure(figsize=(10, 10))
plt.imshow(figure, cmap='Greys_r')
plt.show()