n_train = 60000
X_train, y_train = imagelib.train_img[:n_train,:], imagelib.train_lab[:n_train]
n_test = 10000
X_test, y_test = imagelib.test_img[:n_test,:], imagelib.test_lab[:n_test]
# create a neural network object
myNN = NN2()
# use the numerical gradient check to verify the backpropagation
myNN.CheckGradient(Lambda=1)
# use train_set to train the neural network with backpropagation
start = clock()
theta1, theta2 = myNN.Train(X_train,y_train,max_iter=100, alpha=0.3)
finish = clock()
print "The training costs %d H, %d Min, %f S" %(hms(finish-start))
# use the test_set to check the accuracy of the neural network
myNN.Test(X_test,y_test)
# and display 10 examples from test_set
for i in range(10):
random.seed()
myNN.Display(X_test[random.randrange(0,n_test)])
# save weights matrics
with open('theta1.dat','wb') as f:
f.write(theta1)
with open('theta2.dat','wb') as f:
f.write(theta2)
except:
usage()
groups, pairs = findGroupsOfFiles( ibDir )
if len(pairs) == 0:
print 'no valid sets of ib error and stats files found. need to gather some stats first?'
sys.exit(1)
if listOnly:
cnt = 0
for i in range(len(pairs),0,-1):
p0, p1 = pairs[i-1]
grp0 = groups[p0]
grp1 = groups[p1]
interval = grp1['perfstats'][0] - grp0['ibclearerrors'][0]
print cnt, 'interval', hms(interval), grp0['perfstats'][1].split('.')[0], 'to', grp1['perfstats'][1].split('.')[0]
cnt += 1
sys.exit(0)
if allHosts:
print 'warning: rebooted/down hosts are included'
else:
print 'only displaying non-rebooted/down hosts'
if allErrs:
print 'showing all errors, not just', normalErrors
else:
print 'warning: only displaying', normalErrors
if lowToo:
print 'showing ports with low error counts'
from hms import hms
import time, sys
ibDir = '/root/ib'
suffix = 'ibclearerrors'
f, fTime = findMostRecentFile( ibDir, suffix )
#print 'using', f, 'for errors, time', fTime, 'now', time.time(), 'diff', time.time() - fTime, 'hrs', (time.time() - fTime)/3600.0
uptime, down = uptimes()
if uptime == None: # failed
sys.exit(1)
#print 'len(uptime)', len(uptime) #, 'uptime', uptime # uptimes by hostname
#print 'len(down)', len(down), 'down', down
ignore = filterHosts( uptime, fTime )
ignore.sort()
#print 'recently rebooted - ignore hosts', ignore, 'len', len(ignore)
print '# nodes rebooted in last', hms(time.time() - fTime)
for i in ignore:
if i in down:
continue
print i
# no idea about down nodes, so assume they're evil
print '# down'
for i in down:
print i