def draw_computational_graph(var_or_func, save_path=None):
'''
Use d3viz to provide interactive html graph for theano
For tensorflow, use: tensorboard --logdir [save_path]
'''
from matplotlib import pyplot as plt
import matplotlib.image as mpimg
if save_path is None:
save_path = os.path.join(utils.get_tmp_dir(), 'tmp.html')
if config.backend() == 'theano':
import theano.d3viz as d3v
d3v.d3viz(var_or_func, save_path)
# theano.printing.pydotprint(
# var_or_func,
# outfile = save_path)
return save_path
elif config.backend() == 'tensorflow':
import tensorflow as tf
sess = tensor.get_session()
writer = tf.python.training.summary_io.SummaryWriter(
save_path, sess.graph_def)
tensor.eval(var_or_func)
writer.close()
return save_path
def check(self, f, reference=None, verbose=False):
tmp_dir = tempfile.mkdtemp()
html_file = pt.join(tmp_dir, 'index.html')
if verbose:
print(html_file)
d3v.d3viz(f, html_file)
assert pt.getsize(html_file) > 0
if reference:
assert filecmp.cmp(html_file, reference)
def train(self, Xtrain, Ytrain, batchSize):
X = T.matrix("X")
Y = T.matrix("Y")
index = T.iscalar("index")
cost = self.binaryCrossEntropyCostFunction(self.forwardProp(X, True), Y)
inputs = [index]
if self.optAlg.lower() == "irpropplus":
updates = iRpropPlus(cost, self.params, self.c)
elif self.optAlg.lower() == "vsgdfd":
slowStart = T.fscalar("slowStart")
updates = vSGDfd(cost, self.params, batchSize, self.layers[0].theta.shape.eval()[1], self.c, slowStart)
inputs.append(slowStart)
elif self.optAlg.lower() == "rmsprop":
updates = RMSProp(cost, self.params, self.c, self.learningRate, self.rho, self.epsilon, self.momentum)
elif self.optAlg.lower() == "adadelta":
updates = adaDelta(cost, self.params, self.c, self.rho, self.epsilon)
else:
updates = RMSProp(cost, self.params, self.c, self.learningRate, self.rho, self.epsilon, self.momentum)
train = theano.function(
inputs=inputs,
outputs=theano.Out(cost, borrow=True),
updates=updates + self.extraParams,
name="train",
givens={
X: Xtrain[index * batchSize : (index + 1) * batchSize, :],
Y: Ytrain[index * batchSize : (index + 1) * batchSize, :],
},
allow_input_downcast=True
# ,mode = theano.compile.MonitorMode(pre_func = inspect_inputs, post_func = inspect_outputs)
# ,mode = NanGuardMode(nan_is_error = True, inf_is_error = True, big_is_error = True)
# ,mode = theano.compile.MonitorMode(post_func = detect_nan)
# ,mode = theano.compile.MonitorMode(post_func = detect_inf)
,
profile=True,
)
# print theano.printing.debugprint(train)
# theano.printing.pydotprint(train, outfile="symbolic_graph_of_nn_opt.svg", format='svg', var_with_name_simple=True)
d3v.d3viz(train, "d3viz/mlp.html")
return train
def build_model(self):
self.w1 = theano.shared(np.random.uniform(-1,
1,
size=(3072,
self.hidden_layer)),
name="w1")
self.b1 = theano.shared(np.zeros((self.hidden_layer, )), name="b1")
self.w2 = theano.shared(np.random.uniform(-1,
1,
size=(self.hidden_layer,
10)),
name="w2")
self.b2 = theano.shared(np.zeros((10, )), name="b2")
self.p = T.dmatrix("input")
self.target = T.dmatrix("target")
self.net1 = T.dot(self.p, self.w1) + self.b1
#print theano.printing.debugprint(self.net1)
self.a1 = self.activation(self.net1)
self.net2 = T.dot(self.a1, self.w2) + self.b2
self.a2 = T.nnet.softmax(self.net2)
self.output = T.argmax(self.a2, axis=1)
self.target_one = np.argmax(self.target, axis=1)
self.cost = T.mean(
T.nnet.categorical_crossentropy(
self.a2, self.target)) + self.Lambda * (
self.w1**2).sum() + self.Lambda * (self.w2**2).sum()
self.error = T.mean(T.neq(self.output, self.target_one))
self.dw1, self.db1, self.dw2, self.db2 = T.grad(
self.cost, wrt=[self.w1, self.b1, self.w2, self.b2])
self.train = theano.function(
[self.p, self.target],
self.error,
updates=[[self.w1, self.w1 - self.alpha * self.dw1],
[self.b1, self.b1 - self.alpha * self.db1],
[self.w2, self.w2 - self.alpha * self.dw2],
[self.b2, self.b2 - self.alpha * self.db2]])
self.predict = theano.function(inputs=[self.p],
outputs=self.output,
on_unused_input="ignore")
d3viz(self.cost, "sample.html")
def write_profile_files(gp):
d3viz.d3viz(gp.dnlml, 'dnlml.html')
d3viz.d3viz(gp.predict_fn, 'predict.html')
import numpy as np
from pprint import pprint
ninputs = 1000
nfeatures = 100
noutputs = 10
nhiddens = 50
rng = np.random.RandomState(0)
x = T.dmatrix('x')
wh = th.shared(rng.normal(0, 1, (nfeatures, nhiddens)), borrow=True)
bh = th.shared(np.zeros(nhiddens), borrow=True)
h = T.nnet.sigmoid(T.dot(x, wh) + bh)
wy = th.shared(rng.normal(0, 1, (nhiddens, noutputs)))
by = th.shared(np.zeros(noutputs), borrow=True)
y = T.nnet.softmax(T.dot(h, wy) + by)
predict = th.function([x], y)
pprint(predict)
from theano.printing import pydotprint
import os
if not os.path.exists('examples'):
os.makedirs('examples')
pydotprint(predict, 'examples/mlp.png')
import theano.d3viz as d3v
d3v.d3viz(predict, 'examples/mlp.html')
def dumpToHTML(self, filename):
"""saves the function graph into an interactive html file, requires pydot and potentially an internet connection"""
import theano.d3viz as d3v
d3v.d3viz(self.theano_fct, filename)
def dumpToHTML(self, filename) : """saves the function graph into an interactive html file, requires pydot and potentially an internet connection""" import theano.d3viz as d3v d3v.d3viz(self.theano_fct, filename)
def dumpToHTML(self, filename):
"""dumps the graph into a interactive html file"""
import theano.d3viz as d3v
d3v.d3viz(self.theano_fct, filename)
def plot(thing):
theano.printing.pydotprint(thing,
outfile="think.png",
var_with_name_simple=True)
import theano.d3viz as d3v
d3v.d3viz(thing, 'think.html')
hidden = T.nnet.sigmoid(T.dot(x, weightH) + biasH)
weightY = theano.shared(rand_num_gen.normal(0, 1, (nhiddens, noutputs)))
biasY = theano.shared(np.zeros(noutputs), borrow=True)
y = T.nnet.softmax(T.dot(hidden, weightY) + biasY)
predict = theano.function(inputs=[x], outputs=y) #outputs the probability of 10 classes
import pydot
import theano.d3viz as d3v
import os
if not os.path.exists('/Users/dannyg/Desktop/Projects/TheanoCreations/Outputs'):
os.makedirs('/Users/dannyg/Desktop/Projects/TheanoCreations/Outputs')
d3v.d3viz(predict, '/Users/dannyg/Desktop/Projects/TheanoCreations/Outputs/graph.html')
import theano as th
import theano.tensor as T
import numpy as np
ninputs = 1000
nfeatures = 100
noutputs = 10
nhiddens = 50
rng = np.random.RandomState(0)
x = T.dmatrix("x")
wh = th.shared(rng.normal(0, 1, (nfeatures, nhiddens)), borrow=True)
bh = th.shared(np.zeros(nhiddens), borrow=True)
h = T.nnet.sigmoid(T.dot(x, wh) + bh)
wy = th.shared(rng.normal(0, 1, (nhiddens, noutputs)))
by = th.shared(np.zeros(noutputs), borrow=True)
y = T.nnet.softmax(T.dot(h, wy) + by)
predict = th.function([x], y)
import theano.d3viz as d3v
d3v.d3viz(predict, "examples/mlp.html")
def trainWithEarlyStopping(model, numEpochs, patience, validationFrequency, data, variables, batchSize, errorDict = None, AUCDict = None, logfile = None, name = "", debug = False, verbose = True, save = True, subfolder = "", visualize = False):
Xtrain, Ytrain, Xval, Yval, mu, sig, AUCindices = data
numTrainBatches = Xtrain.get_value(borrow=True).shape[0] / batchSize
numValidBatches = Xval.get_value(borrow=True).shape[0] / batchSize
patienceIncrease = 1.75
improvementThreshold = 0.999
stoppingCriteria = 10
bestValError = np.inf
bestvalIter = 0
bestAUCval = 0
skipAUCEval = 0 #800
endIteration = False
t0 = time.time()
trainFunction = model.train(Xtrain, Ytrain, batchSize)
validationErrorFunction = model.validationError(Xval, Yval)
import theano.d3viz as d3v
print "Building graph..."
d3v.d3viz(trainFunction, 'd3viz/mlp.html')
if verbose: print "Compilation time:", (time.time() - t0) / 60.0, "minutes"
if logfile: logfile.write("Compilation time: " + str((time.time() - t0) / 60.0) + " minutes" + "\n")
generalizationLoss = 0
trainingProgress = 0
trainingStrip = deque([0.0 for i in xrange(max(validationFrequency, 20))])
timestamp = time.strftime("%d%H%M%S")
folder = ""
printFrequency = 10
iter = 0
if visualize: import matplotlib.pyplot as plt
if errorDict is not None:
if visualize:
fig = plt.figure(1)
fig.canvas.set_window_title('Error/Epochs')
plt.title('Error/Epochs ' + model.optAlg)
plt.ylabel('Error')
plt.xlabel('Epochs')
plt.axis([0, numEpochs, 0, 50000])
errorDict["epoch"] = []
errorDict["trainError"] = []
errorDict["valError"] = []
if AUCDict is not None:
if visualize:
fig = plt.figure(2)
fig.canvas.set_window_title('AUC/Epochs')
plt.title('AUC/Epochs ' + model.optAlg)
plt.ylabel('AUC')
plt.xlabel('Epochs')
plt.axis([0, numEpochs, 0.5, 1])
AUCDict["epoch"] = []
AUCDict["trainAUC"] = []
AUCDict["valAUC"] = []
if visualize:
plt.ion()
plt.show()
if save:
folder = 'saves\\' + subfolder + name + timestamp + '\\'
if logfile: logfile.write("Folder to save in " + folder + "\n")
os.makedirs(folder)
if model.optAlg == "vSGDfd":
if verbose: print "Slow starting vSGDfd..."
for i in xrange(int(np.clip(0.001 * Xtrain.get_value(borrow=True).shape[0] / batchSize, 1, numTrainBatches))):
cost = trainFunction(i, 1)
if verbose: print "Done with slow start"
t0 = time.time()
if verbose: print "Training..."
if logfile: logfile.write("Training..." + "\n")
for epoch in xrange(numEpochs):
for minibatchIndex in xrange(numTrainBatches):
if model.optAlg == "vSGDfd":
cost = trainFunction(minibatchIndex, 0)
else:
cost = trainFunction(minibatchIndex)
trainingStrip.pop()
trainingStrip.appendleft(cost)
iter = epoch * numTrainBatches + minibatchIndex
if (iter + 1) % validationFrequency == 0:
#valError = np.mean([validationErrorFunction(i) for i in xrange(numValidBatches)])
valError = validationErrorFunction()
if logfile: logfile.write("Iteration: " + str(iter + 1) + ", epoch: " + str(epoch) + ", minibatchIndex: " + str(minibatchIndex) +", validation frequency: " + str(validationFrequency) + ", Error: " + str(cost) + ", validation error: " + str(valError) + "\n")
if valError < bestValError:
if verbose: print "Iteration:", iter + 1, ", epoch", epoch, ", minibatchIndex", minibatchIndex, ", validation frequency:", validationFrequency, ", Error:", cost, ", validation error:", valError
if valError < bestValError * improvementThreshold:
newPatience = max(patience, int(iter * patienceIncrease))
if newPatience > patience:
patience = newPatience
if verbose: print "valError has decreased significantly so the patience is updated to:", patience
if logfile: logfile.write("valError has decreased significantly so the patience is updated to: " + str(patience) + "\n")
if verbose: print "Best valError updated to:" , valError
if logfile: logfile.write("Best valError updated to: " + str(valError) + "\n")
AUCval = 0
if iter > skipAUCEval: AUCval = AUC(model, Xval, Yval, AUCindices["val"])
if verbose: print "current AUCval:", AUCval, ", bestAUCval:", bestAUCval
if logfile: logfile.write("current AUCval: " + str(AUCval) + ", bestAUCval: " + str(bestAUCval) + "\n")
if save and AUCval > bestAUCval:
if verbose: print "AUC has improved. Saving..."
if logfile: logfile.write("AUC has improved. Saving..." + "\n")
model.saveModel(name = "", folder = folder, verbose = False)
bestAUCval = AUCval
bestvalIter = iter
bestValError = valError
# to limit the amount of printing to console
elif (iter + 1) % (printFrequency * validationFrequency) == 0:
if verbose: print "Iteration:", iter + 1, ", epoch", epoch, ", minibatchIndex", minibatchIndex, ", validation frequency:", validationFrequency, ", Error:", cost, ", validation error:", valError
if iter - bestvalIter > validationFrequency * 20 and validationFrequency != 1:
validationFrequency -= 1
bestvalIter = iter
if verbose: print "Validation error has not improved for 20 validations so validation frequency is updated to", validationFrequency
if logfile: logfile.write("Validation error has not improved for 20 validations so validation frequency is updated to " + str(validationFrequency) + "\n")
if iter >= 20 and iter >= validationFrequency: # ensure the strip is filled
# from "Early Stopping - but when?" by Lutz Prechelt
generalizationLoss = 100 * (valError / (bestValError + 1e-9) - 1)
trainingProgress = 1000 * (sum(trainingStrip) / (len(trainingStrip) * min(trainingStrip) + 1e-9) - 1)
if generalizationLoss / (trainingProgress + 1e-9) > stoppingCriteria:
if verbose: print "PQ stopping criteria has been triggered. GL/P:", generalizationLoss / (trainingProgress + 1e-9)
if logfile: logfile.write("PQ stopping criteria has been triggered. GL/P: " + str(generalizationLoss / (trainingProgress + 1e-9)) + "\n")
endIteration = True
break
if patience <= iter:
if verbose: print "Patience exceeded"
if logfile: logfile.write("Patience exceeded" + "\n")
endIteration = True
break
if errorDict is not None:
errorDict["epoch"].append(epoch)
errorDict["trainError"].append(cost)
errorDict["valError"].append(valError)
if visualize:
plt.figure(1)
plt.plot(errorDict["epoch"], errorDict["trainError"], 'r-', errorDict["epoch"], errorDict["valError"], 'b-')
plt.axis([0, max(1, errorDict["epoch"][-1]), min(errorDict["trainError"] + errorDict["valError"]), max(errorDict["trainError"] + errorDict["valError"])])
if AUCDict is not None:
AUCtrain = AUC(model, Xtrain, Ytrain, AUCindices["train"])
AUCval = AUC(model, Xval, Yval, AUCindices["val"])
AUCDict["epoch"].append(epoch)
AUCDict["trainAUC"].append(AUCtrain)
AUCDict["valAUC"].append(AUCval)
if visualize:
plt.figure(2)
plt.plot(AUCDict["epoch"], AUCDict["trainAUC"], 'r-', AUCDict["epoch"], AUCDict["valAUC"], 'b-')
plt.axis([0, max(1, AUCDict["epoch"][-1]), min(AUCDict["trainAUC"] + AUCDict["valAUC"]), max(AUCDict["trainAUC"] + AUCDict["valAUC"])])
if visualize:
plt.draw()
plt.pause(0.0001)
if endIteration: break
if verbose: print "Execution time:", (time.time() - t0) / 60.0, " minutes"
if logfile: logfile.write("Execution time: " + str((time.time() - t0) / 60.0) + " minutes" + "\n")
if save: model.loadModel(name = "", folder = folder, verbose = False)
ks = agreement(model, variables, mu, sig)
cvm = correlation(model, variables, mu, sig)
AUCval = AUC(model, Xval, Yval, AUCindices["val"])
if verbose:
print "Agreement: " + str(ks) + " under 0.09 " + str(ks < 0.09) + ", Correlation: " + str(cvm) + " under 0.002 " + str(cvm < 0.002)
print "AUCval: " + str(AUCval)
if logfile:
logfile.write("Agreement: " + str(ks) + " under 0.09 " + str(ks < 0.09) + ", Correlation: " + str(cvm) + " under 0.002 " + str(cvm < 0.002) + "\n")
logfile.write("AUCval: " + str(AUCval) + "\n")
return -AUCval + ((ks > 0.09) + (cvm > 0.002))