def checkGradient_MiniBatch(dictionary, trees):
model = RNTNModel(dictionary)
theta_init = model.getTheta()
# compute analytical gradient
costObj = ComputeCostAndGradMiniBatch()
cost, grad = costObj.compute(theta_init, dictionary, trees)
eps = 1E-4
numgrad = np.zeros(grad.shape)
# compute numerical gradient
for i in range(model.num_parameters):
if i % 10 == 0:
print '%d/%d' % (i, model.num_parameters)
indicator = np.zeros(model.num_parameters)
indicator[i] = 1
theta_plus = theta_init + eps*indicator
cost_plus, grad_plus = costObj.compute(theta_plus, dictionary, trees)
theta_minus = theta_init - eps*indicator
cost_minus, grad_minus = costObj.compute(theta_minus, dictionary, trees)
numgrad[i] = (cost_plus - cost_minus)/(2*eps)
print 'analytical gradient: ', grad
print 'numerical gradient: ', numgrad
normdiff = np.linalg.norm(numgrad - grad) / np.linalg.norm(numgrad + grad)
print 'Norm difference: ', normdiff
return normdiff
def checkGradient_MiniBatch(dictionary, trees):
model = RNTNModel(dictionary)
theta_init = model.getTheta()
# compute analytical gradient
costObj = ComputeCostAndGradMiniBatch()
cost, grad = costObj.compute(theta_init, dictionary, trees)
eps = 1E-4
numgrad = np.zeros(grad.shape)
# compute numerical gradient
for i in range(model.num_parameters):
if i % 10 == 0:
print '%d/%d' % (i, model.num_parameters)
indicator = np.zeros(model.num_parameters)
indicator[i] = 1
theta_plus = theta_init + eps * indicator
cost_plus, grad_plus = costObj.compute(theta_plus, dictionary, trees)
theta_minus = theta_init - eps * indicator
cost_minus, grad_minus = costObj.compute(theta_minus, dictionary,
trees)
numgrad[i] = (cost_plus - cost_minus) / (2 * eps)
print 'analytical gradient: ', grad
print 'numerical gradient: ', numgrad
normdiff = np.linalg.norm(numgrad - grad) / np.linalg.norm(numgrad + grad)
print 'Norm difference: ', normdiff
return normdiff
def __init__(self, dictionary, X_train, X_dev=None, X_test=None):
self.X_train = X_train
self.X_dev = X_dev
self.X_test = X_test
self.dictionary = dictionary
self.costObj = ComputeCostAndGradMiniBatch()
dumb_model = RNTNModel(dictionary)
self.theta_init = dumb_model.getTheta()
self.num_data = len(X_train)
self.num_parameters = dumb_model.num_parameters
# SGD params
self.batch_size = dumb_model.batch_size
self.num_batches = self.num_data / self.batch_size
self.max_epochs = dumb_model.max_epochs
self.learning_rate = dumb_model.learning_rate
self.fudge = 1E-3
self.epoch_save_freq = 5 # save every 5 epochs
def __init__(self, dictionary, X_train, X_dev=None, X_test=None):
self.X_train = X_train
self.X_dev = X_dev
self.X_test = X_test
self.dictionary = dictionary
self.costObj = ComputeCostAndGradMiniBatch()
dumb_model = RNTNModel(dictionary)
self.theta_init = dumb_model.getTheta()
self.num_data = len(X_train)
self.num_parameters = dumb_model.num_parameters
# SGD params
self.batch_size = dumb_model.batch_size
self.num_batches = self.num_data / self.batch_size
self.max_epochs = dumb_model.max_epochs
self.learning_rate = dumb_model.learning_rate
self.fudge = 1E-3
self.epoch_save_freq = 5 # save every 5 epochs
def checkGradientClean(dictionary, trees):
# Code adopted from UFLDL gradientChecker
# http://ufldl.stanford.edu/wiki/index.php/Gradient_checking_and_advanced_optimization
model = RNTNModel(dictionary)
theta_init = model.getTheta()
# compute analytical gradient
costObj = ComputeCostAndGrad(dictionary, trees)
cost, grad = costObj.compute(theta_init)
eps = 1E-4
numgrad = np.zeros(grad.shape)
# compute numerical gradient
for i in range(model.num_parameters):
if i % 10 == 0:
print '%d/%d' % (i, model.num_parameters)
indicator = np.zeros(model.num_parameters)
indicator[i] = 1
theta_plus = theta_init + eps*indicator
cost_plus, grad_plus = costObj.compute(theta_plus)
theta_minus = theta_init - eps*indicator
cost_minus, grad_minus = costObj.compute(theta_minus)
numgrad[i] = (cost_plus - cost_minus)/(2*eps)
print 'analytical gradient: ', grad
print 'numerical gradient: ', numgrad
normdiff = np.linalg.norm(numgrad - grad) / np.linalg.norm(numgrad + grad)
print 'Norm difference: ', normdiff
return normdiff
def checkGradientClean(dictionary, trees):
# Code adopted from UFLDL gradientChecker
# http://ufldl.stanford.edu/wiki/index.php/Gradient_checking_and_advanced_optimization
model = RNTNModel(dictionary)
theta_init = model.getTheta()
# compute analytical gradient
costObj = ComputeCostAndGrad(dictionary, trees)
cost, grad = costObj.compute(theta_init)
eps = 1E-4
numgrad = np.zeros(grad.shape)
# compute numerical gradient
for i in range(model.num_parameters):
if i % 10 == 0:
print '%d/%d' % (i, model.num_parameters)
indicator = np.zeros(model.num_parameters)
indicator[i] = 1
theta_plus = theta_init + eps * indicator
cost_plus, grad_plus = costObj.compute(theta_plus)
theta_minus = theta_init - eps * indicator
cost_minus, grad_minus = costObj.compute(theta_minus)
numgrad[i] = (cost_plus - cost_minus) / (2 * eps)
print 'analytical gradient: ', grad
print 'numerical gradient: ', numgrad
normdiff = np.linalg.norm(numgrad - grad) / np.linalg.norm(numgrad + grad)
print 'Norm difference: ', normdiff
return normdiff
def __init__(self, dictionary, X):
self.costObj = ComputeCostAndGrad(dictionary, X)
dumb_model = RNTNModel(dictionary)
self.theta_init = dumb_model.getTheta()
def __init__(self, dictionary, X):
self.costObj = ComputeCostAndGrad(dictionary, X)
dumb_model = RNTNModel(dictionary)
self.theta_init = dumb_model.getTheta()
