def run(self, epochs=EPOCHS, learning_rate=LEARNING_RATE, regularization=REGULARIZATION, momentum=MOMENTUM, w2v_model=None):
if w2v_model is not None:
processor = NN_process.lengthTweetPairProcessor('../data/tweets/pairs/sets/tweet-pairs-train.txt',
'../data/tweets/pairs/sets/tweet-no-pairs-train.txt',
'../data/wiki/model/docfreq.npy', w2v_model, WORDS, EMBEDDING_DIM, BATCH_SIZE, cutoff)
else:
processor = NN_process.lengthTweetPairProcessor('../data/tweets/pairs/sets/tweet-pairs-train.txt',
'../data/tweets/pairs/sets/tweet-no-pairs-train.txt',
'../data/wiki/model/docfreq.npy', '../data/wiki/model/minimal', WORDS, EMBEDDING_DIM, BATCH_SIZE, cutoff)
train_x1 = theano.shared(value=processor.x1, name='train_x1', borrow=False)
train_x2 = theano.shared(value=processor.x2, name='train_x2', borrow=False)
train_i1 = theano.shared(value=processor.indices1, name='train_i1', borrow=False)
train_i2 = theano.shared(value=processor.indices2, name='train_i2', borrow=False)
train_l1 = theano.shared(value=processor.l1, name='train_l1', borrow=False)
train_l2 = theano.shared(value=processor.l2, name='train_l2', borrow=False)
train_y = theano.shared(value=processor.y, name='train_y', borrow=False)
train_z = theano.shared(value=processor.z, name='train_z', borrow=False)
print 'Initializing train function...'
train = self.train_function_momentum(train_x1, train_x2, train_i1, train_i2, train_l1, train_l2, train_y, train_z)
print 'Cost factor: ' + str(COST_FACTOR)
c = []
t = Thread(target=processor.process)
t.daemon = True
t.start()
def signal_handler(signal, frame):
import os
os._exit(0)
signal.signal(signal.SIGINT, signal_handler)
best_cost = float('inf')
best_weights = None
previous_best_cost = float('inf')
second_time = False
global LEARNING_RATE
for e in xrange(epochs):
processor.new_epoch()
processor.lock.acquire()
while not processor.ready:
processor.lock.wait()
processor.lock.release()
train_x1.set_value(processor.x1, borrow=False)
train_x2.set_value(processor.x2, borrow=False)
train_i1.set_value(processor.indices1, borrow=False)
train_i2.set_value(processor.indices2, borrow=False)
train_l1.set_value(processor.l1, borrow=False)
train_l2.set_value(processor.l2, borrow=False)
train_y.set_value(processor.y, borrow=False)
train_z.set_value(processor.z, borrow=False)
processor.lock.acquire()
processor.cont = True
processor.ready = False
processor.lock.notifyAll()
processor.lock.release()
for b in xrange(BATCHES):
c = []
cost = train(lr=learning_rate, reg=regularization, mom=momentum)
c.append(cost)
processor.lock.acquire()
while not processor.ready:
processor.lock.wait()
processor.lock.release()
# print 'Training, batch %d (from %d), cost %.5f' % (b, BATCHES, cost)
# we = self.model1.W.get_value()
# print we
train_x1.set_value(processor.x1, borrow=False)
train_x2.set_value(processor.x2, borrow=False)
train_i1.set_value(processor.indices1, borrow=False)
train_i2.set_value(processor.indices2, borrow=False)
train_l1.set_value(processor.l1, borrow=False)
train_l2.set_value(processor.l2, borrow=False)
train_y.set_value(processor.y, borrow=False)
train_z.set_value(processor.z, borrow=False)
processor.lock.acquire()
if b < BATCHES-2:
processor.cont = True
processor.ready = False
if b == BATCHES-1 and e == epochs-1:
processor.stop = True
processor.cont = True
processor.lock.notifyAll()
processor.lock.release()
print 'Training, factor %d, lr %.5f, epoch %d, cost %.5f' % (int(COST_FACTOR), LEARNING_RATE, e, numpy.mean(c))
we = self.model1.W.get_value()
print repr(we)
# UNCOMMENT WHEN THIS PIECE OF CODE IS CALLED EXTERNALLY
# if numpy.mean(c) < best_cost - 0.0005:
# previous_best_cost = best_cost
# best_cost = numpy.mean(c)
# best_weights = we
# elif second_time:
# processor.lock.acquire()
# processor.stop = True
# processor.cont = True
# processor.lock.notifyAll()
# processor.lock.release()
# break
# else:
# best_cost = previous_best_cost
# LEARNING_RATE = 0.001
# second_time = True
t.join()
print "Best weights:"
print repr(best_weights)
return best_weights
def run(self, epochs=EPOCHS, learning_rate=LEARNING_RATE, regularization=REGULARIZATION, momentum=MOMENTUM):
processor = NN_process.lengthLinTweetPairProcessor('../data/tweets/pairs/sets/tweet-pairs-train.txt',
'../data/tweets/pairs/sets/tweet-no-pairs-train.txt',
'../data/wiki/model/docfreq.npy', '../data/wiki/model/minimal', WORDS, EMBEDDING_DIM, BATCH_SIZE)
train_x1 = theano.shared(value=processor.x1, name='train_x1', borrow=False)
train_x2 = theano.shared(value=processor.x2, name='train_x2', borrow=False)
train_i1 = theano.shared(value=processor.indices1, name='train_i1', borrow=False)
train_i2 = theano.shared(value=processor.indices2, name='train_i2', borrow=False)
train_l1 = theano.shared(value=processor.l1, name='train_l1', borrow=False)
train_l2 = theano.shared(value=processor.l2, name='train_l2', borrow=False)
train_y = theano.shared(value=processor.y, name='train_y', borrow=False)
train_z = theano.shared(value=processor.z, name='train_z', borrow=False)
print 'Initializing train function...'
train = self.train_function_momentum(train_x1, train_x2, train_i1, train_i2, train_l1, train_l2, train_y, train_z)
c = []
t = Thread(target=processor.process)
t.daemon = True
t.start()
def signal_handler(signal, frame):
import os
os._exit(0)
signal.signal(signal.SIGINT, signal_handler)
best_cost = float('inf')
best_weights = None
for e in xrange(epochs):
processor.new_epoch()
processor.lock.acquire()
while not processor.ready:
processor.lock.wait()
processor.lock.release()
train_x1.set_value(processor.x1, borrow=False)
train_x2.set_value(processor.x2, borrow=False)
train_i1.set_value(processor.indices1, borrow=False)
train_i2.set_value(processor.indices2, borrow=False)
train_l1.set_value(processor.l1, borrow=False)
train_l2.set_value(processor.l2, borrow=False)
train_y.set_value(processor.y, borrow=False)
train_z.set_value(processor.z, borrow=False)
processor.lock.acquire()
processor.cont = True
processor.ready = False
processor.lock.notifyAll()
processor.lock.release()
for b in xrange(BATCHES):
c = []
cost = train(lr=learning_rate, reg=regularization, mom=momentum)
c.append(cost)
processor.lock.acquire()
while not processor.ready:
processor.lock.wait()
processor.lock.release()
# print 'Training, batch %d (from %d), cost %.5f' % (b, BATCHES, cost)
# we = self.model1.W.get_value()
# print we
train_x1.set_value(processor.x1, borrow=False)
train_x2.set_value(processor.x2, borrow=False)
train_i1.set_value(processor.indices1, borrow=False)
train_i2.set_value(processor.indices2, borrow=False)
train_l1.set_value(processor.l1, borrow=False)
train_l2.set_value(processor.l2, borrow=False)
train_y.set_value(processor.y, borrow=False)
train_z.set_value(processor.z, borrow=False)
processor.lock.acquire()
if b < BATCHES-2:
processor.cont = True
processor.ready = False
if b == BATCHES-1 and e == epochs-1:
processor.stop = True
processor.cont = True
processor.lock.notifyAll()
processor.lock.release()
print 'Training, epoch %d, cost %.5f' % (e, numpy.mean(c))
we = self.model1.W.get_value()
print repr(we)
# if numpy.mean(c) < best_cost - 0.0001:
# best_cost = numpy.mean(c)
# best_weights = we
# else:
# processor.lock.acquire()
# processor.stop = True
# processor.cont = True
# processor.lock.notifyAll()
# processor.lock.release()
# break
t.join()
return best_weights
def run(self,
epochs=EPOCHS,
learning_rate=LEARNING_RATE,
regularization=REGULARIZATION,
momentum=MOMENTUM):
processor = NN_process.lengthTweetPairProcessor(
'../data/tweets/pairs/sets/tweet-pairs-train.txt',
'../data/tweets/pairs/sets/tweet-no-pairs-train.txt',
'../data/wiki/model/docfreq.npy', '../data/wiki/model/minimal',
WORDS, EMBEDDING_DIM, BATCH_SIZE, cutoff)
train_x1 = theano.shared(value=processor.x1,
name='train_x1',
borrow=False)
train_x2 = theano.shared(value=processor.x2,
name='train_x2',
borrow=False)
train_i1 = theano.shared(value=processor.indices1,
name='train_i1',
borrow=False)
train_i2 = theano.shared(value=processor.indices2,
name='train_i2',
borrow=False)
train_l1 = theano.shared(value=processor.l1,
name='train_l1',
borrow=False)
train_l2 = theano.shared(value=processor.l2,
name='train_l2',
borrow=False)
train_y = theano.shared(value=processor.y,
name='train_y',
borrow=False)
train_z = theano.shared(value=processor.z,
name='train_z',
borrow=False)
print 'Initializing train function...'
train = self.train_function_momentum(train_x1, train_x2, train_i1,
train_i2, train_l1, train_l2,
train_y, train_z)
lin = numpy.linspace(0, float(WORDS) - 1.0, float(WORDS))
c = []
t = Thread(target=processor.process)
t.daemon = True
t.start()
def signal_handler(signal, frame):
import os
os._exit(0)
signal.signal(signal.SIGINT, signal_handler)
for e in xrange(epochs):
processor.new_epoch()
processor.lock.acquire()
while not processor.ready:
processor.lock.wait()
processor.lock.release()
train_x1.set_value(processor.x1, borrow=False)
train_x2.set_value(processor.x2, borrow=False)
train_i1.set_value(processor.indices1, borrow=False)
train_i2.set_value(processor.indices2, borrow=False)
train_l1.set_value(processor.l1, borrow=False)
train_l2.set_value(processor.l2, borrow=False)
train_y.set_value(processor.y, borrow=False)
train_z.set_value(processor.z, borrow=False)
processor.lock.acquire()
processor.cont = True
processor.ready = False
processor.lock.notifyAll()
processor.lock.release()
for b in xrange(BATCHES):
c = []
cost = train(lr=learning_rate,
reg=regularization,
mom=momentum)
c.append(cost)
processor.lock.acquire()
while not processor.ready:
processor.lock.wait()
processor.lock.release()
# print 'Training, batch %d (from %d), cost %.5f' % (b, BATCHES, cost)
# we = self.model1.W.get_value()
# s = numpy.log(1 + numpy.exp(we)) + 1.0
# print we
# print s
# temp = numpy.log(1.0 - 1.0/s[0]) / (WORDS - 1.0)
# f = s[0] * numpy.exp(temp * lin) + 1 - s[0]
# print f
train_x1.set_value(processor.x1, borrow=False)
train_x2.set_value(processor.x2, borrow=False)
train_i1.set_value(processor.indices1, borrow=False)
train_i2.set_value(processor.indices2, borrow=False)
train_l1.set_value(processor.l1, borrow=False)
train_l2.set_value(processor.l2, borrow=False)
train_y.set_value(processor.y, borrow=False)
train_z.set_value(processor.z, borrow=False)
processor.lock.acquire()
if b < BATCHES - 2:
processor.cont = True
processor.ready = False
if b == BATCHES - 1 and e == epochs - 1:
processor.stop = True
processor.lock.notifyAll()
processor.lock.release()
print 'Training, epoch %d, cost %.5f' % (e, numpy.mean(c))
we = self.model1.W.get_value()
s = numpy.log(1 + numpy.exp(we)) + 1.0
print we
print s
temp = numpy.log(1.0 - 1.0 / s[0]) / (WORDS - 1.0)
f = s[0] * numpy.exp(temp * lin) + 1 - s[0]
print f
t.join()
self.save_me('run7.npy')
def run(self, epochs=EPOCHS, learning_rate=LEARNING_RATE, regularization=REGULARIZATION, momentum=MOMENTUM):
processor = NN_process.lengthPairProcessor('../data/tweets/pairs/sets/tweet-pairs-train.txt',
'../data/tweets/pairs/sets/tweet-no-pairs-train.txt',
'../data/wiki/model/docfreq.npy', '../data/wiki/model/minimal', WORDS, EMBEDDING_DIM, BATCH_SIZE)
train_x1 = theano.shared(value=processor.x1, name='train_x1', borrow=False)
train_x2 = theano.shared(value=processor.x2, name='train_x2', borrow=False)
train_l1 = theano.shared(value=processor.l1, name='train_l1', borrow=False)
train_l2 = theano.shared(value=processor.l2, name='train_l2', borrow=False)
train_y = theano.shared(value=processor.y, name='train_y', borrow=False)
train_z = theano.shared(value=processor.z, name='train_z', borrow=False)
print 'Initializing train function...'
train = self.train_function_momentum(train_x1, train_x2, train_l1, train_l2, train_y, train_z)
lin = numpy.linspace(0, float(WORDS) - 1.0, float(WORDS))
c = []
t = Thread(target=processor.process)
t.start()
for e in xrange(epochs):
processor.new_epoch()
processor.lock.acquire()
while not processor.ready:
processor.lock.wait()
processor.lock.release()
train_x1.set_value(processor.x1, borrow=False)
train_x2.set_value(processor.x2, borrow=False)
train_l1.set_value(processor.l1, borrow=False)
train_l2.set_value(processor.l2, borrow=False)
train_y.set_value(processor.y, borrow=False)
train_z.set_value(processor.z, borrow=False)
processor.lock.acquire()
processor.cont = True
processor.ready = False
processor.lock.notifyAll()
processor.lock.release()
for b in xrange(BATCHES):
c = []
cost = train(lr=learning_rate, reg=regularization, mom=momentum)
c.append(cost)
processor.lock.acquire()
while not processor.ready:
processor.lock.wait()
processor.lock.release()
train_x1.set_value(processor.x1, borrow=False)
train_x2.set_value(processor.x2, borrow=False)
train_l1.set_value(processor.l1, borrow=False)
train_l2.set_value(processor.l2, borrow=False)
train_y.set_value(processor.y, borrow=False)
train_z.set_value(processor.z, borrow=False)
processor.lock.acquire()
if b < BATCHES-2:
processor.cont = True
processor.ready = False
if b == BATCHES-1 and e == epochs-1:
processor.stop = True
processor.lock.notifyAll()
processor.lock.release()
print 'Training, epoch %d, cost %.5f' % (e, numpy.mean(c))
we = self.model1.W.get_value()
print we
f = we[0] * (lin ** 4) + we[1] * (lin ** 3) + we[2] * (lin ** 2) + we[3] * lin + 1.0
print f
self.save_me('run5.npy')
def run(self, epochs=EPOCHS, learning_rate=LEARNING_RATE, regularization=REGULARIZATION, momentum=MOMENTUM):
processor = NN_process.lengthTweetPairProcessor(
"../data/tweets/pairs/sets/tweet-pairs-train.txt",
"../data/tweets/pairs/sets/tweet-no-pairs-train.txt",
"../data/wiki/model/docfreq.npy",
"../data/wiki/model/minimal",
WORDS,
EMBEDDING_DIM,
BATCH_SIZE,
cutoff,
)
train_x1 = theano.shared(value=processor.x1, name="train_x1", borrow=False)
train_x2 = theano.shared(value=processor.x2, name="train_x2", borrow=False)
train_i1 = theano.shared(value=processor.indices1, name="train_i1", borrow=False)
train_i2 = theano.shared(value=processor.indices2, name="train_i2", borrow=False)
train_l1 = theano.shared(value=processor.l1, name="train_l1", borrow=False)
train_l2 = theano.shared(value=processor.l2, name="train_l2", borrow=False)
train_y = theano.shared(value=processor.y, name="train_y", borrow=False)
train_z = theano.shared(value=processor.z, name="train_z", borrow=False)
print "Initializing train function..."
train = self.train_function_momentum(
train_x1, train_x2, train_i1, train_i2, train_l1, train_l2, train_y, train_z
)
lin = numpy.linspace(0, float(WORDS) - 1.0, float(WORDS))
c = []
t = Thread(target=processor.process)
t.daemon = True
t.start()
def signal_handler(signal, frame):
import os
os._exit(0)
signal.signal(signal.SIGINT, signal_handler)
for e in xrange(epochs):
processor.new_epoch()
processor.lock.acquire()
while not processor.ready:
processor.lock.wait()
processor.lock.release()
train_x1.set_value(processor.x1, borrow=False)
train_x2.set_value(processor.x2, borrow=False)
train_i1.set_value(processor.indices1, borrow=False)
train_i2.set_value(processor.indices2, borrow=False)
train_l1.set_value(processor.l1, borrow=False)
train_l2.set_value(processor.l2, borrow=False)
train_y.set_value(processor.y, borrow=False)
train_z.set_value(processor.z, borrow=False)
processor.lock.acquire()
processor.cont = True
processor.ready = False
processor.lock.notifyAll()
processor.lock.release()
for b in xrange(BATCHES):
c = []
cost = train(lr=learning_rate, reg=regularization, mom=momentum)
c.append(cost)
processor.lock.acquire()
while not processor.ready:
processor.lock.wait()
processor.lock.release()
# print 'Training, batch %d (from %d), cost %.5f' % (b, BATCHES, cost)
# we = self.model1.W.get_value()
# s = numpy.log(1 + numpy.exp(we)) + 1.0
# print we
# print s
# temp = numpy.log(1.0 - 1.0/s[0]) / (WORDS - 1.0)
# f = s[0] * numpy.exp(temp * lin) + 1 - s[0]
# print f
train_x1.set_value(processor.x1, borrow=False)
train_x2.set_value(processor.x2, borrow=False)
train_i1.set_value(processor.indices1, borrow=False)
train_i2.set_value(processor.indices2, borrow=False)
train_l1.set_value(processor.l1, borrow=False)
train_l2.set_value(processor.l2, borrow=False)
train_y.set_value(processor.y, borrow=False)
train_z.set_value(processor.z, borrow=False)
processor.lock.acquire()
if b < BATCHES - 2:
processor.cont = True
processor.ready = False
if b == BATCHES - 1 and e == epochs - 1:
processor.stop = True
processor.lock.notifyAll()
processor.lock.release()
print "Training, epoch %d, cost %.5f" % (e, numpy.mean(c))
we = self.model1.W.get_value()
s = numpy.log(1 + numpy.exp(we)) + 1.0
print we
print s
temp = numpy.log(1.0 - 1.0 / s[0]) / (WORDS - 1.0)
f = s[0] * numpy.exp(temp * lin) + 1 - s[0]
print f
t.join()
self.save_me("run7.npy")
def run(self, epochs=1, learning_rate=1.5, regularization=0.0, momentum=0.1):
processor = NN_process.unsortedPairProcessor('../data/pairs/sets/enwiki_pairs_20-train.txt', '../data/pairs/sets/enwiki_no_pairs_20-train.txt',
'../data/model/docfreq.npy', '../data/model/minimal', WORDS, EMBEDDING_DIM, BATCH_SIZE)
train_x1 = theano.shared(value=processor.x1, name='train_x1', borrow=False)
train_x2 = theano.shared(value=processor.x2, name='train_x2', borrow=False)
train_y = theano.shared(value=processor.y, name='train_y', borrow=False)
train_z = theano.shared(value=processor.z, name='train_z', borrow=False)
print 'Initializing train function...'
train = self.train_function_momentum(train_x1, train_x2, train_y, train_z)
t = Thread(target=processor.process)
t.start()
for e in xrange(epochs):
processor.new_epoch()
processor.lock.acquire()
while not processor.ready:
processor.lock.wait()
processor.lock.release()
train_x1.set_value(processor.x1, borrow=False)
train_x2.set_value(processor.x2, borrow=False)
train_y.set_value(processor.y, borrow=False)
train_z.set_value(processor.z, borrow=False)
processor.lock.acquire()
processor.cont = True
processor.ready = False
processor.lock.notifyAll()
processor.lock.release()
for b in xrange(BATCHES):
#c = []
cost = train(lr=learning_rate, reg=regularization, mom=momentum)
#c.append(cost)
print 'Training, batch %d, cost %.5f' % (b, cost)
print numpy.transpose(self.model1.W.get_value())
processor.lock.acquire()
while not processor.ready:
processor.lock.wait()
processor.lock.release()
train_x1.set_value(processor.x1, borrow=False)
train_x2.set_value(processor.x2, borrow=False)
train_y.set_value(processor.y, borrow=False)
train_z.set_value(processor.z, borrow=False)
processor.lock.acquire()
processor.cont = True
processor.ready = False
if b == BATCHES-1 and e == epochs-1:
processor.stop = True
processor.lock.notifyAll()
processor.lock.release()
#print 'Training, epoch %d, cost %.5f' % (e, numpy.mean(c))
self.save_me('run2.npy')
import NN_process
from threading import Thread
import time
import copy
NO_DATA = 4900000
BATCH_SIZE = 100
BATCHES = NO_DATA / BATCH_SIZE
EMBEDDING_DIM = 400
WORDS = 20
INPUT_SHAPE = (400, 20)
OUTPUT_SHAPE = (400, 1)
processor = NN_process.PairProcessor('../data/pairs/enwiki_pairs_20.txt',
'../data/pairs/enwiki_no_pairs_20.txt',
'../data/model/docfreq.npy',
'../data/model/minimal', WORDS,
EMBEDDING_DIM, BATCH_SIZE)
t = Thread(target=processor.process)
t.start()
print 'Start processor thread'
processor.new_epoch()
processor.lock.acquire()
while not processor.ready:
processor.lock.wait()
processor.lock.release()
processor.lock.acquire()
processor.cont = True
processor.ready = False
def run(self, epochs=EPOCHS, learning_rate=LEARNING_RATE, regularization=REGULARIZATION, momentum=MOMENTUM):
processor = NN_process.PairProcessor('../data/wiki/pairs/sets/enwiki_pairs_' + str(WORDS) + '-train.txt',
'../data/wiki/pairs/sets/enwiki_no_pairs_' + str(WORDS) + '-train.txt',
'../data/wiki/model/docfreq.npy', '../data/wiki/model/minimal', WORDS, EMBEDDING_DIM, BATCH_SIZE)
train_x1 = theano.shared(value=processor.x1, name='train_x1', borrow=False)
train_x2 = theano.shared(value=processor.x2, name='train_x2', borrow=False)
train_y = theano.shared(value=processor.y, name='train_y', borrow=False)
train_z = theano.shared(value=processor.z, name='train_z', borrow=False)
print 'Initializing train function...'
train = self.train_function_momentum(train_x1, train_x2, train_y, train_z)
t = Thread(target=processor.process)
t.daemon = True
t.start()
import signal
def signal_handler(signal, frame):
import os
os._exit(0)
signal.signal(signal.SIGINT, signal_handler)
for e in xrange(epochs):
processor.new_epoch()
processor.lock.acquire()
while not processor.ready:
processor.lock.wait()
processor.lock.release()
train_x1.set_value(processor.x1, borrow=False)
train_x2.set_value(processor.x2, borrow=False)
train_y.set_value(processor.y, borrow=False)
train_z.set_value(processor.z, borrow=False)
processor.lock.acquire()
processor.cont = True
processor.ready = False
processor.lock.notifyAll()
processor.lock.release()
for b in xrange(BATCHES):
#c = []
cost = train(lr=learning_rate, reg=regularization, mom=momentum)
#c.append(cost)
print 'Training, batch %d, cost %.5f' % (b, cost)
print repr(self.model1.W.get_value())
processor.lock.acquire()
while not processor.ready:
processor.lock.wait()
processor.lock.release()
train_x1.set_value(processor.x1, borrow=False)
train_x2.set_value(processor.x2, borrow=False)
train_y.set_value(processor.y, borrow=False)
train_z.set_value(processor.z, borrow=False)
processor.lock.acquire()
if b < BATCHES-2:
processor.cont = True
processor.ready = False
if b == BATCHES-1 and e == epochs-1:
processor.stop = True
processor.cont = True
processor.lock.notifyAll()
processor.lock.release()
#print 'Training, epoch %d, cost %.5f' % (e, numpy.mean(c))
t.join()
self.save_me('run1.npy')
def run(self, epochs=EPOCHS, learning_rate=LEARNING_RATE, regularization=REGULARIZATION, momentum=MOMENTUM, w2v_model=None):
processor = NN_process.lengthTweetPairProcessor('../data/wiki/pairs/sets/enwiki_pairs_' + 'r' + '-train.txt',
'../data/wiki/pairs/sets/enwiki_no_pairs_' + 'r' + '-train.txt',
'../data/wiki/model/docfreq.npy', '../data/wiki/model/minimal', WORDS, EMBEDDING_DIM, BATCH_SIZE, cutoff)
train_x1 = theano.shared(value=processor.x1, name='train_x1', borrow=False)
train_x2 = theano.shared(value=processor.x2, name='train_x2', borrow=False)
train_i1 = theano.shared(value=processor.indices1, name='train_i1', borrow=False)
train_i2 = theano.shared(value=processor.indices2, name='train_i2', borrow=False)
train_l1 = theano.shared(value=processor.l1, name='train_l1', borrow=False)
train_l2 = theano.shared(value=processor.l2, name='train_l2', borrow=False)
train_y = theano.shared(value=processor.y, name='train_y', borrow=False)
train_z = theano.shared(value=processor.z, name='train_z', borrow=False)
print 'Initializing train function...'
train = self.train_function_momentum(train_x1, train_x2, train_i1, train_i2, train_l1, train_l2, train_y, train_z)
print 'Cost factor: ' + str(COST_FACTOR)
t = Thread(target=processor.process)
t.daemon = True
t.start()
def signal_handler(signal, frame):
import os
os._exit(0)
signal.signal(signal.SIGINT, signal_handler)
best_cost = float('inf')
best_weights = None
previous_best_cost = float('inf')
second_time = False
global LEARNING_RATE
for e in xrange(epochs):
processor.new_epoch()
processor.lock.acquire()
while not processor.ready:
processor.lock.wait()
processor.lock.release()
train_x1.set_value(processor.x1, borrow=False)
train_x2.set_value(processor.x2, borrow=False)
train_i1.set_value(processor.indices1, borrow=False)
train_i2.set_value(processor.indices2, borrow=False)
train_l1.set_value(processor.l1, borrow=False)
train_l2.set_value(processor.l2, borrow=False)
train_y.set_value(processor.y, borrow=False)
train_z.set_value(processor.z, borrow=False)
processor.lock.acquire()
processor.cont = True
processor.ready = False
processor.lock.notifyAll()
processor.lock.release()
for b in xrange(BATCHES):
cost = train(lr=learning_rate, reg=regularization, mom=momentum)
processor.lock.acquire()
while not processor.ready:
processor.lock.wait()
processor.lock.release()
print 'Training, batch %d (from %d), cost %.5f' % (b, BATCHES, cost)
we = self.model1.W.get_value()
print repr(we)
train_x1.set_value(processor.x1, borrow=False)
train_x2.set_value(processor.x2, borrow=False)
train_i1.set_value(processor.indices1, borrow=False)
train_i2.set_value(processor.indices2, borrow=False)
train_l1.set_value(processor.l1, borrow=False)
train_l2.set_value(processor.l2, borrow=False)
train_y.set_value(processor.y, borrow=False)
train_z.set_value(processor.z, borrow=False)
processor.lock.acquire()
if b < BATCHES-2:
processor.cont = True
processor.ready = False
if b == BATCHES-1 and e == epochs-1:
processor.stop = True
processor.cont = True
processor.lock.notifyAll()
processor.lock.release()
#print 'Training, factor %d, lr %.5f, epoch %d, cost %.5f' % (int(COST_FACTOR), LEARNING_RATE, e, numpy.mean(c))
#we = self.model1.W.get_value()
#print repr(we)
# UNCOMMENT WHEN THIS PIECE OF CODE IS CALLED EXTERNALLY
# if numpy.mean(c) < best_cost - 0.0005:
# previous_best_cost = best_cost
# best_cost = numpy.mean(c)
# best_weights = we
# elif second_time:
# processor.lock.acquire()
# processor.stop = True
# processor.cont = True
# processor.lock.notifyAll()
# processor.lock.release()
# break
# else:
# best_cost = previous_best_cost
# LEARNING_RATE = 0.001
# second_time = True
t.join()
def run(self,
epochs=EPOCHS,
learning_rate=LEARNING_RATE,
regularization=REGULARIZATION,
momentum=MOMENTUM):
processor = NN_process.lengthPairProcessor(
'../data/tweets/pairs/sets/tweet-pairs-train.txt',
'../data/tweets/pairs/sets/tweet-no-pairs-train.txt',
'../data/wiki/model/docfreq.npy', '../data/wiki/model/minimal',
WORDS, EMBEDDING_DIM, BATCH_SIZE)
train_x1 = theano.shared(value=processor.x1,
name='train_x1',
borrow=False)
train_x2 = theano.shared(value=processor.x2,
name='train_x2',
borrow=False)
train_l1 = theano.shared(value=processor.l1,
name='train_l1',
borrow=False)
train_l2 = theano.shared(value=processor.l2,
name='train_l2',
borrow=False)
train_y = theano.shared(value=processor.y,
name='train_y',
borrow=False)
train_z = theano.shared(value=processor.z,
name='train_z',
borrow=False)
print 'Initializing train function...'
train = self.train_function_momentum(train_x1, train_x2, train_l1,
train_l2, train_y, train_z)
lin = numpy.linspace(0, float(WORDS) - 1.0, float(WORDS))
c = []
t = Thread(target=processor.process)
t.start()
for e in xrange(epochs):
processor.new_epoch()
processor.lock.acquire()
while not processor.ready:
processor.lock.wait()
processor.lock.release()
train_x1.set_value(processor.x1, borrow=False)
train_x2.set_value(processor.x2, borrow=False)
train_l1.set_value(processor.l1, borrow=False)
train_l2.set_value(processor.l2, borrow=False)
train_y.set_value(processor.y, borrow=False)
train_z.set_value(processor.z, borrow=False)
processor.lock.acquire()
processor.cont = True
processor.ready = False
processor.lock.notifyAll()
processor.lock.release()
for b in xrange(BATCHES):
c = []
cost = train(lr=learning_rate,
reg=regularization,
mom=momentum)
c.append(cost)
processor.lock.acquire()
while not processor.ready:
processor.lock.wait()
processor.lock.release()
# print 'Training, batch %d (from %d), cost %.5f' % (b, BATCHES, cost)
# we = self.model1.W.get_value()
# print we
# temp = math.log(1 - math.exp(-we[0])) / (WORDS - 1.0)
# f = numpy.exp(we[0] + lin * temp) + (1 - math.exp(we[0]))
# print f
train_x1.set_value(processor.x1, borrow=False)
train_x2.set_value(processor.x2, borrow=False)
train_l1.set_value(processor.l1, borrow=False)
train_l2.set_value(processor.l2, borrow=False)
train_y.set_value(processor.y, borrow=False)
train_z.set_value(processor.z, borrow=False)
processor.lock.acquire()
if b < BATCHES - 2:
processor.cont = True
processor.ready = False
if b == BATCHES - 1 and e == epochs - 1:
processor.stop = True
processor.lock.notifyAll()
processor.lock.release()
print 'Training, epoch %d, cost %.5f' % (e, numpy.mean(c))
we = self.model1.W.get_value()
s = 1.0 / (1.0 + numpy.exp(-we))
print we
print s
temp = math.log(1 - math.exp(-s[0])) / (WORDS - 1.0)
f = numpy.exp(s[0] + lin * temp) + (1 - math.exp(s[0]))
print f
t.join()
self.save_me('run6.npy')
def run(self,
epochs=EPOCHS,
learning_rate=LEARNING_RATE,
regularization=REGULARIZATION,
momentum=MOMENTUM):
processor = NN_process.lengthLinTweetPairProcessor(
'../data/tweets/pairs/sets/tweet-pairs-train.txt',
'../data/tweets/pairs/sets/tweet-no-pairs-train.txt',
'../data/wiki/model/docfreq.npy', '../data/wiki/model/minimal',
WORDS, EMBEDDING_DIM, BATCH_SIZE)
train_x1 = theano.shared(value=processor.x1,
name='train_x1',
borrow=False)
train_x2 = theano.shared(value=processor.x2,
name='train_x2',
borrow=False)
train_i1 = theano.shared(value=processor.indices1,
name='train_i1',
borrow=False)
train_i2 = theano.shared(value=processor.indices2,
name='train_i2',
borrow=False)
train_l1 = theano.shared(value=processor.l1,
name='train_l1',
borrow=False)
train_l2 = theano.shared(value=processor.l2,
name='train_l2',
borrow=False)
train_y = theano.shared(value=processor.y,
name='train_y',
borrow=False)
train_z = theano.shared(value=processor.z,
name='train_z',
borrow=False)
print 'Initializing train function...'
train = self.train_function_momentum(train_x1, train_x2, train_i1,
train_i2, train_l1, train_l2,
train_y, train_z)
c = []
t = Thread(target=processor.process)
t.daemon = True
t.start()
def signal_handler(signal, frame):
import os
os._exit(0)
signal.signal(signal.SIGINT, signal_handler)
best_cost = float('inf')
best_weights = None
for e in xrange(epochs):
processor.new_epoch()
processor.lock.acquire()
while not processor.ready:
processor.lock.wait()
processor.lock.release()
train_x1.set_value(processor.x1, borrow=False)
train_x2.set_value(processor.x2, borrow=False)
train_i1.set_value(processor.indices1, borrow=False)
train_i2.set_value(processor.indices2, borrow=False)
train_l1.set_value(processor.l1, borrow=False)
train_l2.set_value(processor.l2, borrow=False)
train_y.set_value(processor.y, borrow=False)
train_z.set_value(processor.z, borrow=False)
processor.lock.acquire()
processor.cont = True
processor.ready = False
processor.lock.notifyAll()
processor.lock.release()
for b in xrange(BATCHES):
c = []
cost = train(lr=learning_rate,
reg=regularization,
mom=momentum)
c.append(cost)
processor.lock.acquire()
while not processor.ready:
processor.lock.wait()
processor.lock.release()
# print 'Training, batch %d (from %d), cost %.5f' % (b, BATCHES, cost)
# we = self.model1.W.get_value()
# print we
train_x1.set_value(processor.x1, borrow=False)
train_x2.set_value(processor.x2, borrow=False)
train_i1.set_value(processor.indices1, borrow=False)
train_i2.set_value(processor.indices2, borrow=False)
train_l1.set_value(processor.l1, borrow=False)
train_l2.set_value(processor.l2, borrow=False)
train_y.set_value(processor.y, borrow=False)
train_z.set_value(processor.z, borrow=False)
processor.lock.acquire()
if b < BATCHES - 2:
processor.cont = True
processor.ready = False
if b == BATCHES - 1 and e == epochs - 1:
processor.stop = True
processor.cont = True
processor.lock.notifyAll()
processor.lock.release()
print 'Training, epoch %d, cost %.5f' % (e, numpy.mean(c))
we = self.model1.W.get_value()
print repr(we)
# if numpy.mean(c) < best_cost - 0.0001:
# best_cost = numpy.mean(c)
# best_weights = we
# else:
# processor.lock.acquire()
# processor.stop = True
# processor.cont = True
# processor.lock.notifyAll()
# processor.lock.release()
# break
t.join()
return best_weights
def run(self,
epochs=EPOCHS,
learning_rate=LEARNING_RATE,
regularization=REGULARIZATION,
momentum=MOMENTUM):
processor = NN_process.lengthPairProcessor(
'../data/pairs/sets/enwiki_pairs_' + WORDS_FILE + '-train.txt',
'../data/pairs/sets/enwiki_no_pairs_' + WORDS_FILE + '-train.txt',
'../data/model/docfreq.npy', '../data/model/minimal', WORDS,
EMBEDDING_DIM, BATCH_SIZE)
train_x1 = theano.shared(value=processor.x1,
name='train_x1',
borrow=False)
train_x2 = theano.shared(value=processor.x2,
name='train_x2',
borrow=False)
train_l1 = theano.shared(value=processor.l1,
name='train_l1',
borrow=False)
train_l2 = theano.shared(value=processor.l2,
name='train_l2',
borrow=False)
train_y = theano.shared(value=processor.y,
name='train_y',
borrow=False)
train_z = theano.shared(value=processor.z,
name='train_z',
borrow=False)
print 'Initializing train function...'
train = self.train_function_momentum(train_x1, train_x2, train_l1,
train_l2, train_y, train_z)
lin = numpy.linspace(0, 29.0, 30.0)
t = Thread(target=processor.process)
t.start()
for e in xrange(epochs):
processor.new_epoch()
processor.lock.acquire()
while not processor.ready:
processor.lock.wait()
processor.lock.release()
train_x1.set_value(processor.x1, borrow=False)
train_x2.set_value(processor.x2, borrow=False)
train_l1.set_value(processor.l1, borrow=False)
train_l2.set_value(processor.l2, borrow=False)
train_y.set_value(processor.y, borrow=False)
train_z.set_value(processor.z, borrow=False)
processor.lock.acquire()
processor.cont = True
processor.ready = False
processor.lock.notifyAll()
processor.lock.release()
for b in xrange(BATCHES):
#c = []
cost = train(lr=learning_rate,
reg=regularization,
mom=momentum)
#c.append(cost)
print 'Training, batch %d, cost %.5f' % (b, cost)
we = self.model1.W.get_value()
print repr(we)
f = we[0] * (lin**2) + we[1] * lin + we[2]
print f
processor.lock.acquire()
while not processor.ready:
processor.lock.wait()
processor.lock.release()
train_x1.set_value(processor.x1, borrow=False)
train_x2.set_value(processor.x2, borrow=False)
train_l1.set_value(processor.l1, borrow=False)
train_l2.set_value(processor.l2, borrow=False)
train_y.set_value(processor.y, borrow=False)
train_z.set_value(processor.z, borrow=False)
processor.lock.acquire()
if b < BATCHES - 2:
processor.cont = True
processor.ready = False
if b == BATCHES - 1 and e == epochs - 1:
processor.stop = True
processor.lock.notifyAll()
processor.lock.release()
#print 'Training, epoch %d, cost %.5f' % (e, numpy.mean(c))
self.save_me('run4.npy')
