def export_data(cursor, filename='test.csv', stdout_only=False):
if stdout_only:
result = cursor.fetchone()
print("{0} rows returned".format(result[0])) # print count column
return
with open(filename, 'w', newline='') as csv_file:
csv_writer = csv.writer(csv_file,
delimiter=',',
quoting=csv.QUOTE_MINIMAL)
row_count = 0
for row in cursor:
if row_count == 0:
debug.print_debug("{0} results found".format(cursor.rowcount))
col_names = [desc[0] for desc in cursor.description]
csv_writer.writerow(col_names)
row_count += 1
csv_writer.writerow(row)
if row_count == 0:
print("No results found!".format(cursor.rowcount))
else:
print("{0} results exported to {1}".format(row_count, filename))
def delete_hit(self,hitId):
print_debug("deleting hit "+ hitId)
try:
index = self.ls['labels'].index(hitId)
self.ls['samples'].pop(index)
self.ls['labels'].pop(index)
except Exception, e:
print("[DEBUG] : hit not found on database")
def handle_cmd(self, cmd):
print("[DEBUG] : handle_cmd " + cmd)
try:
if cmd[0] == "d":
self.delete_hit(cmd[2:])
elif cmd[0] == "c":
self.learn_hit(cmd[2:])
except:
print_debug("Error parsing command " + str(cmd))
def get_query(self):
select_fields = ", ".join(self.__get_selection_fields())
where_clauses = " AND ".join(self.__get_where_clauses())
query_str = "SELECT {0} FROM {1} {2} WHERE {3} {4}" \
.format(select_fields, self.get_table(), self.__get_join(), where_clauses, self.__get_limit())
debug.print_debug(query_str)
return query_str
def __init__(self,ls):
"""Initialize shelves and algorithm"""
self.ls = ls
self.clf = svm.SVC(kernel='poly')
if not self.ls.has_key('samples'):
self.ls['samples'] = []
self.ls['labels'] = []
if len(self.ls['samples']) != len(self.ls['labels']):
print_debug("erreur : database malforme")
if len(self.ls['samples']) > 1:
self.clf.fit(self.ls['samples'],np.arange(len(self.ls['labels'])))
def guessing(self, data):
"""
from one hit data, return the label of the most look-alike hit from the database
"""
if len(self.ls['samples']) > 1:
self.clf.fit(self.ls['samples'],np.arange(len(self.ls['labels'])))
if len(self.ls['samples']) == 1:
return self.ls['labels'][0]
if len(self.ls['samples']) == 0:
print_debug("Aucune action faite car bibliotheque de coup vide")
return "-1"
return self.ls['labels'][self.clf.predict(self.format(data).reshape(1, -1))[0]]
def load_facets(node):
facets = {}
try:
conn = SearchConnection(
'http://' + node + '/esg-search/', distrib=True)
context = conn.new_context()
for facet in context.get_facet_options():
facets[facet] = context.facet_counts[facet]
except Exception as e:
print_debug(e)
return facets
def get_data(connection_data, query):
conn_string = "host='{0}' dbname='{1}' user='******' password='******' port='{4}'" \
.format(connection_data['host_raw'] if query.is_raw() else connection_data['host_agg'],
(connection_data['raw_db'] if query.is_raw() else connection_data['agg_db']),
connection_data['user'],
connection_data['password'],
connection_data['port'])
# print the connection string we will use to connect
debug.print_debug("Connecting to database\n -> {0}".format([conn_string]))
# get a connection, if a connect cannot be made an exception will be raised here
conn = psycopg2.connect(conn_string)
# HERE IS THE IMPORTANT PART, by specifying a name for the cursor
# psycopg2 creates a server-side cursor, which prevents all of the
# records from being downloaded at once from the server.
cursor = conn.cursor('cursor_unique_name')
cursor.execute(query.get_query())
return cursor
def api_query(self, method, req={}):
print_debug("Acquiring the semaphore.")
self.lock.acquire()
print_debug("Acquired.")
try:
if method == "marketdata" or method == "orderdata" or method == "marketdatav2":
ret = urllib2.urlopen(urllib2.Request('http://pubapi.cryptsy.com/api.php?method=' + method))
return json.loads(ret.read())
elif(method == "singlemarketdata" or method == "singleorderdata"):
ret = urllib2.urlopen(urllib2.Request('http://pubapi.cryptsy.com/api.php?method=' + method + '&marketid=' + str(req['marketid'])))
return json.loads(ret.read())
else:
req['method'] = method
req['nonce'] = int(time.time())
post_data = urllib.urlencode(req)
sign = hmac.new(self.Secret, post_data, hashlib.sha512).hexdigest()
headers = {
'Sign': sign,
'Key': self.APIKey
}
print_debug("Trying to open the URL.")
#FIXME: Here there is a bug.
ret = urllib2.urlopen(urllib2.Request('https://api.cryptsy.com/api', post_data, headers))
print_debug("URL opened.")
print_debug("Converting the data to JSON.")
jsonRet = json.loads(ret.read())
print_debug("Data converted.")
print_debug("Calling post_process() and returning.")
return self.post_process(jsonRet)
finally:
print_debug("Releasing the semaphore.")
self.lock.release()
print_debug("Released.")
def fit(self, train_X, optimizer, param_init = None, sample_every=None):
self.opt = optimizer
n_train, n_vis = train_X.shape
batch_size = self.batch_size
if sample_every == None:
sample_every = 10000000
#theano.config.profile = True
#theano.config.exception_verbosity='high'
assert(n_vis == self.nv)
train_X = self.shared_dataset(train_X)
n_batches = np.ceil(n_train / float(batch_size)).astype('int')
# theano variables for managing data (index minibatches, n examples in batch)
index, n_ex = T.iscalars('batch_index', 'n_ex')
batch_start = index*batch_size
batch_stop = T.minimum(n_ex, (index + 1)*batch_size)
effective_batch_size = batch_stop - batch_start
# theano variables for learning
lr = T.scalar('lr', dtype=theano.config.floatX)
mom = T.scalar('mom', dtype=theano.config.floatX)
if self.k == 1:
# this one is for scaning over a batch and getting connectivity for each example
# return grads too because T.grads through scan is awful
# takes ~3x longer, but can experiment connectivity
#K, grads = self.mpf.rbm_K2G(self.X, effective_batch_size)
# this tiles out the minibatch matrix into a 3D tensor to compute connectivity
#K, offs, y, y1, z= self.mpf.rbm_K(self.X, effective_batch_size)
K = self.mpf.rbm_K(self.X, effective_batch_size)
elif self.k == 2:
if DEBUG:
return_values = self.mpf.debug_rbm_K_2wise(self.X, effective_batch_size)
K = return_values[-1]
else:
K = self.mpf.rbm_K_2wise(self.X, effective_batch_size)
else:
raise('NotImplemented')
reg = self.L1_reg * self.mpf.L1 + self.L2_reg * self.mpf.L2
reg_grad = T.grad(reg, self.mpf.theta)
# if not scan (tile out matrix into tensor)
cost = K + reg
grads = T.grad(cost, self.mpf.theta)
# otherwise
#grads = grads + reg_grad
if param_init == None:
self.mpf.theta.set_value(random_theta(D, DH, k=self.k))
else:
self.mpf.theta.set_value(np.asarray(np.concatenate(param_init), dtype=theano.config.floatX))
if optimizer == 'sgd':
updates = []
theta = self.mpf.theta
theta_update = self.mpf.theta_update
upd = mom * theta_update - lr * grads
updates.append((theta_update, upd))
updates.append((theta, theta + upd))
print 'compiling theano function'
if DEBUG:
return_values = list(return_values)
return_values.append(cost)
return_values.append(grads)
train_model = theano.function(inputs=[index, n_ex, lr, mom], outputs=return_values, updates=updates, givens={self.X: train_X[batch_start:batch_stop]})
else:
train_model = theano.function(inputs=[index, n_ex, lr, mom], outputs=cost, updates=updates, givens={self.X: train_X[batch_start:batch_stop]})
self.current_epoch = 0
start = time.time()
learning_rate_init = self.learning_rate
while self.current_epoch < self.n_epochs:
print 'epoch:', self.current_epoch
self.current_epoch += 1
effective_mom = self.final_momentum if self.current_epoch > self.momentum_switchover else self.initial_momentum
avg_epoch_cost = 0
last_debug = None
for minibatch_idx in xrange(n_batches):
avg_cost = train_model(minibatch_idx, n_train, self.learning_rate, effective_mom)
#print '\t\t', np.isnan(gr).sum(), np.isnan(yy).sum(), np.isnan(yy1).sum(), np.isnan(zz).sum()
if DEBUG:
return_values, avg_cost, gradients = avg_cost[:-2], avg_cost[-2], avg_cost[-1]
print_debug(return_values, last_debug)
last_debug = return_values
avg_epoch_cost += avg_cost
#print '\t', minibatch_idx, avg_cost
print '\t avg epoch cost:', avg_epoch_cost/n_batches
self.learning_rate *= self.learning_rate_decay
theta_fit = split_theta(self.mpf.theta.get_value(), self.mpf.n_visible, self.mpf.n_hidden, k=self.mpf.k)
if (self.current_epoch % sample_every == 0):
sample_and_save(theta_fit, self.mpf.n_hidden, self.current_epoch, learning_rate_init, self.mpf.k, self.opt)
theta_opt = self.mpf.theta.get_value()
end = time.time()
elif optimizer == 'cg' or optimizer == 'bfgs':
print "compiling theano functions"
get_batch_size = theano.function([index, n_ex], effective_batch_size, name='get_batch_size')
batch_cost_grads = theano.function([index, n_ex], [cost, grads], givens={self.X: train_X[batch_start:batch_stop, :]}, name='batch_cost')
batch_cost = theano.function([index, n_ex], cost, givens={self.X: train_X[batch_start:batch_stop, :]}, name='batch_cost')
batch_grads = theano.function([index, n_ex], grads, givens={self.X: train_X[batch_start:batch_stop, :]}, name='batch_cost')
def train_fn_cost_grads(theta_value):
print 'nbatches', n_batches
self.mpf.theta.set_value(np.asarray(theta_value, dtype=theano.config.floatX), borrow=True)
train_losses_grads = [batch_cost_gradst(i, n_train) for i in xrange(n_batches)]
train_losses = [i[0] for i in train_losses_grads]
train_grads = [i[1] for i in train_losses_grads]
train_batch_sizes = [get_batch_size(i, n_train) for i in xrange(n_batches)]
print len(train_losses), len(train_grads)
print train_losses[0].shape, train_grads[0].shape
returns = np.average(train_losses, weights=train_batch_sizes), np.average(train_grads, weights=train_batch_sizes, axis=0)
return returns
def train_fn_cost(theta_value):
print 'nbatches', n_batches
self.mpf.theta.set_value(np.asarray(theta_value, dtype=theano.config.floatX), borrow=True)
train_costs = [batch_cost(i, n_train) for i in xrange(n_batches)]
train_batch_sizes = [get_batch_size(i, n_train) for i in xrange(n_batches)]
return np.average(train_costs, weights=train_batch_sizes)
def train_fn_grads(theta_value):
print 'nbatches', n_batches
self.mpf.theta.set_value(np.asarray(theta_value, dtype=theano.config.floatX), borrow=True)
train_grads = [batch_grads(i, n_train) for i in xrange(n_batches)]
train_batch_sizes = [get_batch_size(i, n_train) for i in xrange(n_batches)]
return np.average(train_grads, weights=train_batch_sizes, axis=0)
###############
# TRAIN MODEL #
###############
def my_callback():
print 'wtf'
from scipy.optimize import minimize
from scipy.optimize import fmin_bfgs, fmin_l_bfgs_b
if optimizer == 'cg':
pass
elif optimizer == 'bfgs':
print 'using bfgs'
#theta_opt, f_theta_opt, info = fmin_l_bfgs_b(train_fn, self.mpf.theta.get_value(), iprint=1, maxfun=self.n_epochs)
start = time.time()
disp = True
print 'ready to minimize'
#result_obj = minimize(train_fn, self.mpf.theta.get_value(), jac=True, method='BFGS', options={'maxiter':self.n_epochs, 'disp':disp}, callback=my_callback())
#theta_opt = fmin_bfgs(f=train_fn_cost, x0=self.mpf.theta.get_value(), fprime=train_fn_grads, disp=1, maxiter=self.n_epochs)
theta_opt, fff, ddd = fmin_l_bfgs_b(func=train_fn_cost, x0=self.mpf.theta.get_value(), fprime=train_fn_grads, disp=1, maxiter=self.n_epochs)
print 'done minimize ya right'
end = time.time()
elif optimizer == 'sof':
print "compiling theano functions"
batch_cost_grads = theano.function([index, n_ex], [cost, grads], givens={self.X: train_X[batch_start:batch_stop, :]}, name='batch_cost')
batch_cost = theano.function([index, n_ex], cost, givens={self.X: train_X[batch_start:batch_stop, :]}, name='batch_cost')
batch_grads = theano.function([index, n_ex], grads, givens={self.X: train_X[batch_start:batch_stop, :]}, name='batch_cost')
def train_fn(theta_value, i):
self.mpf.theta.set_value(np.asarray(theta_value, dtype=theano.config.floatX), borrow=True)
train_losses, train_grads = batch_cost_grads(i, n_train)
return train_losses, train_grads
###############
# TRAIN MODEL #
###############
if param_init == None:
theta.set_value(random_theta(D, DH))
else:
w0, bh0, bv0 = param_init
self.mpf.theta.set_value(np.asarray(np.concatenate((w0, bh0, bv0)), dtype=theano.config.floatX))
print 'using sof'
sys.path.append('/export/mlrg/ebuchman/Programming/Sum-of-Functions-Optimizer')
from sfo import SFO
print 'n batches', n_batches
print 'n epochs' , self.n_epochs
optimizer = SFO(train_fn, self.mpf.theta.get_value(), np.arange(n_batches))
start = time.time()
theta_opt = optimizer.optimize(num_passes = self.n_epochs)
end = time.time()
self.mpf.theta.set_value(theta_opt.astype(theano.config.floatX), borrow=True)
return end-start
def handle_hit(self, hitId):
print_debug("handle hit " + hitId)
# openhab.post_command(scriptListener,hitId)
self.openHab.post_command("scriptListener",hitId)
