def export_data(cursor, filename='test.csv', stdout_only=False): if stdout_only: result = cursor.fetchone() print("{0} rows returned".format(result[0])) # print count column return with open(filename, 'w', newline='') as csv_file: csv_writer = csv.writer(csv_file, delimiter=',', quoting=csv.QUOTE_MINIMAL) row_count = 0 for row in cursor: if row_count == 0: debug.print_debug("{0} results found".format(cursor.rowcount)) col_names = [desc[0] for desc in cursor.description] csv_writer.writerow(col_names) row_count += 1 csv_writer.writerow(row) if row_count == 0: print("No results found!".format(cursor.rowcount)) else: print("{0} results exported to {1}".format(row_count, filename))
def delete_hit(self,hitId): print_debug("deleting hit "+ hitId) try: index = self.ls['labels'].index(hitId) self.ls['samples'].pop(index) self.ls['labels'].pop(index) except Exception, e: print("[DEBUG] : hit not found on database")
def handle_cmd(self, cmd): print("[DEBUG] : handle_cmd " + cmd) try: if cmd[0] == "d": self.delete_hit(cmd[2:]) elif cmd[0] == "c": self.learn_hit(cmd[2:]) except: print_debug("Error parsing command " + str(cmd))
def get_query(self): select_fields = ", ".join(self.__get_selection_fields()) where_clauses = " AND ".join(self.__get_where_clauses()) query_str = "SELECT {0} FROM {1} {2} WHERE {3} {4}" \ .format(select_fields, self.get_table(), self.__get_join(), where_clauses, self.__get_limit()) debug.print_debug(query_str) return query_str
def __init__(self,ls): """Initialize shelves and algorithm""" self.ls = ls self.clf = svm.SVC(kernel='poly') if not self.ls.has_key('samples'): self.ls['samples'] = [] self.ls['labels'] = [] if len(self.ls['samples']) != len(self.ls['labels']): print_debug("erreur : database malforme") if len(self.ls['samples']) > 1: self.clf.fit(self.ls['samples'],np.arange(len(self.ls['labels'])))
def guessing(self, data): """ from one hit data, return the label of the most look-alike hit from the database """ if len(self.ls['samples']) > 1: self.clf.fit(self.ls['samples'],np.arange(len(self.ls['labels']))) if len(self.ls['samples']) == 1: return self.ls['labels'][0] if len(self.ls['samples']) == 0: print_debug("Aucune action faite car bibliotheque de coup vide") return "-1" return self.ls['labels'][self.clf.predict(self.format(data).reshape(1, -1))[0]]
def load_facets(node): facets = {} try: conn = SearchConnection( 'http://' + node + '/esg-search/', distrib=True) context = conn.new_context() for facet in context.get_facet_options(): facets[facet] = context.facet_counts[facet] except Exception as e: print_debug(e) return facets
def get_data(connection_data, query): conn_string = "host='{0}' dbname='{1}' user='******' password='******' port='{4}'" \ .format(connection_data['host_raw'] if query.is_raw() else connection_data['host_agg'], (connection_data['raw_db'] if query.is_raw() else connection_data['agg_db']), connection_data['user'], connection_data['password'], connection_data['port']) # print the connection string we will use to connect debug.print_debug("Connecting to database\n -> {0}".format([conn_string])) # get a connection, if a connect cannot be made an exception will be raised here conn = psycopg2.connect(conn_string) # HERE IS THE IMPORTANT PART, by specifying a name for the cursor # psycopg2 creates a server-side cursor, which prevents all of the # records from being downloaded at once from the server. cursor = conn.cursor('cursor_unique_name') cursor.execute(query.get_query()) return cursor
def api_query(self, method, req={}): print_debug("Acquiring the semaphore.") self.lock.acquire() print_debug("Acquired.") try: if method == "marketdata" or method == "orderdata" or method == "marketdatav2": ret = urllib2.urlopen(urllib2.Request('http://pubapi.cryptsy.com/api.php?method=' + method)) return json.loads(ret.read()) elif(method == "singlemarketdata" or method == "singleorderdata"): ret = urllib2.urlopen(urllib2.Request('http://pubapi.cryptsy.com/api.php?method=' + method + '&marketid=' + str(req['marketid']))) return json.loads(ret.read()) else: req['method'] = method req['nonce'] = int(time.time()) post_data = urllib.urlencode(req) sign = hmac.new(self.Secret, post_data, hashlib.sha512).hexdigest() headers = { 'Sign': sign, 'Key': self.APIKey } print_debug("Trying to open the URL.") #FIXME: Here there is a bug. ret = urllib2.urlopen(urllib2.Request('https://api.cryptsy.com/api', post_data, headers)) print_debug("URL opened.") print_debug("Converting the data to JSON.") jsonRet = json.loads(ret.read()) print_debug("Data converted.") print_debug("Calling post_process() and returning.") return self.post_process(jsonRet) finally: print_debug("Releasing the semaphore.") self.lock.release() print_debug("Released.")
def fit(self, train_X, optimizer, param_init = None, sample_every=None): self.opt = optimizer n_train, n_vis = train_X.shape batch_size = self.batch_size if sample_every == None: sample_every = 10000000 #theano.config.profile = True #theano.config.exception_verbosity='high' assert(n_vis == self.nv) train_X = self.shared_dataset(train_X) n_batches = np.ceil(n_train / float(batch_size)).astype('int') # theano variables for managing data (index minibatches, n examples in batch) index, n_ex = T.iscalars('batch_index', 'n_ex') batch_start = index*batch_size batch_stop = T.minimum(n_ex, (index + 1)*batch_size) effective_batch_size = batch_stop - batch_start # theano variables for learning lr = T.scalar('lr', dtype=theano.config.floatX) mom = T.scalar('mom', dtype=theano.config.floatX) if self.k == 1: # this one is for scaning over a batch and getting connectivity for each example # return grads too because T.grads through scan is awful # takes ~3x longer, but can experiment connectivity #K, grads = self.mpf.rbm_K2G(self.X, effective_batch_size) # this tiles out the minibatch matrix into a 3D tensor to compute connectivity #K, offs, y, y1, z= self.mpf.rbm_K(self.X, effective_batch_size) K = self.mpf.rbm_K(self.X, effective_batch_size) elif self.k == 2: if DEBUG: return_values = self.mpf.debug_rbm_K_2wise(self.X, effective_batch_size) K = return_values[-1] else: K = self.mpf.rbm_K_2wise(self.X, effective_batch_size) else: raise('NotImplemented') reg = self.L1_reg * self.mpf.L1 + self.L2_reg * self.mpf.L2 reg_grad = T.grad(reg, self.mpf.theta) # if not scan (tile out matrix into tensor) cost = K + reg grads = T.grad(cost, self.mpf.theta) # otherwise #grads = grads + reg_grad if param_init == None: self.mpf.theta.set_value(random_theta(D, DH, k=self.k)) else: self.mpf.theta.set_value(np.asarray(np.concatenate(param_init), dtype=theano.config.floatX)) if optimizer == 'sgd': updates = [] theta = self.mpf.theta theta_update = self.mpf.theta_update upd = mom * theta_update - lr * grads updates.append((theta_update, upd)) updates.append((theta, theta + upd)) print 'compiling theano function' if DEBUG: return_values = list(return_values) return_values.append(cost) return_values.append(grads) train_model = theano.function(inputs=[index, n_ex, lr, mom], outputs=return_values, updates=updates, givens={self.X: train_X[batch_start:batch_stop]}) else: train_model = theano.function(inputs=[index, n_ex, lr, mom], outputs=cost, updates=updates, givens={self.X: train_X[batch_start:batch_stop]}) self.current_epoch = 0 start = time.time() learning_rate_init = self.learning_rate while self.current_epoch < self.n_epochs: print 'epoch:', self.current_epoch self.current_epoch += 1 effective_mom = self.final_momentum if self.current_epoch > self.momentum_switchover else self.initial_momentum avg_epoch_cost = 0 last_debug = None for minibatch_idx in xrange(n_batches): avg_cost = train_model(minibatch_idx, n_train, self.learning_rate, effective_mom) #print '\t\t', np.isnan(gr).sum(), np.isnan(yy).sum(), np.isnan(yy1).sum(), np.isnan(zz).sum() if DEBUG: return_values, avg_cost, gradients = avg_cost[:-2], avg_cost[-2], avg_cost[-1] print_debug(return_values, last_debug) last_debug = return_values avg_epoch_cost += avg_cost #print '\t', minibatch_idx, avg_cost print '\t avg epoch cost:', avg_epoch_cost/n_batches self.learning_rate *= self.learning_rate_decay theta_fit = split_theta(self.mpf.theta.get_value(), self.mpf.n_visible, self.mpf.n_hidden, k=self.mpf.k) if (self.current_epoch % sample_every == 0): sample_and_save(theta_fit, self.mpf.n_hidden, self.current_epoch, learning_rate_init, self.mpf.k, self.opt) theta_opt = self.mpf.theta.get_value() end = time.time() elif optimizer == 'cg' or optimizer == 'bfgs': print "compiling theano functions" get_batch_size = theano.function([index, n_ex], effective_batch_size, name='get_batch_size') batch_cost_grads = theano.function([index, n_ex], [cost, grads], givens={self.X: train_X[batch_start:batch_stop, :]}, name='batch_cost') batch_cost = theano.function([index, n_ex], cost, givens={self.X: train_X[batch_start:batch_stop, :]}, name='batch_cost') batch_grads = theano.function([index, n_ex], grads, givens={self.X: train_X[batch_start:batch_stop, :]}, name='batch_cost') def train_fn_cost_grads(theta_value): print 'nbatches', n_batches self.mpf.theta.set_value(np.asarray(theta_value, dtype=theano.config.floatX), borrow=True) train_losses_grads = [batch_cost_gradst(i, n_train) for i in xrange(n_batches)] train_losses = [i[0] for i in train_losses_grads] train_grads = [i[1] for i in train_losses_grads] train_batch_sizes = [get_batch_size(i, n_train) for i in xrange(n_batches)] print len(train_losses), len(train_grads) print train_losses[0].shape, train_grads[0].shape returns = np.average(train_losses, weights=train_batch_sizes), np.average(train_grads, weights=train_batch_sizes, axis=0) return returns def train_fn_cost(theta_value): print 'nbatches', n_batches self.mpf.theta.set_value(np.asarray(theta_value, dtype=theano.config.floatX), borrow=True) train_costs = [batch_cost(i, n_train) for i in xrange(n_batches)] train_batch_sizes = [get_batch_size(i, n_train) for i in xrange(n_batches)] return np.average(train_costs, weights=train_batch_sizes) def train_fn_grads(theta_value): print 'nbatches', n_batches self.mpf.theta.set_value(np.asarray(theta_value, dtype=theano.config.floatX), borrow=True) train_grads = [batch_grads(i, n_train) for i in xrange(n_batches)] train_batch_sizes = [get_batch_size(i, n_train) for i in xrange(n_batches)] return np.average(train_grads, weights=train_batch_sizes, axis=0) ############### # TRAIN MODEL # ############### def my_callback(): print 'wtf' from scipy.optimize import minimize from scipy.optimize import fmin_bfgs, fmin_l_bfgs_b if optimizer == 'cg': pass elif optimizer == 'bfgs': print 'using bfgs' #theta_opt, f_theta_opt, info = fmin_l_bfgs_b(train_fn, self.mpf.theta.get_value(), iprint=1, maxfun=self.n_epochs) start = time.time() disp = True print 'ready to minimize' #result_obj = minimize(train_fn, self.mpf.theta.get_value(), jac=True, method='BFGS', options={'maxiter':self.n_epochs, 'disp':disp}, callback=my_callback()) #theta_opt = fmin_bfgs(f=train_fn_cost, x0=self.mpf.theta.get_value(), fprime=train_fn_grads, disp=1, maxiter=self.n_epochs) theta_opt, fff, ddd = fmin_l_bfgs_b(func=train_fn_cost, x0=self.mpf.theta.get_value(), fprime=train_fn_grads, disp=1, maxiter=self.n_epochs) print 'done minimize ya right' end = time.time() elif optimizer == 'sof': print "compiling theano functions" batch_cost_grads = theano.function([index, n_ex], [cost, grads], givens={self.X: train_X[batch_start:batch_stop, :]}, name='batch_cost') batch_cost = theano.function([index, n_ex], cost, givens={self.X: train_X[batch_start:batch_stop, :]}, name='batch_cost') batch_grads = theano.function([index, n_ex], grads, givens={self.X: train_X[batch_start:batch_stop, :]}, name='batch_cost') def train_fn(theta_value, i): self.mpf.theta.set_value(np.asarray(theta_value, dtype=theano.config.floatX), borrow=True) train_losses, train_grads = batch_cost_grads(i, n_train) return train_losses, train_grads ############### # TRAIN MODEL # ############### if param_init == None: theta.set_value(random_theta(D, DH)) else: w0, bh0, bv0 = param_init self.mpf.theta.set_value(np.asarray(np.concatenate((w0, bh0, bv0)), dtype=theano.config.floatX)) print 'using sof' sys.path.append('/export/mlrg/ebuchman/Programming/Sum-of-Functions-Optimizer') from sfo import SFO print 'n batches', n_batches print 'n epochs' , self.n_epochs optimizer = SFO(train_fn, self.mpf.theta.get_value(), np.arange(n_batches)) start = time.time() theta_opt = optimizer.optimize(num_passes = self.n_epochs) end = time.time() self.mpf.theta.set_value(theta_opt.astype(theano.config.floatX), borrow=True) return end-start
def handle_hit(self, hitId): print_debug("handle hit " + hitId) # openhab.post_command(scriptListener,hitId) self.openHab.post_command("scriptListener",hitId)