def __init__(self, ticker_len, start_amount, histdepth, base_sym): self.trade_hist = {} self.polo = Poloniex() self.hd = histdepth self.ticker_len = ticker_len self.end_ts = datetime.now() + timedelta(seconds=(ticker_len * 24)) self.start_amount = start_amount self.hs = HistWorker() self.hs.combine_live_usd_frames() print(self.hs.currentHists.keys()) self.end_idx = len(self.hs.hist_shaped[0]) - 1 self.but_target = .1 self.inputs = self.hs.hist_shaped.shape[0] * ( self.hs.hist_shaped[0].shape[1]) self.outputs = self.hs.hist_shaped.shape[0] self.folio = CryptoFolio(start_amount, list(self.hs.currentHists.keys()), base_sym) self.base_sym = base_sym sign = 1 for ix in range(1, self.outputs + 1): sign = sign * -1 self.out_shapes.append((0.0 - (sign * .005 * ix), 0.0, -1.0)) for ix2 in range(1, (self.inputs // self.outputs) + 1): self.in_shapes.append( (0.0 + (sign * .01 * ix2), 0.0 - (sign * .01 * ix2), 0.0)) self.subStrate = Substrate(self.in_shapes, self.out_shapes) self.load_net() print(self.hs.coin_dict) self.poloTrader()
def __init__(self, ticker_len, start_amount, histdepth): self.in_shapes = [] self.out_shapes = [] self.trade_hist = {} self.polo = Poloniex() self.hist_depth = histdepth self.ticker_len = ticker_len self.end_ts = datetime.now()+timedelta(seconds=(ticker_len*24)) self.start_amount = start_amount self.hs = HistWorker() self.refresh_data() self.inputs = self.hs.hist_shaped.shape[0]*(self.hs.hist_shaped[0].shape[1]) self.outputs = self.hs.hist_shaped.shape[0] #self.make_shapes() self.folio = CryptoFolio(start_amount, list(self.hs.currentHists.keys())) self.leaf_names = [] for ix in range(1,self.inputs+1): sign = sign *-1 self.in_shapes.append((0.0-(sign*.005*ix), -1.0, 0.0+(sign*.005*ix))) self.out_shapes.append((0.0, 1.0, 0.0)) self.subStrate = Substrate(self.in_shapes, self.out_shapes) for l in range(len(self.in_shapes[0])): self.leaf_names.append('leaf_one_'+str(l)) self.leaf_names.append('leaf_two_'+str(l)) self.load_net() print(self.hs.coin_dict) self.poloTrader()
def __init__(self, hist_depth): self.hs = HistWorker() self.hs.combine_binance_frames() self.hd = hist_depth print(self.hs.currentHists.keys()) self.end_idx = len(self.hs.hist_shaped[0]) self.but_target = .1 self.inputs = self.hs.hist_shaped.shape[0] * ( self.hs.hist_shaped[0].shape[1]) self.outputs = self.hs.hist_shaped.shape[0] sign = 1 for ix in range(1, self.outputs + 1): sign = sign * -1 self.out_shapes.append((0.0 - (sign * .005 * ix), -1.0, -1.0)) for ix2 in range(1, (self.inputs // self.outputs) + 1): self.in_shapes.append( (0.0 + (sign * .01 * ix2), 0.0 - (sign * .01 * ix2), 1.0)) self.subStrate = Substrate(self.in_shapes, self.out_shapes) self.epoch_len = 144 #self.node_names = ['x1', 'y1', 'z1', 'x2', 'y2', 'z2', 'weight'] self.leaf_names = [] #num_leafs = 2**(len(self.node_names)-1)//2 for l in range(len(self.in_shapes[0])): self.leaf_names.append('leaf_one_' + str(l)) self.leaf_names.append('leaf_two_' + str(l))
def __init__(self, hist_depth): self.hs = HistWorker() self.hs.combine_binance_frames_vol_sorted(3) self.hd = hist_depth print(self.hs.currentHists.keys()) self.end_idx = len(self.hs.hist_shaped[0]) self.but_target = .1 self.inputs = self.hs.hist_shaped.shape[0] * ( self.hs.hist_shaped[0].shape[1]) self.outputs = self.hs.hist_shaped.shape[0] self.leaf_names = [] #num_leafs = 2**(len(self.node_names)-1)//2 #self.tree = nDimensionGoldenTree((0.0, 0.0, 0.0), 1.0, 1) #self.tree.divide_childrens() self.set_substrate() self.set_leaf_names() self.epoch_len = hist_depth
def __init__(self, hist_depth): self.hs = HistWorker() self.hs.combine_binance_frames_vol_sorted(21) self.hd = hist_depth print(self.hs.currentHists.keys()) self.end_idx = len(self.hs.hist_shaped[0]) self.but_target = .1 self.inputs = self.hs.hist_shaped.shape[0]*(self.hs.hist_shaped[0].shape[1]) self.outputs = len(self.hs.coin_dict) print(self.inputs, self.outputs) self.epoch_len = 144 #self.node_names = ['x1', 'y1', 'z1', 'x2', 'y2', 'z2', 'weight'] self.leaf_names = [] #num_leafs = 2**(len(self.node_names)-1)//2 self.tree = nDimensionTree((0.0, 0.0, 0.0), 1.0, 1) self.tree.divide_childrens() self.set_substrate() self.set_leaf_names()
def refresh(self): self.in_shapes = [] self.out_shapes = [(0.0, -1.0, -1.0)] self.hs = HistWorker() self.hs.get_binance_train() print(self.hs.currentHists.keys()) self.end_idx = len(self.hs.hist_shaped[0]) self.but_target = .1 print(self.hs.hist_shaped.shape) self.num_syms = self.hs.hist_shaped.shape[0] self.inputs = self.hs.hist_shaped[0].shape[1] + 1 self.outputs = 1 sign = 1 for ix2 in range(1,self.inputs+1): sign *= -1 self.in_shapes.append((0.0+(sign*.01*ix2), 0.0-(sign*.01*ix2), 0.0)) self.substrate = Substrate(self.in_shapes, self.out_shapes) self.set_leaf_names()
def refresh(self): self.in_shapes = [] self.out_shapes = [] self.hs = HistWorker() self.hs.get_kraken_train() print(self.hs.currentHists.keys()) self.end_idx = len(self.hs.hist_shaped[0]) self.but_target = .1 self.inputs = self.hs.hist_shaped.shape[0] * ( self.hs.hist_shaped[0].shape[1]) self.outputs = self.hs.hist_shaped.shape[0] sign = 1 for ix in range(1, self.outputs + 1): sign = sign * -1 self.out_shapes.append((0.0 - (sign * .005 * ix), 0.0, -1.0)) for ix2 in range(1, (self.inputs // self.outputs) + 1): self.in_shapes.append( (0.0 + (sign * .01 * ix2), 0.0 - (sign * .01 * ix2), 0.0)) self.subStrate = Substrate(self.in_shapes, self.out_shapes)
def refresh(self): self.in_shapes = [] self.out_shapes = [(0.0, -1.0, -1.0)] self.hs = HistWorker() self.hs.get_kraken_train() print(self.hs.currentHists.keys()) self.end_idx = len(self.hs.hist_shaped[0]) self.but_target = .1 print(self.hs.hist_shaped.shape) self.num_syms = self.hs.hist_shaped.shape[0] # add one to number of symbols to account for current position size # input we pass for context self.inputs = self.hs.hist_shaped[0].shape[1] + 1 self.outputs = 1 sign = 1 for ix2 in range(1,self.inputs+1): sign *= -1 self.in_shapes.append((0.0+(sign*.01*ix2), 0.0-(sign*.01*ix2), 0.0)) self.substrate = Substrate(self.in_shapes, self.out_shapes) self.set_leaf_names()
def refresh(self, reload_data=False): print("refreshing") self.in_shapes = [] self.out_shapes = [] self.hs = HistWorker() if(reload_data != False): self.hs.pull_robinhood_train_data() self.hs.get_robinhood_train() print(self.hs.currentHists.keys()) self.end_idx = len(self.hs.hist_shaped[0]) self.but_target = 1.0 self.inputs = self.hs.hist_shaped.shape[0]*(self.hs.hist_shaped[0].shape[1]) self.outputs = self.hs.hist_shaped.shape[0] sign = 1 for ix in range(1,self.outputs+1): sign = sign *-1 self.out_shapes.append((0.0-(sign*.005*ix), 0.0, -1.0)) for ix2 in range(1,(self.inputs//self.outputs)+1): self.in_shapes.append((0.0+(sign*.01*ix2), 0.0-(sign*.01*ix2), 0.0)) self.subStrate = Substrate(self.in_shapes, self.out_shapes)
def __init__(self, ticker_len, target_percent, hd, base_sym="BTC"): self.base_sym = base_sym self.load_polo_client() self.hd = hd self.target_percent = target_percent self.ticker_len = ticker_len self.end_ts = datetime.now() + timedelta(seconds=(ticker_len * 55)) self.hs = HistWorker() self.refresh_data() self.tickers = self.polo.returnTicker() self.refresh_balances() self.sellCoins() self.set_target() self.inputs = self.hs.hist_shaped.shape[0] * ( self.hs.hist_shaped[0].shape[1]) self.outputs = self.hs.hist_shaped.shape[0] self.end_idx = len(self.hs.hist_shaped[0]) - 1 self.make_shapes() self.load_net() self.poloTrader()
def __init__(self, ticker_len, target_percent, hd): self.polo = Poloniex(key, secret) self.hist_depth = hd self.target_percent = target_percent self.ticker_len = ticker_len self.end_ts = datetime.now()+timedelta(seconds=(ticker_len*55)) self.hs = HistWorker() self.refresh_data() self.tickers = self.polo.returnTicker() self.bal = self.polo.returnBalances() self.sellCoins() self.set_target() self.inputs = self.hs.hist_shaped.shape[0]*(self.hs.hist_shaped[0].shape[1]) self.outputs = self.hs.hist_shaped.shape[0] self.make_shapes() self.leaf_names = [] for l in range(len(self.in_shapes[0])): self.leaf_names.append('leaf_one_'+str(l)) self.leaf_names.append('leaf_two_'+str(l)) #self.load_net() self.poloTrader()
def __init__(self, hist_depth): self.hs = HistWorker() self.hs.combine_polo_usd_frames() self.hd = hist_depth print(self.hs.currentHists.keys()) self.end_idx = len(self.hs.hist_shaped[0]) self.but_target = .1 self.inputs = self.hs.hist_shaped.shape[0] * ( self.hs.hist_shaped[0].shape[1]) self.outputs = len(self.hs.coin_dict) print(self.inputs, self.outputs) self.epoch_len = 144 #self.node_names = ['x1', 'y1', 'z1', 'x2', 'y2', 'z2', 'weight'] self.leaf_names = [] #num_leafs = 2**(len(self.node_names)-1)//2 self.initial_depth_tree = nDimensionTree([0.0, 0.0, 0.0], 1.0, 0) self.divide_to_depth(self.initial_depth_tree, self.initial_depth_tree.lvl, self.params["initial_depth"]) self.set_substrate() self.set_leaf_names()
def __init__(self, ticker_len, start_amount, histdepth): self.trade_hist = {} self.polo = Poloniex() self.hist_depth = histdepth self.ticker_len = ticker_len self.end_ts = datetime.now()+timedelta(seconds=(ticker_len*89)) self.start_amount = start_amount self.hs = HistWorker() self.refresh_data() self.folio = CryptoFolio(start_amount, list(self.hs.currentHists.keys())) self.inputs = self.hs.hist_shaped.shape[0]*(self.hs.hist_shaped[0].shape[1]) self.outputs = self.hs.hist_shaped.shape[0] self.leaf_names = [] #num_leafs = 2**(len(self.node_names)-1)//2 self.tree = nDimensionGoldenTree((0.0, 0.0, 0.0), 1.0, 1) self.tree.divide_childrens() self.set_substrate() self.set_leaf_names() self.epoch_len = histdepth self.load_net() print(self.hs.coin_dict) self.db.purge() self.poloTrader()
def __init__(self, hist_depth): self.hs = HistWorker() self.hd = hist_depth self.end_idx = len(self.hs.currentHists["XCP"]) self.but_target = .1 self.inputs = self.hs.hist_shaped.shape[0]*(self.hs.hist_shaped[0].shape[1]-1) self.outputs = self.hs.hist_shaped.shape[0] sign = 1 for ix in range(1,self.outputs+1): sign = sign *-1 self.out_shapes.append((sign/ix, .0, 1.0*sign)) for ix2 in range(1,len(self.hs.hist_shaped[0][0])): self.in_shapes.append((-sign/ix, (sign/ix2), 1.0*sign)) self.subStrate = Substrate(self.in_shapes, self.out_shapes) self.epoch_len = 360
def __init__(self, hist_depth): self.hs = HistWorker() self.hd = hist_depth self.end_idx = len(self.hs.currentHists["DASH"]) self.but_target = .1 self.inputs = self.hs.hist_shaped.shape[0]*(self.hs.hist_shaped[0].shape[1]-1) * self.hd self.outputs = self.hs.hist_shaped.shape[0] sign = 1 for ix in range(self.outputs): sign = sign *-1 self.out_shapes.append((sign*ix, 1)) for ix2 in range(len(self.hs.hist_shaped[0][0])-1): for ix3 in range(self.hd): self.in_shapes.append((sign*ix, ((1+ix2)*.1))) self.subStrate = Substrate(self.in_shapes, self.out_shapes) self.epoch_len = 55
class LiveTrader: params = { "initial_depth": 3, "max_depth": 4, "variance_threshold": 0.00013, "band_threshold": 0.00013, "iteration_level": 3, "division_threshold": 0.00013, "max_weight": 8.0, "activation": "tanh" } # Config for CPPN. config = neat.config.Config(neat.genome.DefaultGenome, neat.reproduction.DefaultReproduction, neat.species.DefaultSpeciesSet, neat.stagnation.DefaultStagnation, 'config_trader') def __init__(self, ticker_len, target_percent, hd, base_sym="BTC"): self.base_sym = base_sym self.load_polo_client() self.hd = hd self.target_percent = target_percent self.ticker_len = ticker_len self.end_ts = datetime.now() + timedelta(seconds=(ticker_len * 55)) self.hs = HistWorker() self.refresh_data() self.tickers = self.polo.returnTicker() self.refresh_balances() self.sellCoins() self.set_target() self.inputs = self.hs.hist_shaped.shape[0] * ( self.hs.hist_shaped[0].shape[1]) self.outputs = self.hs.hist_shaped.shape[0] self.end_idx = len(self.hs.hist_shaped[0]) - 1 self.make_shapes() self.load_net() self.poloTrader() def load_polo_client(self): keys = self.get_keys() self.polo = Poloniex(keys[0], keys[1]) def purge_polo_client(self): self.polo = None def load_net(self): champ_file = open("./champ_data/latest_greatest.pkl", 'rb') g = pickle.load(champ_file) #file.close() the_cppn = neat.nn.FeedForwardNetwork.create(g, self.config) self.cppn = the_cppn def refresh_data(self): try: self.hs.pull_polo_usd_live(21) self.hs.combine_live_usd_frames() except Exception as e: print(e) time.sleep(360) self.refresh_data() def refresh_balances(self): try: self.bal = self.polo.returnCompleteBalances() except Exception as e: print(e) time.sleep(360) self.refresh_balances() def get_one_bar_input_2d(self): master_active = [] try: for x in range(0, self.hd): active = [] #print(self.outputs) for y in range(0, self.outputs): sym_data = self.hs.hist_shaped[y][self.end_idx - x] #print(len(sym_data)) active += sym_data.tolist() master_active.append(active) except: print("error getting look back data") self.refresh_data() self.get_one_bar_input_2d() #print(active) return master_active def closeOrders(self): try: orders = self.polo.returnOpenOrders() except Exception as e: print(e) print('error getting open orers') time.sleep(360) self.closeOrders() for o in orders: if orders[o] != []: try: ordnum = orders[o][0]['orderNumber'] self.polo.cancelOrder(ordnum) except Exception as e: print(e) print('error closing') def sellCoins(self): for b in self.tickers: if (b.split("_")[0] == self.base_sym): price = self.get_price(b) price = price - (price * .005) self.sell_coin(b, price) def buy_coin(self, coin, price): amt = self.target / price if (self.bal[self.base_sym]["available"] > self.target): try: self.polo.buy(coin, price, amt) print("buying: ", coin) except Exception as e: print("error buying ", coin) print(e) return def sell_coin(self, coin, price): if (self.base_sym != "BTC"): amt = self.bal[coin.split("_")[1]]["available"] else: amt = self.bal[coin.split("_")[1]]["btcValue"] if (amt * price > .0001): try: self.polo.sell(coin, price, amt) print("selling this shit: ", coin) except Exception as e: print("error selling ", coin) print(e) return def reset_tickers(self): try: self.tickers = self.polo.returnTicker() self.bal = self.polo.returnCompleteBalances() except Exception as e: print(e) time.sleep(360) self.reset_tickers() return def get_keys(self): with open("./godsplan.txt") as f: content = f.readlines() content[0] = content[0][:-1] if (content[1][-1:] == "\n"): content[1] = content[1][:-1] return content def make_shapes(self): sign = 1 self.out_shapes = [] self.in_shapes = [] for ix in range(1, self.outputs + 1): sign = sign * -1 self.out_shapes.append((0.0 - (sign * .005 * ix), 0.0, -1.0)) for ix2 in range(1, (self.inputs // self.outputs) + 1): self.in_shapes.append( (0.0 + (sign * .01 * ix2), 0.0 - (sign * .01 * ix2), 0.0)) self.subStrate = Substrate(self.in_shapes, self.out_shapes) def get_price(self, coin): return self.tickers[coin]['last'] def set_target(self): total = 0 full_bal = self.polo.returnCompleteBalances() for x in full_bal: total += full_bal[x]["btcValue"] if (self.base_sym != "BTC"): total = total * self.get_price(self.base_sym + "_" + "BTC") * self.target_percent print(total) self.target = total def poloTrader(self): self.refresh_balances() end_prices = {} active = self.get_one_bar_input_2d() self.load_net() network = ESNetwork(self.subStrate, self.cppn, self.params, self.hd) net = network.create_phenotype_network_nd('paper_net.png') net.reset() sell_syms = [] buy_syms = [] buy_signals = [] sell_signals = [] self.closeOrders() for n in range(1, self.hd): net.activate(active[self.hd - n]) out = net.activate(active[0]) for x in range(len(out)): sym = self.hs.coin_dict[x] end_prices[sym] = self.get_price(self.base_sym + "_" + sym) if (out[x] > .5): buy_signals.append(out[x]) buy_syms.append(sym) if (out[x] < -.5): sell_signals.append(out[x]) sell_syms.append(sym) #rng = iter(shuffle(rng)) sorted_buys = np.argsort(buy_signals)[::-1] sorted_sells = np.argsort(sell_signals) self.reset_tickers() for x in sorted_sells: sym = sell_syms[x] p = self.get_price(self.base_sym + "_" + sym) price = p - (p * .005) self.sell_coin(self.base_sym + "_" + sym, price) for x in sorted_buys: sym = buy_syms[x] self.target_percent = .1 + out[x] - .45 p = self.get_price(self.base_sym + "_" + sym) price = p * 1.005 self.buy_coin(self.base_sym + "_" + sym, price) if datetime.now() >= self.end_ts: return else: self.purge_polo_client() time.sleep(self.ticker_len) self.load_polo_client() self.refresh_data() self.make_shapes() #self.closeOrders() self.poloTrader()
class PaperTrader: params = { "initial_depth": 3, "max_depth": 4, "variance_threshold": 0.00013, "band_threshold": 0.00013, "iteration_level": 3, "division_threshold": 0.00013, "max_weight": 8.0, "activation": "tanh" } # Config for CPPN. config = neat.config.Config(neat.genome.DefaultGenome, neat.reproduction.DefaultReproduction, neat.species.DefaultSpeciesSet, neat.stagnation.DefaultStagnation, 'config_trader') in_shapes = [] out_shapes = [] def __init__(self, ticker_len, start_amount, histdepth, base_sym): self.trade_hist = {} self.polo = Poloniex() self.hd = histdepth self.ticker_len = ticker_len self.end_ts = datetime.now() + timedelta(seconds=(ticker_len * 24)) self.start_amount = start_amount self.hs = HistWorker() self.hs.combine_live_usd_frames() print(self.hs.currentHists.keys()) self.end_idx = len(self.hs.hist_shaped[0]) - 1 self.but_target = .1 self.inputs = self.hs.hist_shaped.shape[0] * ( self.hs.hist_shaped[0].shape[1]) self.outputs = self.hs.hist_shaped.shape[0] self.folio = CryptoFolio(start_amount, list(self.hs.currentHists.keys()), base_sym) self.base_sym = base_sym sign = 1 for ix in range(1, self.outputs + 1): sign = sign * -1 self.out_shapes.append((0.0 - (sign * .005 * ix), 0.0, -1.0)) for ix2 in range(1, (self.inputs // self.outputs) + 1): self.in_shapes.append( (0.0 + (sign * .01 * ix2), 0.0 - (sign * .01 * ix2), 0.0)) self.subStrate = Substrate(self.in_shapes, self.out_shapes) self.load_net() print(self.hs.coin_dict) self.poloTrader() def refresh_data(self): try: self.hs.pull_polo_usd_live(21) self.hs.combine_live_usd_frames() self.end_idx = len(self.hs.hist_shaped[0]) - 1 except Exception as e: print(e) time.sleep(360) self.refresh_data() return def load_net(self): champ_file = open("./champ_data/latest_greatest.pkl", 'rb') g = pickle.load(champ_file) #file.close() the_cppn = neat.nn.FeedForwardNetwork.create(g, self.config) self.cppn = the_cppn def make_shapes(self): self.in_shapes = [] self.out_shapes = [] sign = 1 for ix in range(1, self.outputs + 1): sign = sign * -1 self.out_shapes.append((0.0 - (sign * .005 * ix), -1.0, -1.0)) for ix2 in range(1, (self.inputs // self.outputs) + 1): self.in_shapes.append( (0.0 + (sign * .01 * ix2), 0.0 - (sign * .01 * ix2), 1.0)) def reset_tickers(self): try: self.tickers = self.polo.returnTicker() except Exception as e: time.sleep(360) self.reset_tickers() return def get_price(self, coin): return self.tickers[coin]['last'] def get_current_balance(self): #self.refresh_data() self.reset_tickers() c_prices = {} for s in self.hs.currentHists.keys(): if s != self.base_sym: c_prices[s] = self.get_price(self.base_sym + "_" + s) return self.folio.get_total_btc_value_no_sell(c_prices) def get_one_bar_input_2d(self): master_active = [] try: for x in range(0, self.hd): active = [] #print(self.outputs) for y in range(0, self.outputs): sym_data = self.hs.hist_shaped[y][self.end_idx - x] #print(len(sym_data)) active += sym_data.tolist() master_active.append(active) except: self.refresh_data() self.get_one_bar_input_2d() #print(active) return master_active def poloTrader(self): try: trade_df = pd.read_json("./live_hist/json_hist.json") except Exception as e: trade_df = pd.DataFrame() end_prices = {} active = self.get_one_bar_input_2d() self.load_net() sub = Substrate(self.in_shapes, self.out_shapes) net = ESNetwork(sub, self.cppn, self.params, self.hd) network = net.create_phenotype_network_nd('paper_net.png') sell_syms = [] buy_syms = [] buy_signals = [] sell_signals = [] for n in range(1, self.hd): network.activate(active[self.hd - n]) out = network.activate(active[0]) self.reset_tickers() for x in range(len(out)): sym = self.hs.coin_dict[x] end_prices[sym] = self.get_price(self.base_sym + "_" + sym) if (out[x] > .5): buy_signals.append(out[x]) buy_syms.append(sym) if (out[x] < -.5): sell_signals.append(out[x]) sell_syms.append(sym) #rng = iter(shuffle(rng)) sorted_buys = np.argsort(buy_signals)[::-1] sorted_sells = np.argsort(sell_signals) for x in sorted_sells: try: sym = sell_syms[x] p = end_prices[sym] print("selling: ", sym) self.folio.sell_coin(sym, p) except Exception as e: print("error placing order") for x in sorted_buys: try: sym = buy_syms[x] p = end_prices[sym] print("buying: ", sym) self.folio.buy_coin(sym, p) except Exception as e: print("error placing order") ''' self.trade_hist["date"] = datetime.now() self.trade_hist["portfoliovalue"] = self.folio.get_total_btc_value_no_sell(end_prices)[0] self.trade_hist["portfolio"] = self.folio.ledger self.trade_hist["percentchange"] = ((self.trade_hist["portfoliovalue"] - self.folio.start)/self.folio.start)*100 trade_df.append(self.trade_hist) trade_df.to_json("./live_hist/json_hist.json") if(self.trade_hist["portfoliovalue"] > self.folio.start *1.1): self.folio.start = self.folio.get_total_btc_value(end_prices)[0] ''' if datetime.now() >= self.end_ts: port_info = self.folio.get_total_btc_value(end_prices) print("total val: ", port_info[0], "btc balance: ", port_info[1]) return else: print(self.get_current_balance()) for t in range(2): p_vals = self.get_current_balance() print("current value: ", p_vals[0], "current holdings: ", p_vals[1]) time.sleep(self.ticker_len / 2) self.refresh_data() self.poloTrader()
class PurpleTrader: #needs to be initialized so as to allow for 62 outputs that return a coordinate # ES-HyperNEAT specific parameters. params = {"initial_depth": 3, "max_depth": 4, "variance_threshold": 0.00013, "band_threshold": 0.00013, "iteration_level": 3, "division_threshold": 0.00013, "max_weight": 3.0, "activation": "tanh"} # Config for CPPN. config = neat.config.Config(neat.genome.DefaultGenome, neat.reproduction.DefaultReproduction, neat.species.DefaultSpeciesSet, neat.stagnation.DefaultStagnation, 'config_trader') start_idx = 0 highest_returns = 0 portfolio_list = [] rand_start = 0 in_shapes = [] out_shapes = [] def __init__(self, hist_depth): self.hs = HistWorker() self.hs.combine_binance_frames_vol_sorted(8) self.hd = hist_depth print(self.hs.currentHists.keys()) self.end_idx = len(self.hs.hist_shaped[0]) self.but_target = .1 self.inputs = self.hs.hist_shaped.shape[0]*(self.hs.hist_shaped[0].shape[1]) self.outputs = len(self.hs.coin_dict) print(self.inputs, self.outputs) self.epoch_len = 144 self.node_names = ['x1', 'y1', 'z1', 'x2', 'y2', 'z2', 'weight'] self.leaf_names = [] num_leafs = 2**(len(self.node_names)-1)//2 self.tree = nDimensionGoldenTree((0.0, 0.0, 0.0), 1.0, 1) self.tree.divide_childrens() self.set_substrate() self.set_leaf_names() def set_leaf_names(self): for l in range(len(self.in_shapes[0])): self.leaf_names.append('leaf_one_'+str(l)) self.leaf_names.append('leaf_two_'+str(l)) #self.leaf_names.append('bias') def set_substrate(self): sign = 1 x_increment = 1.0 / self.outputs y_increment = 1.0 / len(self.hs.hist_shaped[0][0]) for ix in range(self.outputs): self.out_shapes.append((1.0-(ix*x_increment), 0.0, -1.0)) for ix2 in range(self.inputs//self.outputs): if(ix2 >= len(self.tree.cs)-1): treex = ix2 - len(self.tree.cs)-1 else: treex = ix2 center = self.tree.cs[treex] self.in_shapes.append((center.coord[0]+(ix*x_increment), center.coord[1] - (ix2*y_increment), center.coord[2]+.5)) self.subStrate = Substrate(self.in_shapes, self.out_shapes) def set_portfolio_keys(self, folio): for k in self.hs.currentHists.keys(): folio.ledger[k] = 0 def get_one_epoch_input(self,end_idx): master_active = [] for x in range(0, self.hd): active = [] #print(self.outputs) for y in range(0, self.outputs): try: sym_data = self.hs.hist_shaped[y][end_idx-x] #print(len(sym_data)) active += sym_data.tolist() except: print('error') master_active.append(active) #print(active) return master_active def evaluate(self, g, config): # k so here we create a random start index # then we make our nets, and start fitness evaluation rand_start = self.rand_start [cppn] = create_cppn(g, config, self.leaf_names, ['cppn_out']) net = ESNetwork(self.subStrate, cppn, self.params) network = net.create_phenotype_network_nd() portfolio_start = 1.0 key_list = list(self.hs.currentHists.keys()) portfolio = CryptoFolio(portfolio_start, self.hs.coin_dict) end_prices = {} buys = 0 sells = 0 if(len(network.node_evals) > 0): for z in range(rand_start, rand_start+self.epoch_len): active = self.get_one_epoch_input(z) signals = [] network.reset() for n in range(1, self.hd+1): out = network.activate(active[self.hd-n]) for x in range(len(out)): signals.append(out[x]) sorted_shit = np.argsort(signals)[::-1] for x in sorted_shit: sym = self.hs.coin_dict[x] #print(out[x]) #try: if(out[x] < -.5): #print("selling") portfolio.sell_coin(sym, self.hs.currentHists[sym]['close'][z]) #print("bought ", sym) if(out[x] > .5): #print("buying") portfolio.target_amount = .1 + (out[x] * .1) portfolio.buy_coin(sym, self.hs.currentHists[sym]['close'][z]) #print("sold ", sym) #skip the hold case because we just dont buy or sell hehe if(z > self.epoch_len+rand_start-2): end_prices[sym] = self.hs.currentHists[sym]['close'][self.epoch_len+rand_start] result_val = portfolio.get_total_btc_value(end_prices) print(result_val[0], "buys: ", result_val[1], "sells: ", result_val[2]) ft = result_val[0] else: ft = 0.0 return ft def eval_fitness(self, genomes, config): #generate random batch size between 1/8 and 1/5 our whole data set length min_batch_size = (self.hs.hist_full_size-self.hd) // 8 max_batch_size = (self.hs.hist_full_size-self.hd) // 5 self.epoch_len = randint(min_batch_size, max_batch_size) #generate random start here too ? self.rand_start = randint(self.hd, self.hs.hist_full_size - self.epoch_len) runner = neat.ParallelEvaluator(8, self.evaluate) runner.evaluate(genomes, config)
from hist_service import HistWorker hs = HistWorker() hs.pull_polo() #hs.write_binance_training_files()
class PurpleTrader: # ES-HyperNEAT specific parameters. params = { "initial_depth": 1, "max_depth": 3, "variance_threshold": 0.055, "band_threshold": 0.034, "iteration_level": 3, "division_threshold": 0.021, "max_weight": 34.55, "activation": "tanh" } # Config for CPPN. config = neat.config.Config(neat.genome.DefaultGenome, neat.reproduction.DefaultReproduction, neat.species.DefaultSpeciesSet, neat.stagnation.DefaultStagnation, 'config_trader') start_idx = 0 highest_returns = 0 portfolio_list = [] leaf_names = [] def __init__(self, hist_depth, num_gens, gen_count=1): self.hd = hist_depth if gen_count != 1: self.num_gens = num_gens else: self.num_gens = gen_count + num_gens self.gen_count = gen_count self.refresh() def refresh(self): self.in_shapes = [] self.out_shapes = [(0.0, -1.0, -1.0)] self.hs = HistWorker() self.hs.get_binance_train() print(self.hs.currentHists.keys()) self.end_idx = len(self.hs.hist_shaped[0]) self.but_target = .1 print(self.hs.hist_shaped.shape) self.num_syms = self.hs.hist_shaped.shape[0] self.inputs = self.hs.hist_shaped[0].shape[1] + 1 self.outputs = 1 sign = 1 for ix2 in range(1, self.inputs + 1): sign *= -1 self.in_shapes.append( (0.0 + (sign * .01 * ix2), 0.0 - (sign * .01 * ix2), 0.0)) self.substrate = Substrate(self.in_shapes, self.out_shapes) self.set_leaf_names() # informing the substrate def reset_substrate(self, input_row): current_inputs = self.substrate.input_coordinates new_input = [] for ix, t in enumerate(current_inputs): t = list(t) offset = input_row[ix] * .5 t[2] = t[2] + .5 new_input.append(tuple(t)) #self.in_shapes = new_input self.substrate = Substrate(new_input, self.out_shapes) def set_portfolio_keys(self, folio): for k in self.hs.currentHists.keys(): folio.ledger[k] = 0 def set_leaf_names(self): for l in range(len(self.in_shapes[0])): self.leaf_names.append('leaf_one_' + str(l)) self.leaf_names.append('leaf_two_' + str(l)) print(self.leaf_names) def get_one_epoch_input(self, end_idx): master_active = [] for x in range(1, self.hd + 1): active = [] #print(self.outputs) for y in range(0, self.outputs): sym_data = self.hs.hist_shaped[y][end_idx + x] #print(len(sym_data)) active += sym_data.tolist() master_active.append(active) #print(active) return master_active def get_single_symbol_epoch_recurrent(self, end_idx, symbol_idx): master_active = [] for x in range(0, self.hd): try: sym_data = self.hs.hist_shaped[symbol_idx][end_idx - x] #print(len(sym_data)) master_active.append(sym_data.tolist()) except: print('error') return master_active def get_single_symbol_epoch_recurrent_with_position_size( self, end_idx, symbol_idx, current_position): master_active = [] for x in range(0, self.hd): try: sym_data = self.hs.hist_shaped[symbol_idx][end_idx - x] #print(len(sym_data)) sym_data = sym_data.tolist() sym_data.append(current_position) master_active.append(sym_data) except: print('error') return master_active def evaluate_champ(self, builder, rand_start, g, verbose=False): portfolio_start = 1000.0 portfolio = CryptoFolio(portfolio_start, self.hs.coin_dict, "USDT") end_prices = {} phenotypes = {} buys = 0 sells = 0 with open("./trade_hists/kraken/" + str(g.key) + "_hist.txt", "w") as ft: ft.write('date,current_balance \n') for z_minus in range(0, self.epoch_len - 1): for x in range(self.num_syms): z = rand_start - z_minus sym = self.hs.coin_dict[x] z = rand_start - z_minus pos_size = portfolio.ledger[sym] active = self.get_single_symbol_epoch_recurrent_with_position_size( z, x, pos_size) if (z_minus == 0 or (z_minus + 1) % 8 == 0): self.reset_substrate(active[0]) builder.substrate = self.substrate phenotypes[sym] = builder.create_phenotype_network_nd() network = phenotypes[sym] network.reset() for n in range(1, self.hd + 1): network.activate([active[self.hd - n]]) out = network.activate([active[0]]) end_prices[sym] = self.hs.currentHists[sym]['close'][z] if (out[0] < .5 or (z_minus + 1) % 16 == 0): portfolio.sell_coin( sym, self.hs.currentHists[sym]['close'][z]) else: portfolio.buy_coin( sym, self.hs.currentHists[sym]['close'][z]) ft.write(str(self.hs.currentHists[sym]['time'][z]) + ",") ft.write( str( portfolio.get_total_btc_value_no_sell(end_prices) [0]) + " \n") result_val = portfolio.get_total_btc_value(end_prices) print("genome id ", g.key, " : ") print(result_val[0], "buys: ", result_val[1], "sells: ", result_val[2]) ft = result_val[0] return ft def evaluate(self, builder, rand_start, g, verbose=False): portfolio_start = 1000.0 portfolio = CryptoFolio(portfolio_start, self.hs.coin_dict, "USD") end_prices = {} phenotypes = {} buys = 0 sells = 0 last_val = portfolio_start ft = 0.0 for z_minus in range(0, self.epoch_len): for x in range(self.num_syms): sym = self.hs.coin_dict[x] z = rand_start - z_minus pos_size = portfolio.ledger[sym] active = self.get_single_symbol_epoch_recurrent_with_position_size( z, x, pos_size) #print(active) if (z_minus == 0 or (z_minus + 1) % 8 == 0): self.reset_substrate(active[0]) builder.substrate = self.substrate phenotypes[sym] = builder.create_phenotype_network_nd() network = phenotypes[sym] network.reset() for n in range(1, self.hd + 1): network.activate([active[self.hd - n]]) out = network.activate([active[0]]) if (out[0] < -0.5): portfolio.sell_coin(sym, self.hs.currentHists[sym]['close'][z]) #print("bought ", sym) elif (out[0] > 0.5): did_buy = portfolio.buy_coin( sym, self.hs.currentHists[sym]['close'][z]) #rng = iter(shuffle(rng)) end_prices[sym] = self.hs.currentHists[sym]['close'][z] bal_now = portfolio.get_total_btc_value_no_sell(end_prices)[0] ft += bal_now - last_val last_val = bal_now result_val = portfolio.get_total_btc_value(end_prices) print(g.key, " : ") print(result_val[0], "buys: ", result_val[1], "sells: ", result_val[2]) if result_val[0] == portfolio_start: ft = -.2 return ft def eval_fitness(self, genomes, config): self.epoch_len = 16 r_start = randint(0 + self.epoch_len, self.hs.hist_full_size - self.hd) best_g_fit = 0.0 champ_counter = self.gen_count % 10 #img_count = 0 for idx, g in genomes: [cppn] = create_cppn(g, config, self.leaf_names, ["cppn_out"]) net_builder = ESNetwork(self.substrate, cppn, self.params) #cppn = neat.nn.FeedForwardNetwork.create(g, config) #network = ESNetwork(self.subStrate, cppn, self.params, self.hd) #net = network.create_phenotype_network_nd() train_ft = self.evaluate(net_builder, r_start, g) g.fitness = train_ft if (g.fitness > best_g_fit): best_g_fit = g.fitness with open( "./champ_data/kraken/latest_greatest" + str(champ_counter) + ".pkl", 'wb') as output: pickle.dump(g, output) #img_count += 1 ''' if(champ_counter == 0): self.refresh() self.compare_champs() ''' self.gen_count += 1 return def write_hists(self): self.epoch_len = self.hs.hist_full_size - (self.hd + 1) r_start = self.epoch_len champ_current = open("./champ_data/binance/latest_greatest.pkl", 'rb') g = pickle.load(champ_current) champ_current.close() [cppn] = create_cppn(g, self.config, self.leaf_names, ["cppn_out"]) net_builder = ESNetwork(self.substrate, cppn, self.params) champ_fit = self.evaluate_champ(net_builder, r_start, g) for f in os.listdir("./champ_data/binance"): if (f != "lastest_greatest.pkl"): champ_file = open("./champ_data/binance/" + f, 'rb') g = pickle.load(champ_file) champ_file.close() [cppn] = create_cppn(g, self.config, self.leaf_names, ["cppn_out"]) net_builder = ESNetwork(self.substrate, cppn, self.params) g.fitness = self.evaluate_champ(net_builder, r_start, g) if (g.fitness > champ_fit): with open("./champ_data/binance/latest_greatest.pkl", 'wb') as output: pickle.dump(g, output) return def validate_fitness(self): config = self.config genomes = neat.Checkpointer.restore_checkpoint( "./pkl_pops/pop-checkpoint-27").population self.epoch_len = 233 r_start = self.hs.hist_full_size - self.epoch_len - 1 best_g_fit = 1.0 for idx in genomes: g = genomes[idx] cppn = neat.nn.FeedForwardNetwork.create(g, config) network = ESNetwork(self.subStrate, cppn, self.params, self.hd) net = network.create_phenotype_network_nd() g.fitness = self.evaluate(net, network, r_start, g) if (g.fitness > best_g_fit): best_g_fit = g.fitness with open('./champ_data/binance/latest_greatest.pkl', 'wb') as output: pickle.dump(g, output) return # Create the population and run the XOR task by providing the above fitness function. def run_pop(self, checkpoint=""): if (checkpoint == ""): pop = neat.population.Population(self.config) else: pop = neat.Checkpointer.restore_checkpoint( "./pkl_pops/binance/pop-checkpoint-" + checkpoint) checkpoints = neat.Checkpointer( generation_interval=2, time_interval_seconds=None, filename_prefix='./pkl_pops/binance/pop-checkpoint-') stats = neat.statistics.StatisticsReporter() pop.add_reporter(stats) pop.add_reporter(checkpoints) pop.add_reporter(neat.reporting.StdOutReporter(True)) print(self.num_gens) winner = pop.run(self.eval_fitness, self.num_gens) return winner, stats # If run as script. def run_training(self, checkpoint=""): #print(task.trial_run()) if checkpoint == "": winner = self.run_pop()[0] else: winner = self.run_pop(checkpoint)[0] print('\nBest genome:\n{!s}'.format(winner)) checkpoint_string = str(self.num_gens - 1) self.num_gens += self.num_gens self.run_training(checkpoint_string) def run_validation(self): self.validate_fitness()
class LiveTrader: params = {"initial_depth": 4, "max_depth": 4, "variance_threshold": 0.00013, "band_threshold": 0.00013, "iteration_level": 3, "division_threshold": 0.00013, "max_weight": 8.0, "activation": "tanh"} # Config for CPPN. config = neat.config.Config(neat.genome.DefaultGenome, neat.reproduction.DefaultReproduction, neat.species.DefaultSpeciesSet, neat.stagnation.DefaultStagnation, 'config_trader') def __init__(self, ticker_len, target_percent, hd): self.polo = Poloniex(key, secret) self.hist_depth = hd self.target_percent = target_percent self.ticker_len = ticker_len self.end_ts = datetime.now()+timedelta(seconds=(ticker_len*55)) self.hs = HistWorker() self.refresh_data() self.tickers = self.polo.returnTicker() self.bal = self.polo.returnBalances() self.sellCoins() self.set_target() self.inputs = self.hs.hist_shaped.shape[0]*(self.hs.hist_shaped[0].shape[1]) self.outputs = self.hs.hist_shaped.shape[0] self.make_shapes() self.leaf_names = [] self.db = tinydb.database("live_hist/memories.json") for l in range(len(self.in_shapes[0])): self.leaf_names.append('leaf_one_'+str(l)) self.leaf_names.append('leaf_two_'+str(l)) #self.load_net() self.poloTrader() def load_net(self): #file = open("./champ_gens/thot-checkpoint-13",'rb') g = neat.Checkpointer.restore_checkpoint("./champ_gens/thot-checkpoint-25") best_fit = 0.0 for gx in g.population: if g.population[gx].fitness != None: if g.population[gx].fitness > best_fit: bestg = g.population[gx] g = bestg #file.close() [the_cppn] = create_cppn(g, self.config, self.leaf_names, ['cppn_out']) self.cppn = the_cppn def refresh_data(self): self.hs.pull_polo_live(20) self.hs.combine_live_frames(self.hist_depth) def make_shapes(self): self.in_shapes = [] self.out_shapes = [] sign = 1 for ix in range(1,self.outputs+1): sign = sign *-1 self.out_shapes.append((0.0-(sign*.005*ix), -1.0, -1.0)) for ix2 in range(1,(self.inputs//self.outputs)+1): self.in_shapes.append((0.0+(sign*.01*ix2), 0.0-(sign*.01*ix2), 1.0)) def get_one_bar_input_2d(self,end_idx=10): master_active = [] for x in range(0, self.hist_depth): active = [] #print(self.outputs) for y in range(0, self.outputs): sym_data = self.hs.hist_shaped[y][self.hist_depth-x] #print(len(sym_data)) active += sym_data.tolist() master_active.append(active) #print(active) return master_active def closeOrders(self): try: orders = self.polo.returnOpenOrders() except: print('error getting open orers') time.sleep(360) self.closeOrder() for o in orders: if orders[o] != []: try: ordnum = orders[o][0]['orderNumber'] self.polo.cancelOrder(ordnum) except: print('error closing') def sellCoins(self): for b in self.tickers: if(b[:3] == "BTC"): price = self.get_price(b) price = price - (price * .005) self.sell_coin(b, price) def buy_coin(self, coin, price): amt = self.target / price if(self.bal['BTC'] > self.target): self.polo.buy(coin, price, amt, fillOrKill=1) print("buying: ", coin) return def sell_coin(self, coin, price): amt = self.bal[coin[4:]] if (amt*price > .0001): try: self.polo.sell(coin, price, amt,fillOrKill=1) print("selling this shit: ", coin) except: print('error selling', coin) return def reset_tickers(self): try: self.tickers = self.polo.returnTicker() self.bal = self.polo.returnBalances() except: time.sleep(360) self.reset_tickers() return def get_price(self, coin): return self.tickers[coin]['last'] def set_target(self): total = 0 full_bal = self.polo.returnCompleteBalances() for x in full_bal: total += full_bal[x]["btcValue"] self.target = total*self.target_percent def poloTrader(self): end_prices = {} active = self.get_one_bar_input_2d() self.load_net() sub = Substrate(self.in_shapes, self.out_shapes) network = ESNetwork(sub, self.cppn, self.params) net = network.create_phenotype_network_nd('paper_net.png') net.reset() for n in range(1, self.hist_depth+1): out = net.activate(active[self.hist_depth-n]) #print(len(out)) rng = len(out) #rng = iter(shuffle(rng)) self.reset_tickers() for x in np.random.permutation(rng): sym = self.hs.coin_dict[x] #print(out[x]) try: if(out[x] < -.5): print("selling: ", sym) p = self.get_price('BTC_'+sym) price = p -(p*.01) self.sell_coin('BTC_'+sym, price) elif(out[x] > .5): print("buying: ", sym) self.target_percent = .1 + out[x] - .45 p = self.get_price('BTC_'+sym) price = p*1.01 self.buy_coin('BTC_'+sym, price) except: print('error', sym) #skip the hold case because we just dont buy or sell hehe if datetime.now() >= self.end_ts: return else: time.sleep(self.ticker_len) self.refresh_data() #self.closeOrders() self.poloTrader()
class PaperTrader: params = {"initial_depth": 3, "max_depth": 4, "variance_threshold": 0.00013, "band_threshold": 0.00013, "iteration_level": 3, "division_threshold": 0.00013, "max_weight": 3.0, "activation": "tanh"} in_shapes = [] out_shapes = [] db = TinyDB('./live_hist/memories.json') # Config for CPPN. config = neat.config.Config(neat.genome.DefaultGenome, neat.reproduction.DefaultReproduction, neat.species.DefaultSpeciesSet, neat.stagnation.DefaultStagnation, 'config_trader') def __init__(self, ticker_len, start_amount, histdepth): self.trade_hist = {} self.polo = Poloniex() self.hist_depth = histdepth self.ticker_len = ticker_len self.end_ts = datetime.now()+timedelta(seconds=(ticker_len*89)) self.start_amount = start_amount self.hs = HistWorker() self.refresh_data() self.folio = CryptoFolio(start_amount, list(self.hs.currentHists.keys())) self.inputs = self.hs.hist_shaped.shape[0]*(self.hs.hist_shaped[0].shape[1]) self.outputs = self.hs.hist_shaped.shape[0] self.leaf_names = [] #num_leafs = 2**(len(self.node_names)-1)//2 self.tree = nDimensionGoldenTree((0.0, 0.0, 0.0), 1.0, 1) self.tree.divide_childrens() self.set_substrate() self.set_leaf_names() self.epoch_len = histdepth self.load_net() print(self.hs.coin_dict) self.db.purge() self.poloTrader() def set_leaf_names(self): for l in range(len(self.in_shapes[0])): self.leaf_names.append('leaf_one_'+str(l)) self.leaf_names.append('leaf_two_'+str(l)) #self.leaf_names.append('bias') def set_substrate(self): sign = 1 x_increment = 1.0 / self.outputs y_increment = 1.0 / len(self.hs.hist_shaped[0][0]) for ix in range(self.outputs): self.out_shapes.append((1.0-(ix*x_increment), 0.0, -1.0)) for ix2 in range(self.inputs//self.outputs): if(ix2 >= len(self.tree.cs)-1): treex = ix2 - len(self.tree.cs)-1 else: treex = ix2 center = self.tree.cs[treex] self.in_shapes.append((center.coord[0]+(ix*x_increment), center.coord[1] - (ix2*y_increment), center.coord[2]+.5)) self.subStrate = Substrate(self.in_shapes, self.out_shapes) def set_portfolio_keys(self, folio): for k in self.hs.currentHists.keys(): folio.ledger[k] = 0 def refresh_data(self): self.hs.pull_polo_live(21) self.hs.combine_live_frames(89) def load_net(self): #file = open("./champ_gens/thot-checkpoint-13",'rb') g = neat.Checkpointer.restore_checkpoint("./binance_champs_2/tradegod-checkpoint-48") ''' best_fit = 0.0 for gx in g.population: if g.population[gx].fitness != None: if g.population[gx].fitness > best_fit: bestg = g.population[gx] g = bestg ''' #file.close() g = g.population[5910] [the_cppn] = create_cppn(g, self.config, self.leaf_names, ['cppn_out']) self.cppn = the_cppn def make_shapes(self): self.in_shapes = [] self.out_shapes = [] sign = 1 for ix in range(1,self.outputs+1): sign = sign *-1 self.out_shapes.append((0.0-(sign*.005*ix), -1.0, -1.0)) for ix2 in range(1,(self.inputs//self.outputs)+1): self.in_shapes.append((0.0+(sign*.01*ix2), 0.0-(sign*.01*ix2), 1.0)) def reset_tickers(self): try: self.tickers = self.polo.returnTicker() except: time.sleep(360) self.reset_tickers() return def get_price(self, coin): return self.tickers[coin]['last'] def get_current_balance(self): #self.refresh_data() c_prices = {} for s in self.hs.currentHists.keys(): if s != 'BTC': c_prices[s] = self.hs.currentHists[s]['close'].iloc[-1] return self.folio.get_total_btc_value_no_sell(c_prices) def get_one_bar_input_2d(self,end_idx=10): master_active = [] for x in range(0, self.hist_depth): active = [] #print(self.outputs) for y in range(0, self.outputs): sym_data = self.hs.hist_shaped[y][self.hist_depth-x] #print(len(sym_data)) active += sym_data.tolist() master_active.append(active) #print(active) return master_active def poloTrader(self): try: trade_df = pd.read_json("./live_hist/json_hist.json") except: trade_df = pd.DataFrame() end_prices = {} active = self.get_one_bar_input_2d() self.load_net() sub = Substrate(self.in_shapes, self.out_shapes) network = ESNetwork(sub, self.cppn, self.params) net = network.create_phenotype_network_nd() net.reset() signals = [] net.reset() for n in range(1, self.hist_depth+1): out = net.activate(active[self.hist_depth-n]) for x in range(len(out)): signals.append(out[x]) #rng = iter(shuffle(rng)) sorted_shit = np.argsort(signals)[::-1] self.reset_tickers() sym = "" for x in sorted_shit: sym = self.hs.coin_dict[x] #print(out[x]) try: if(out[x] < -.5): p = self.get_price('BTC_'+sym) print("selling: ", sym) self.folio.sell_coin(sym, p) elif(out[x] > .5): p = self.get_price('BTC_'+sym) print("buying: ", sym) self.folio.buy_coin(sym, p) except: print("error buying or selling") #skip the hold case because we just dont buy or sell hehe end_prices[sym] = self.hs.currentHists[sym]["close"].iloc[-1] #self.trade_hist["date"] = self.hs.currentHists[sym]["date"].iloc[-1] self.trade_hist["date"] = time.time() self.trade_hist["portfoliovalue"] = self.folio.get_total_btc_value_no_sell(end_prices)[0] self.trade_hist["portfolio"] = self.folio.ledger self.trade_hist["percentchange"] = ((self.trade_hist["portfoliovalue"] - self.folio.start)/self.folio.start)*100 print(self.trade_hist) print(self.folio.ledger) self.db.insert(self.trade_hist) self.trade_hist = {} ''' if(self.trade_hist["portfoliovalue"] > self.folio.start *1.1): self.folio.start = self.folio.get_total_btc_value(end_prices)[0] ''' if datetime.now() >= self.end_ts: port_info = self.folio.get_total_btc_value(end_prices) print("total val: ", port_info[0], "btc balance: ", port_info[1]) return else: print(self.get_current_balance()) for t in range(2): time.sleep(self.ticker_len/2) #print(self.folio.ledger) self.refresh_data self.poloTrader()
class PurpleTrader: #needs to be initialized so as to allow for 62 outputs that return a coordinate # ES-HyperNEAT specific parameters. params = { "initial_depth": 1, "max_depth": 3, "variance_threshold": 0.3, "band_threshold": 0.3, "iteration_level": 3, "division_threshold": 0.5, "max_weight": 8.0, "activation": "tanh" } # Config for CPPN. config = neat.config.Config(neat.genome.DefaultGenome, neat.reproduction.DefaultReproduction, neat.species.DefaultSpeciesSet, neat.stagnation.DefaultStagnation, 'config_trader') start_idx = 0 highest_returns = 0 portfolio_list = [] def __init__(self, hist_depth, num_gens, gen_count=1): self.hd = hist_depth if gen_count != 1: self.num_gens = num_gens else: self.num_gens = gen_count + num_gens self.gen_count = gen_count self.refresh() def refresh(self): self.in_shapes = [] self.out_shapes = [] self.hs = HistWorker() self.hs.get_kraken_train() print(self.hs.currentHists.keys()) self.end_idx = len(self.hs.hist_shaped[0]) self.but_target = .1 self.inputs = self.hs.hist_shaped.shape[0] * ( self.hs.hist_shaped[0].shape[1]) self.outputs = self.hs.hist_shaped.shape[0] sign = 1 for ix in range(1, self.outputs + 1): sign = sign * -1 self.out_shapes.append((0.0 - (sign * .005 * ix), 0.0, -1.0)) for ix2 in range(1, (self.inputs // self.outputs) + 1): self.in_shapes.append( (0.0 + (sign * .01 * ix2), 0.0 - (sign * .01 * ix2), 0.0)) self.subStrate = Substrate(self.in_shapes, self.out_shapes) #self.leaf_names.append('bias') def set_portfolio_keys(self, folio): for k in self.hs.currentHists.keys(): folio.ledger[k] = 0 def get_one_epoch_input(self, end_idx): master_active = [] for x in range(1, self.hd + 1): active = [] #print(self.outputs) for y in range(0, self.outputs): sym_data = self.hs.hist_shaped[y][end_idx + x] #print(len(sym_data)) active += sym_data.tolist() master_active.append(active) #print(active) return master_active def evaluate_champ(self, network, es, rand_start, g, verbose=False): portfolio_start = 10000.0 portfolio = CryptoFolio(portfolio_start, self.hs.coin_dict, "USDT") end_prices = {} buys = 0 sells = 0 with open("./trade_hists/kraken/" + str(g.key) + "_hist.txt", "w") as ft: ft.write('date,current_balance \n') for z_minus in range(0, self.epoch_len - 1): z = rand_start - z_minus active = self.get_one_epoch_input(z) buy_signals = [] buy_syms = [] sell_syms = [] sell_signals = [] network.reset() for n in range(1, self.hd + 1): network.activate(active[self.hd - n]) out = network.activate(active[0]) for x in range(len(out)): sym = self.hs.coin_dict[x] end_prices[sym] = self.hs.currentHists[sym]['close'][z] if (out[x] > .5): buy_signals.append(out[x]) buy_syms.append(sym) if (out[x] < -.5): sell_signals.append(out[x]) sell_syms.append(sym) #rng = iter(shuffle(rng)) sorted_buys = np.argsort(buy_signals)[::-1] sorted_sells = np.argsort(sell_signals) #print(len(sorted_shit), len(key_list)) for x in sorted_sells: sym = sell_syms[x] portfolio.sell_coin(sym, self.hs.currentHists[sym]['close'][z]) for x in sorted_buys: sym = buy_syms[x] portfolio.buy_coin(sym, self.hs.currentHists[sym]['close'][z]) ft.write(str(self.hs.currentHists[sym]['time'][z]) + ",") ft.write( str(portfolio.get_total_btc_value_no_sell(end_prices)[0]) + " \n") result_val = portfolio.get_total_btc_value(end_prices) print("genome id ", g.key, " : ") print(result_val[0], "buys: ", result_val[1], "sells: ", result_val[2]) if result_val[1] == 0: ft = .7 else: ft = result_val[0] return ft def evaluate(self, network, es, rand_start, g, verbose=False): portfolio_start = 1.0 portfolio = CryptoFolio(portfolio_start, self.hs.coin_dict, "USDT") end_prices = {} buys = 0 sells = 0 last_val = portfolio_start ft = 0.0 for z_minus in range(0, self.epoch_len): #TODO add comments to clarify all the #shit im doing here z = rand_start - z_minus active = self.get_one_epoch_input(z) buy_signals = [] buy_syms = [] sell_syms = [] sell_signals = [] network.reset() for n in range(1, self.hd + 1): network.activate(active[self.hd - n]) out = network.activate(active[0]) for x in range(len(out)): sym = self.hs.coin_dict[x] end_prices[sym] = self.hs.currentHists[sym]['close'][z] if (out[x] > .5): buy_signals.append(out[x]) buy_syms.append(sym) if (out[x] < -.5): sell_signals.append(out[x]) sell_syms.append(sym) #rng = iter(shuffle(rng)) sorted_buys = np.argsort(buy_signals)[::-1] sorted_sells = np.argsort(sell_signals) #print(len(sorted_shit), len(key_list)) for x in sorted_sells: sym = sell_syms[x] portfolio.sell_coin(sym, self.hs.currentHists[sym]['close'][z]) for x in sorted_buys: sym = buy_syms[x] portfolio.buy_coin(sym, self.hs.currentHists[sym]['close'][z]) bal_now = portfolio.get_total_btc_value_no_sell(end_prices)[0] ft += bal_now - last_val last_val = bal_now result_val = portfolio.get_total_btc_value(end_prices) print(g.key, " : ") print(result_val[0], "buys: ", result_val[1], "sells: ", result_val[2]) if result_val[0] == portfolio_start: ft = ft * .9 return ft def solve(self, network): return self.evaluate(network) >= self.highest_returns def trial_run(self): r_start = 0 file = open("es_trade_god_cppn_3d.pkl", 'rb') [cppn] = pickle.load(file) network = ESNetwork(self.subStrate, cppn, self.params) net = network.create_phenotype_network_nd() fitness = self.evaluate(net, network, r_start) return fitness def eval_fitness(self, genome, config): self.epoch_len = randint(42, 42 * 4) r_start = randint(0 + self.epoch_len, self.hs.hist_full_size - self.hd) #r_start_2 = self.hs.hist_full_size - self.epoch_len-1 best_g_fit = 0.0 champ_counter = self.gen_count % 10 #img_count = 0 for idx, g in genomes: cppn = neat.nn.FeedForwardNetwork.create(g, config) network = ESNetwork(self.subStrate, cppn, self.params, self.hd) net = network.create_phenotype_network_nd() train_ft = self.evaluate(net, network, r_start, g) #validate_ft = self.evaluate(net, network, r_start_2, g) g.fitness = train_ft if (g.fitness > best_g_fit): best_g_fit = g.fitness with open( "./champ_data/kraken/latest_greatest" + str(champ_counter) + ".pkl", 'wb') as output: pickle.dump(g, output) #img_count += 1 if (champ_counter == 0): self.refresh() self.compare_champs() self.gen_count += 1 return def compare_champs(self): self.epoch_len = self.hs.hist_full_size - (self.hd + 1) r_start = self.epoch_len champ_current = open("./champ_data/kraken/latest_greatest.pkl", 'rb') g = pickle.load(champ_current) champ_current.close() cppn = neat.nn.FeedForwardNetwork.create(g, self.config) network = ESNetwork(self.subStrate, cppn, self.params, self.hd) net = network.create_phenotype_network_nd() champ_fit = self.evaluate(net, network, r_start, g) for f in os.listdir("./champ_data/kraken"): if (f != "lastest_greatest.pkl"): champ_file = open("./champ_data/kraken/" + f, 'rb') g = pickle.load(champ_file) champ_file.close() cppn = neat.nn.FeedForwardNetwork.create(g, self.config) network = ESNetwork(self.subStrate, cppn, self.params, self.hd) net = network.create_phenotype_network_nd() g.fitness = self.evaluate_champ(net, network, r_start, g) if (g.fitness > champ_fit): with open("./champ_data/kraken/latest_greatest.pkl", 'wb') as output: pickle.dump(g, output) return def validate_fitness(self): config = self.config genomes = neat.Checkpointer.restore_checkpoint( "./pkl_pops/pop-checkpoint-27").population self.epoch_len = 233 r_start = self.hs.hist_full_size - self.epoch_len - 1 best_g_fit = 1.0 for idx in genomes: g = genomes[idx] cppn = neat.nn.FeedForwardNetwork.create(g, config) network = ESNetwork(self.subStrate, cppn, self.params, self.hd) net = network.create_phenotype_network_nd() g.fitness = self.evaluate(net, network, r_start, g) if (g.fitness > best_g_fit): best_g_fit = g.fitness with open('./champ_data/kraken/latest_greatest.pkl', 'wb') as output: pickle.dump(g, output) return # Create the population and run the XOR task by providing the above fitness function. def run_pop(self, checkpoint=""): if (checkpoint == ""): pop = neat.population.Population(self.config) else: pop = neat.Checkpointer.restore_checkpoint( "./pkl_pops/kraken/pop-checkpoint-" + checkpoint) checkpoints = neat.Checkpointer( generation_interval=2, time_interval_seconds=None, filename_prefix='./pkl_pops/kraken/pop-checkpoint-') stats = neat.statistics.StatisticsReporter() pop.add_reporter(stats) pop.add_reporter(checkpoints) pop.add_reporter(neat.reporting.StdOutReporter(True)) pe = neat.ThreadedEvaluator(4, self.eval_fitness) winner = pop.run(pe.evaluate, self.num_gens) return winner, stats # If run as script. def run_training(self, checkpoint=""): #print(task.trial_run()) if checkpoint == "": winner = self.run_pop()[0] else: winner = self.run_pop(checkpoint)[0] print('\nBest genome:\n{!s}'.format(winner)) checkpoint_string = str(self.num_gens - 1) self.num_gens += self.num_gens self.run_training(checkpoint_string) def run_validation(self): self.validate_fitness()
class PurpleTrader: #needs to be initialized so as to allow for 62 outputs that return a coordinate # ES-HyperNEAT specific parameters. params = { "initial_depth": 3, "max_depth": 4, "variance_threshold": 0.00013, "band_threshold": 0.00013, "iteration_level": 3, "division_threshold": 0.00013, "max_weight": 5.0, "activation": "tanh" } # Config for CPPN. config = neat.config.Config(neat.genome.DefaultGenome, neat.reproduction.DefaultReproduction, neat.species.DefaultSpeciesSet, neat.stagnation.DefaultStagnation, 'config_trader') start_idx = 0 highest_returns = 0 portfolio_list = [] in_shapes = [] out_shapes = [] def __init__(self, hist_depth): self.hs = HistWorker() self.hs.combine_binance_frames() self.hd = hist_depth print(self.hs.currentHists.keys()) self.end_idx = len(self.hs.hist_shaped[0]) self.but_target = .1 self.inputs = self.hs.hist_shaped.shape[0] * ( self.hs.hist_shaped[0].shape[1]) self.outputs = self.hs.hist_shaped.shape[0] sign = 1 for ix in range(1, self.outputs + 1): sign = sign * -1 self.out_shapes.append((0.0 - (sign * .005 * ix), -1.0, -1.0)) for ix2 in range(1, (self.inputs // self.outputs) + 1): self.in_shapes.append( (0.0 + (sign * .01 * ix2), 0.0 - (sign * .01 * ix2), 1.0)) self.subStrate = Substrate(self.in_shapes, self.out_shapes) self.epoch_len = 144 #self.node_names = ['x1', 'y1', 'z1', 'x2', 'y2', 'z2', 'weight'] self.leaf_names = [] #num_leafs = 2**(len(self.node_names)-1)//2 for l in range(len(self.in_shapes[0])): self.leaf_names.append('leaf_one_' + str(l)) self.leaf_names.append('leaf_two_' + str(l)) #self.leaf_names.append('bias') def set_portfolio_keys(self, folio): for k in self.hs.currentHists.keys(): folio.ledger[k] = 0 def get_one_epoch_input(self, end_idx): master_active = [] for x in range(0, self.hd): active = [] #print(self.outputs) for y in range(0, self.outputs): try: sym_data = self.hs.hist_shaped[y][end_idx - x] #print(len(sym_data)) active += sym_data.tolist() except: print('error') master_active.append(active) #print(active) return master_active def evaluate(self, network, es, rand_start, g, verbose=False): portfolio_start = .05 portfolio = CryptoFolio(portfolio_start, self.hs.coin_dict) end_prices = {} buys = 0 sells = 0 if (len(g.connections) > 0.0): for z in range(rand_start, rand_start + self.epoch_len): active = self.get_one_epoch_input(z) network.reset() for n in range(1, self.hd + 1): out = network.activate(active[self.hd - n]) for x in range(len(out)): signals.append(out[x]) #rng = iter(shuffle(rng)) sorted_shit = np.argsort(signals)[::-1] #print(sorted_shit, len(sorted_shit)) #print(len(sorted_shit), len(key_list)) for x in sorted_shit: sym = self.hs.coin_dict[x] #print(out[x]) #try: if (out[x] < -.5): #print("selling") portfolio.sell_coin( sym, self.hs.currentHists[sym]['close'][z]) #print("bought ", sym) if (out[x] > .5): #print("buying") portfolio.target_amount = .1 + (out[x] * .1) portfolio.buy_coin( sym, self.hs.currentHists[sym]['close'][z]) #print("sold ", sym) #skip the hold case because we just dont buy or sell hehe if (z == self.epoch_len + rand_start): end_prices[sym] = self.hs.currentHists[sym]['close'][ self.epoch_len + rand_start] result_val = portfolio.get_total_btc_value(end_prices) print(result_val[0], "buys: ", result_val[1], "sells: ", result_val[2]) ft = result_val[0] else: ft = 0.0 return ft def solve(self, network): return self.evaluate(network) >= self.highest_returns def trial_run(self): r_start = 0 file = open("es_trade_god_cppn_3d.pkl", 'rb') [cppn] = pickle.load(file) network = ESNetwork(self.subStrate, cppn, self.params) net = network.create_phenotype_network_nd() fitness = self.evaluate(net, network, r_start) return fitness def eval_fitness(self, genomes, config): self.epoch_len = randint(21, 255) r_start = randint(0 + self.hd, self.hs.hist_full_size - self.epoch_len) for idx, g in genomes: [cppn] = create_cppn(g, config, self.leaf_names, ['cppn_out']) network = ESNetwork(self.subStrate, cppn, self.params) net = network.create_phenotype_network_nd('training_net.png') g.fitness = self.evaluate(net, network, r_start, g) return
class PurpleTrader: #needs to be initialized so as to allow for 62 outputs that return a coordinate # ES-HyperNEAT specific parameters. params = {"initial_depth": 3, "max_depth": 4, "variance_threshold": 0.00013, "band_threshold": 0.00013, "iteration_level": 3, "division_threshold": 0.00013, "max_weight": 5.0, "activation": "tanh"} # Config for CPPN. config = neat.config.Config(neat.genome.DefaultGenome, neat.reproduction.DefaultReproduction, neat.species.DefaultSpeciesSet, neat.stagnation.DefaultStagnation, 'config_trader') start_idx = 0 highest_returns = 0 portfolio_list = [] in_shapes = [] out_shapes = [] def __init__(self, hist_depth): self.hs = HistWorker() self.hs.combine_frames() self.hd = hist_depth print(self.hs.currentHists.keys()) self.end_idx = len(self.hs.currentHists["ETH"]) self.but_target = .1 self.inputs = self.hs.hist_shaped[0].shape[1] self.outputs = 1 sign = 1 for ix in range(1,self.inputs+1): sign = sign *-1 self.in_shapes.append((0.0-(sign*.005*ix), -1.0, 0.0+(sign*.005*ix))) self.out_shapes.append((0.0, 1.0, 0.0)) self.subStrate = Substrate(self.in_shapes, self.out_shapes) self.epoch_len = 144 #self.node_names = ['x1', 'y1', 'z1', 'x2', 'y2', 'z2', 'weight'] self.leaf_names = [] #num_leafs = 2**(len(self.node_names)-1)//2 for l in range(len(self.in_shapes[0])): self.leaf_names.append('leaf_one_'+str(l)) self.leaf_names.append('leaf_two_'+str(l)) #self.leaf_names.append('bias') def set_portfolio_keys(self, folio): for k in self.hs.currentHists.keys(): folio.ledger[k] = 0 def get_one_epoch_input(self,end_idx): master_active = [] for x in range(0, self.hd): active = [] #print(self.outputs) for y in range(0, self.outputs): try: sym_data = self.hs.hist_shaped[y][end_idx-x] #print(len(sym_data)) active += sym_data.tolist() except: print('error') master_active.append(active) #print(active) return master_active def get_single_symbol_epoch(self, end_idx, symbol_idx): master_active = [] for x in range(0, self.hd): try: sym_data = self.hs.hist_shaped[symbol_idx][end_idx-x] #print(len(sym_data)) master_active.append(sym_data.tolist()) except: print('error') return master_active def load_net(self, fname): f = open(fname,'rb') g = pickle.load(f) f.close() [the_cppn] = create_cppn(g, self.config, self.leaf_names, ['cppn_out']) self.cppn = the_cppn def run_champs(self): genomes = os.listdir(os.path.join(os.path.dirname(__file__), 'champs_d2_single')) fitness_data = {} best_fitness = 0.0 for g_ix in range(len(genomes)): genome = self.load_net('./champs_d2_single/'+genomes[g_ix]) start = self.hs.hist_full_size - self.epoch_len network = ESNetwork(self.subStrate, self.cppn, self.params) net = network.create_phenotype_network_nd('./champs_visualized2/genome_'+str(g_ix)) fitness = self.evaluate(net, network, start, g_ix, genomes[g_ix]) if fitness > best_fitness: best_genome = genome def evaluate(self, network, es, rand_start, g, p_name): portfolio_start = 1.0 portfolio = CryptoFolio(portfolio_start, self.hs.coin_dict) end_prices = {} buys = 0 sells = 0 th = [] with open('./champs_hist2/trade_hist'+p_name + '.txt', 'w') as ft: ft.write('date,symbol,type,amnt,price,current_balance \n') for z in range(self.hd, self.hs.hist_full_size -1): for x in np.random.permutation(self.outputs): sym = self.hs.coin_dict[x] active = self.get_single_symbol_epoch(z, x) network.reset() for n in range(1, self.hd+1): out = network.activate(active[self.hd-n]) end_prices[sym] = self.hs.currentHists[sym]['close'][self.hs.hist_full_size-1] #rng = iter(shuffle(rng)) #print(out[x]) #try: if(out[0] < -.5): #print("selling") did_sell = portfolio.sell_coin(sym, self.hs.currentHists[sym]['close'][z]) if did_sell: ft.write(str(self.hs.currentHists[sym]['date'][z]) + ",") ft.write(sym +",") ft.write('sell,') ft.write(str(portfolio.ledger[sym])+",") ft.write(str(self.hs.currentHists[sym]['close'][z])+",") ft.write(str(portfolio.get_total_btc_value_no_sell(end_prices)[0])+ " \n") #print("bought ", sym) elif(out[0] > .5): did_buy = portfolio.buy_coin(sym, self.hs.currentHists[sym]['close'][z]) if did_buy: ft.write(str(self.hs.currentHists[sym]['date'][z]) + ",") ft.write(sym +",") ft.write('buy,') ft.write(str(portfolio.target_amount)+",") ft.write(str(self.hs.currentHists[sym]['close'][z])+",") ft.write(str(portfolio.get_total_btc_value_no_sell(end_prices)[0])+ " \n") #print("sold ", sym) #skip the hold case because we just dont buy or sell heh result_val = portfolio.get_total_btc_value(end_prices) print(result_val[0], "buys: ", result_val[1], "sells: ", result_val[2], p_name) ft = result_val[0] return ft def solve(self, network): return self.evaluate(network) >= self.highest_returns def report_back(self, portfolio, prices): print(portfolio.get_total_btc_value(prices)) def trial_run(self): r_start = 0 file = open("es_trade_god_cppn_3d.pkl",'rb') [cppn] = pickle.load(file) network = ESNetwork(self.subStrate, cppn, self.params) net = network.create_phenotype_network_nd() fitness = self.evaluate(net, network, r_start) return fitness def eval_fitness(self, genomes, config): r_start = randint(0+self.hd, self.hs.hist_full_size - self.epoch_len) fitter = genomes[0] fitter_val = 0.0 for idx, g in genomes: [cppn] = create_cppn(g, config, self.leaf_names, ['cppn_out']) network = ESNetwork(self.subStrate, cppn, self.params) net = network.create_phenotype_network_nd('current_net.png') g.fitness = self.evaluate(net, network, r_start) if(g.fitness > fitter_val): fitter = g fitter_val = g.fitness with open('./champs/perpetual_champion_'+str(fitter.key)+'.pkl', 'wb') as output: pickle.dump(fitter, output) print("latest_saved")
from hist_service import HistWorker hs = HistWorker() hs.combine_polo_frames_vol_sorted(3) print(next(iter(hs.currentHists.values())).head()) hs.currentHists = {} hs.combine_binance_frames_vol_sorted(3) print(next(iter(hs.currentHists.values())).head())
class PaperTrader: params = {"initial_depth": 3, "max_depth": 4, "variance_threshold": 0.00013, "band_threshold": 0.0000013, "iteration_level": 3, "division_threshold": 0.00013, "max_weight": 8.0, "activation": "tanh"} # Config for CPPN. config = neat.config.Config(neat.genome.DefaultGenome, neat.reproduction.DefaultReproduction, neat.species.DefaultSpeciesSet, neat.stagnation.DefaultStagnation, 'config_trader') def __init__(self, ticker_len, start_amount, histdepth): self.in_shapes = [] self.out_shapes = [] self.trade_hist = {} self.polo = Poloniex() self.hist_depth = histdepth self.ticker_len = ticker_len self.end_ts = datetime.now()+timedelta(seconds=(ticker_len*24)) self.start_amount = start_amount self.hs = HistWorker() self.refresh_data() self.inputs = self.hs.hist_shaped.shape[0]*(self.hs.hist_shaped[0].shape[1]) self.outputs = self.hs.hist_shaped.shape[0] #self.make_shapes() self.folio = CryptoFolio(start_amount, list(self.hs.currentHists.keys())) self.leaf_names = [] for ix in range(1,self.inputs+1): sign = sign *-1 self.in_shapes.append((0.0-(sign*.005*ix), -1.0, 0.0+(sign*.005*ix))) self.out_shapes.append((0.0, 1.0, 0.0)) self.subStrate = Substrate(self.in_shapes, self.out_shapes) for l in range(len(self.in_shapes[0])): self.leaf_names.append('leaf_one_'+str(l)) self.leaf_names.append('leaf_two_'+str(l)) self.load_net() print(self.hs.coin_dict) self.poloTrader() def refresh_data(self): self.hs.pull_polo_live(12*30) self.hs.combine_live_frames(12*30) def load_net(self): #file = open("./champ_gens/thot-checkpoint-13",'rb') g = neat.Checkpointer.restore_checkpoint("./champ_gens/thot-checkpoint-25") best_fit = 0.0 for gx in g.population: if g.population[gx].fitness != None: if g.population[gx].fitness > best_fit: bestg = g.population[gx] g = bestg #file.close() [the_cppn] = create_cppn(g, self.config, self.leaf_names, ['cppn_out']) self.cppn = the_cppn def make_shapes(self): self.in_shapes = [] self.out_shapes = [] sign = 1 for ix in range(1,self.outputs+1): sign = sign *-1 self.out_shapes.append((0.0-(sign*.005*ix), -1.0, -1.0)) for ix2 in range(1,(self.inputs//self.outputs)+1): self.in_shapes.append((0.0+(sign*.01*ix2), 0.0-(sign*.01*ix2), 1.0)) def reset_tickers(self): try: self.tickers = self.polo.returnTicker() except: time.sleep(360) self.reset_tickers() return def get_price(self, coin): return self.tickers[coin]['last'] def get_current_balance(self): #self.refresh_data() c_prices = {} for s in self.hs.currentHists.keys(): if s != 'BTC': c_prices[s] = self.hs.currentHists[s]['close'].iloc[-1] return self.folio.get_total_btc_value_no_sell(c_prices) def get_one_bar_input_2d(self,end_idx=10): master_active = [] for x in range(0, self.hist_depth): active = [] #print(self.outputs) for y in range(0, self.outputs): sym_data = self.hs.hist_shaped[y][self.hist_depth-x] #print(len(sym_data)) active += sym_data.tolist() master_active.append(active) #print(active) return master_active def poloTrader(self): try: trade_df = pd.read_json("./live_hist/json_hist.json") except: trade_df = pd.DataFrame() end_prices = {} active = self.get_one_bar_input_2d() self.load_net() sub = Substrate(self.in_shapes, self.out_shapes) network = ESNetwork(sub, self.cppn, self.params) net = network.create_phenotype_network_nd('paper_net.png') net.reset() for n in range(1, self.hist_depth+1): out = net.activate(active[self.hist_depth-n]) #print(len(out)) rng = len(out) #rng = iter(shuffle(rng)) self.reset_tickers() for x in np.random.permutation(rng): sym = self.hs.coin_dict[x] #print(out[x]) try: if(out[x] < -.5): p = self.get_price('BTC_'+sym) print("selling: ", sym) self.folio.sell_coin(sym, p) elif(out[x] > .5): p = self.get_price('BTC_'+sym) print("buying: ", sym) self.folio.buy_coin(sym, p) else: except: print("error buying or selling") #skip the hold case because we just dont buy or sell hehe end_prices[sym] = self.hs.currentHists[sym]["close"].iloc[-1] self.trade_hist["date"] = datetime.now() self.trade_hist["portfoliovalue"] = self.folio.get_total_btc_value_no_sell(end_prices)[0] self.trade_hist["portfolio"] = self.folio.ledger self.trade_hist["percentchange"] = ((self.trade_hist["portfoliovalue"] - self.folio.start)/sefl.folio.start)*100 trade_df.append(self.trade_hist) trade_df.to_json("./live_hist/json_hist.json") ''' if(self.trade_hist["portfoliovalue"] > self.folio.start *1.1): self.folio.start = self.folio.get_total_btc_value(end_prices)[0] ''' if datetime.now() >= self.end_ts: port_info = self.folio.get_total_btc_value(end_prices) print("total val: ", port_info[0], "btc balance: ", port_info[1]) return else: print(self.get_current_balance()) for t in range(2): self.refresh_data p_vals = self.get_current_balance() print("current value: ", p_vals[0], "current btc holdings: ", p_vals[1]) time.sleep(self.ticker_len/2) #print(self.folio.ledger) self.poloTrader()
class PurpleTrader: #needs to be initialized so as to allow for 62 outputs that return a coordinate # ES-HyperNEAT specific parameters. params = { "initial_depth": 3, "max_depth": 4, "variance_threshold": 0.00013, "band_threshold": 0.00013, "iteration_level": 3, "division_threshold": 0.00013, "max_weight": 8.0, "activation": "tanh" } # Config for CPPN. config = neat.config.Config(neat.genome.DefaultGenome, neat.reproduction.DefaultReproduction, neat.species.DefaultSpeciesSet, neat.stagnation.DefaultStagnation, './configs/config_trader') start_idx = 0 highest_returns = 0 portfolio_list = [] in_shapes = [] out_shapes = [] def __init__(self, hist_depth): self.hs = HistWorker() self.hs.combine_binance_frames_vol_sorted(3) self.hd = hist_depth print(self.hs.currentHists.keys()) self.end_idx = len(self.hs.hist_shaped[0]) self.but_target = .1 self.inputs = self.hs.hist_shaped.shape[0] * ( self.hs.hist_shaped[0].shape[1]) self.outputs = self.hs.hist_shaped.shape[0] self.leaf_names = [] #num_leafs = 2**(len(self.node_names)-1)//2 #self.tree = nDimensionGoldenTree((0.0, 0.0, 0.0), 1.0, 1) #self.tree.divide_childrens() self.set_substrate() self.set_leaf_names() self.epoch_len = hist_depth def set_leaf_names(self): for l in range(len(self.in_shapes[0])): self.leaf_names.append('leaf_one_' + str(l)) self.leaf_names.append('leaf_two_' + str(l)) #self.leaf_names.append('bias') def set_substrate(self): sign = 1 x_increment = 1.0 / self.outputs y_increment = 1.0 / len(self.hs.hist_shaped[0][0]) for ix in range(self.outputs): self.out_shapes.append((1.0 - (ix * x_increment), 0.0, -1.0)) for ix2 in range(len(self.hs.hist_shaped[0][0])): self.in_shapes.append((-1.0 + (ix * x_increment), 1.0 - (ix2 * y_increment), 1.0)) self.subStrate = Substrate(self.in_shapes, self.out_shapes) def set_portfolio_keys(self, folio): for k in self.hs.currentHists.keys(): folio.ledger[k] = 0 def get_one_epoch_input(self, end_idx): master_active = [] for x in range(0, self.hd): active = [] #print(self.outputs) for y in range(0, self.outputs): try: sym_data = self.hs.hist_shaped[y][end_idx - x] #print(len(sym_data)) active += sym_data.tolist() except: print('error') master_active.append(active) #print(active) return master_active def load_net(self, fname): f = open(fname, 'rb') g = neat.Checkpointer().restore_checkpoint(fname) bestFit = 0.0 for gx in g.population: if g.population[gx].fitness != None: if g.population[gx].fitness > bestFit: bestg = g.population[gx] g = bestg f.close() [the_cppn] = create_cppn(g, self.config, self.leaf_names, ['cppn_out']) self.cppn = the_cppn def load_net_easy(self, g): [the_cppn] = create_cppn(g, self.config, self.leaf_names, ['cppn_out']) self.cppn = the_cppn def run_champs(self): genomes = neat.Checkpointer.restore_checkpoint( "./binance_champs_2/tradegod-checkpoint-70").population fitness_data = {} best_fitness = 0.0 best_ix = "" count = 0 for g_ix in genomes: self.load_net_easy(genomes[g_ix]) start = self.hs.hist_full_size - self.epoch_len network = ESNetwork(self.subStrate, self.cppn, self.params) net = network.create_phenotype_network_nd( './champs_visualizedd3/genome_' + str(g_ix)) fitness = self.evaluate(net, network, start, genomes[g_ix], g_ix) if (fitness > best_fitness): best_fitness = fitness best_ix = genomes[g_ix] count += 1 with open( './binance_champs_2/perpetual_champion_' + str(g_ix) + '.pkl', 'wb') as output: pickle.dump(best_ix, output) def run_champ(self): genome = neat.Checkpointer.restore_checkpoint( "./binance_champs_2/tradegod-checkpoint-48").population[5808] self.load_net_easy(genome) start = self.hs.hist_full_size - self.epoch_len network = ESNetwork(self.subStrate, self.cppn, self.params) net = network.create_phenotype_network_nd( './champs_visualizedd3/genome_' + str(5808)) fitness = self.evaluate(net, network, start, genome, 5808) with open( './binance_champs_2/perpetual_champion_' + str(5808) + '.pkl', 'wb') as output: pickle.dump(genome, output) def evaluate(self, network, es, rand_start, g, p_name): portfolio_start = 1.0 portfolio = CryptoFolio(portfolio_start, list(self.hs.currentHists.keys())) end_prices = {} port_ref = portfolio_start with open('./champs_histd3/port_hist' + str(p_name) + '.txt', 'w') as pt: pt.write('date,current_balance \n') with open('./champs_histd3/trade_hist' + str(p_name) + '.txt', 'w') as ft: ft.write('date,symbol,type,amnt,price,current_balance \n') for z in range(34, self.hs.hist_full_size - 1): active = self.get_one_epoch_input(z) signals = [] network.reset() for n in range(1, self.hd + 1): out = network.activate(active[self.hd - n]) for x in range(len(out)): signals.append(out[x]) sym2 = list(self.hs.currentHists.keys())[x] end_prices[sym2] = self.hs.currentHists[sym2]['close'][ z] sorted_shit = np.argsort(signals)[::-1] rebalance = portfolio_start #rng = iter(shuffle(rng)) sym = "" for x in sorted_shit: sym = list(self.hs.currentHists.keys())[x] #print(out[x]) #try: if (out[x] < -.5): #print("selling") did_sell = portfolio.sell_coin( sym, self.hs.currentHists[sym]['close'][z]) if did_sell: ft.write( str(self.hs.currentHists[sym]['date'][z]) + ",") ft.write(sym + ",") ft.write('sell,') ft.write(str(portfolio.ledger[sym]) + ",") ft.write( str(self.hs.currentHists[sym]['close'][z]) + ",") ft.write( str( portfolio.get_total_btc_value_no_sell( end_prices)[0]) + " \n") #print("bought ", sym) elif (out[x] > .5): did_buy = portfolio.buy_coin( sym, self.hs.currentHists[sym]['close'][z]) if did_buy: portfolio.target_amount = .1 + (out[x] * .1) ft.write( str(self.hs.currentHists[sym]['date'][z]) + ",") ft.write(sym + ",") ft.write('buy,') ft.write(str(portfolio.target_amount) + ",") ft.write( str(self.hs.currentHists[sym]['close'][z]) + ",") ft.write( str( portfolio.get_total_btc_value_no_sell( end_prices)[0]) + " \n") #else: pt.write(str(self.hs.currentHists[sym]['date'][z]) + ",") #pt.write(str(self.hs.currentHists[sym]['close'][z])+",") pt.write( str( portfolio.get_total_btc_value_no_sell(end_prices) [0]) + " \n") #print("sold ", sym) new_ref = portfolio.get_total_btc_value_no_sell( end_prices)[0] ''' if(new_ref > 1.10 * port_ref or new_ref < .9*port_ref): port_ref = portfolio.get_total_btc_value_no_sell(end_prices)[0] portfolio.start = port_ref #skip the hold case because we just dont buy or sell heh ''' result_val = portfolio.get_total_btc_value(end_prices) print(result_val[0], "buys: ", result_val[1], "sells: ", result_val[2], p_name) fit = result_val[0] return fit def solve(self, network): return self.evaluate(network) >= self.highest_returns def report_back(self, portfolio, prices): print(portfolio.get_total_btc_value(prices)) def trial_run(self): r_start = 0 file = open("es_trade_god_cppn_3d.pkl", 'rb') [cppn] = pickle.load(file) network = ESNetwork(self.subStrate, cppn, self.params) net = network.create_phenotype_network_nd() fitness = self.evaluate(net, network, r_start) return fitness def eval_fitness(self, genomes, config): r_start = randint(0 + self.hd, self.hs.hist_full_size - self.epoch_len) fitter = genomes[0] fitter_val = 0.0 for idx, g in genomes: [cppn] = create_cppn(g, config, self.leaf_names, ['cppn_out']) network = ESNetwork(self.subStrate, cppn, self.params) net = network.create_phenotype_network_nd('current_net.png') g.fitness = self.evaluate(net, network, r_start) if (g.fitness > fitter_val): fitter = g fitter_val = g.fitness with open( './binance_champs_2/perpetual_champion_' + str(fitter.key) + '.pkl', 'wb') as output: pickle.dump(fitter, output) print("latest_saved")
def __init__(self): self.hs = HistWorker() self.end_idx = len(self.hs.currentHists["DASH"]) self.but_target = .1 self.inputs = self.hs.hist_shaped.shape[0]*self.hs.hist_shaped[0].shape[1] self.outputs = self.hs.hist_shaped.shape[0] # times by three for buy | sell | hodl(pass)