def __init__(self, ticker_len, start_amount, histdepth, base_sym):
self.trade_hist = {}
self.polo = Poloniex()
self.hd = histdepth
self.ticker_len = ticker_len
self.end_ts = datetime.now() + timedelta(seconds=(ticker_len * 24))
self.start_amount = start_amount
self.hs = HistWorker()
self.hs.combine_live_usd_frames()
print(self.hs.currentHists.keys())
self.end_idx = len(self.hs.hist_shaped[0]) - 1
self.but_target = .1
self.inputs = self.hs.hist_shaped.shape[0] * (
self.hs.hist_shaped[0].shape[1])
self.outputs = self.hs.hist_shaped.shape[0]
self.folio = CryptoFolio(start_amount,
list(self.hs.currentHists.keys()), base_sym)
self.base_sym = base_sym
sign = 1
for ix in range(1, self.outputs + 1):
sign = sign * -1
self.out_shapes.append((0.0 - (sign * .005 * ix), 0.0, -1.0))
for ix2 in range(1, (self.inputs // self.outputs) + 1):
self.in_shapes.append(
(0.0 + (sign * .01 * ix2), 0.0 - (sign * .01 * ix2), 0.0))
self.subStrate = Substrate(self.in_shapes, self.out_shapes)
self.load_net()
print(self.hs.coin_dict)
self.poloTrader()
def __init__(self, ticker_len, start_amount, histdepth):
self.in_shapes = []
self.out_shapes = []
self.trade_hist = {}
self.polo = Poloniex()
self.hist_depth = histdepth
self.ticker_len = ticker_len
self.end_ts = datetime.now()+timedelta(seconds=(ticker_len*24))
self.start_amount = start_amount
self.hs = HistWorker()
self.refresh_data()
self.inputs = self.hs.hist_shaped.shape[0]*(self.hs.hist_shaped[0].shape[1])
self.outputs = self.hs.hist_shaped.shape[0]
#self.make_shapes()
self.folio = CryptoFolio(start_amount, list(self.hs.currentHists.keys()))
self.leaf_names = []
for ix in range(1,self.inputs+1):
sign = sign *-1
self.in_shapes.append((0.0-(sign*.005*ix), -1.0, 0.0+(sign*.005*ix)))
self.out_shapes.append((0.0, 1.0, 0.0))
self.subStrate = Substrate(self.in_shapes, self.out_shapes)
for l in range(len(self.in_shapes[0])):
self.leaf_names.append('leaf_one_'+str(l))
self.leaf_names.append('leaf_two_'+str(l))
self.load_net()
print(self.hs.coin_dict)
self.poloTrader()
def __init__(self, hist_depth):
self.hs = HistWorker()
self.hs.combine_binance_frames()
self.hd = hist_depth
print(self.hs.currentHists.keys())
self.end_idx = len(self.hs.hist_shaped[0])
self.but_target = .1
self.inputs = self.hs.hist_shaped.shape[0] * (
self.hs.hist_shaped[0].shape[1])
self.outputs = self.hs.hist_shaped.shape[0]
sign = 1
for ix in range(1, self.outputs + 1):
sign = sign * -1
self.out_shapes.append((0.0 - (sign * .005 * ix), -1.0, -1.0))
for ix2 in range(1, (self.inputs // self.outputs) + 1):
self.in_shapes.append(
(0.0 + (sign * .01 * ix2), 0.0 - (sign * .01 * ix2), 1.0))
self.subStrate = Substrate(self.in_shapes, self.out_shapes)
self.epoch_len = 144
#self.node_names = ['x1', 'y1', 'z1', 'x2', 'y2', 'z2', 'weight']
self.leaf_names = []
#num_leafs = 2**(len(self.node_names)-1)//2
for l in range(len(self.in_shapes[0])):
self.leaf_names.append('leaf_one_' + str(l))
self.leaf_names.append('leaf_two_' + str(l))
def __init__(self, hist_depth):
self.hs = HistWorker()
self.hs.combine_binance_frames_vol_sorted(3)
self.hd = hist_depth
print(self.hs.currentHists.keys())
self.end_idx = len(self.hs.hist_shaped[0])
self.but_target = .1
self.inputs = self.hs.hist_shaped.shape[0] * (
self.hs.hist_shaped[0].shape[1])
self.outputs = self.hs.hist_shaped.shape[0]
self.leaf_names = []
#num_leafs = 2**(len(self.node_names)-1)//2
#self.tree = nDimensionGoldenTree((0.0, 0.0, 0.0), 1.0, 1)
#self.tree.divide_childrens()
self.set_substrate()
self.set_leaf_names()
self.epoch_len = hist_depth
def __init__(self, hist_depth):
self.hs = HistWorker()
self.hs.combine_binance_frames_vol_sorted(21)
self.hd = hist_depth
print(self.hs.currentHists.keys())
self.end_idx = len(self.hs.hist_shaped[0])
self.but_target = .1
self.inputs = self.hs.hist_shaped.shape[0]*(self.hs.hist_shaped[0].shape[1])
self.outputs = len(self.hs.coin_dict)
print(self.inputs, self.outputs)
self.epoch_len = 144
#self.node_names = ['x1', 'y1', 'z1', 'x2', 'y2', 'z2', 'weight']
self.leaf_names = []
#num_leafs = 2**(len(self.node_names)-1)//2
self.tree = nDimensionTree((0.0, 0.0, 0.0), 1.0, 1)
self.tree.divide_childrens()
self.set_substrate()
self.set_leaf_names()
def refresh(self):
self.in_shapes = []
self.out_shapes = [(0.0, -1.0, -1.0)]
self.hs = HistWorker()
self.hs.get_binance_train()
print(self.hs.currentHists.keys())
self.end_idx = len(self.hs.hist_shaped[0])
self.but_target = .1
print(self.hs.hist_shaped.shape)
self.num_syms = self.hs.hist_shaped.shape[0]
self.inputs = self.hs.hist_shaped[0].shape[1] + 1
self.outputs = 1
sign = 1
for ix2 in range(1,self.inputs+1):
sign *= -1
self.in_shapes.append((0.0+(sign*.01*ix2), 0.0-(sign*.01*ix2), 0.0))
self.substrate = Substrate(self.in_shapes, self.out_shapes)
self.set_leaf_names()
def refresh(self):
self.in_shapes = []
self.out_shapes = []
self.hs = HistWorker()
self.hs.get_kraken_train()
print(self.hs.currentHists.keys())
self.end_idx = len(self.hs.hist_shaped[0])
self.but_target = .1
self.inputs = self.hs.hist_shaped.shape[0] * (
self.hs.hist_shaped[0].shape[1])
self.outputs = self.hs.hist_shaped.shape[0]
sign = 1
for ix in range(1, self.outputs + 1):
sign = sign * -1
self.out_shapes.append((0.0 - (sign * .005 * ix), 0.0, -1.0))
for ix2 in range(1, (self.inputs // self.outputs) + 1):
self.in_shapes.append(
(0.0 + (sign * .01 * ix2), 0.0 - (sign * .01 * ix2), 0.0))
self.subStrate = Substrate(self.in_shapes, self.out_shapes)
def refresh(self):
self.in_shapes = []
self.out_shapes = [(0.0, -1.0, -1.0)]
self.hs = HistWorker()
self.hs.get_kraken_train()
print(self.hs.currentHists.keys())
self.end_idx = len(self.hs.hist_shaped[0])
self.but_target = .1
print(self.hs.hist_shaped.shape)
self.num_syms = self.hs.hist_shaped.shape[0]
# add one to number of symbols to account for current position size
# input we pass for context
self.inputs = self.hs.hist_shaped[0].shape[1] + 1
self.outputs = 1
sign = 1
for ix2 in range(1,self.inputs+1):
sign *= -1
self.in_shapes.append((0.0+(sign*.01*ix2), 0.0-(sign*.01*ix2), 0.0))
self.substrate = Substrate(self.in_shapes, self.out_shapes)
self.set_leaf_names()
def refresh(self, reload_data=False):
print("refreshing")
self.in_shapes = []
self.out_shapes = []
self.hs = HistWorker()
if(reload_data != False):
self.hs.pull_robinhood_train_data()
self.hs.get_robinhood_train()
print(self.hs.currentHists.keys())
self.end_idx = len(self.hs.hist_shaped[0])
self.but_target = 1.0
self.inputs = self.hs.hist_shaped.shape[0]*(self.hs.hist_shaped[0].shape[1])
self.outputs = self.hs.hist_shaped.shape[0]
sign = 1
for ix in range(1,self.outputs+1):
sign = sign *-1
self.out_shapes.append((0.0-(sign*.005*ix), 0.0, -1.0))
for ix2 in range(1,(self.inputs//self.outputs)+1):
self.in_shapes.append((0.0+(sign*.01*ix2), 0.0-(sign*.01*ix2), 0.0))
self.subStrate = Substrate(self.in_shapes, self.out_shapes)
def __init__(self, ticker_len, target_percent, hd, base_sym="BTC"):
self.base_sym = base_sym
self.load_polo_client()
self.hd = hd
self.target_percent = target_percent
self.ticker_len = ticker_len
self.end_ts = datetime.now() + timedelta(seconds=(ticker_len * 55))
self.hs = HistWorker()
self.refresh_data()
self.tickers = self.polo.returnTicker()
self.refresh_balances()
self.sellCoins()
self.set_target()
self.inputs = self.hs.hist_shaped.shape[0] * (
self.hs.hist_shaped[0].shape[1])
self.outputs = self.hs.hist_shaped.shape[0]
self.end_idx = len(self.hs.hist_shaped[0]) - 1
self.make_shapes()
self.load_net()
self.poloTrader()
def __init__(self, ticker_len, target_percent, hd):
self.polo = Poloniex(key, secret)
self.hist_depth = hd
self.target_percent = target_percent
self.ticker_len = ticker_len
self.end_ts = datetime.now()+timedelta(seconds=(ticker_len*55))
self.hs = HistWorker()
self.refresh_data()
self.tickers = self.polo.returnTicker()
self.bal = self.polo.returnBalances()
self.sellCoins()
self.set_target()
self.inputs = self.hs.hist_shaped.shape[0]*(self.hs.hist_shaped[0].shape[1])
self.outputs = self.hs.hist_shaped.shape[0]
self.make_shapes()
self.leaf_names = []
for l in range(len(self.in_shapes[0])):
self.leaf_names.append('leaf_one_'+str(l))
self.leaf_names.append('leaf_two_'+str(l))
#self.load_net()
self.poloTrader()
def __init__(self, hist_depth):
self.hs = HistWorker()
self.hs.combine_polo_usd_frames()
self.hd = hist_depth
print(self.hs.currentHists.keys())
self.end_idx = len(self.hs.hist_shaped[0])
self.but_target = .1
self.inputs = self.hs.hist_shaped.shape[0] * (
self.hs.hist_shaped[0].shape[1])
self.outputs = len(self.hs.coin_dict)
print(self.inputs, self.outputs)
self.epoch_len = 144
#self.node_names = ['x1', 'y1', 'z1', 'x2', 'y2', 'z2', 'weight']
self.leaf_names = []
#num_leafs = 2**(len(self.node_names)-1)//2
self.initial_depth_tree = nDimensionTree([0.0, 0.0, 0.0], 1.0, 0)
self.divide_to_depth(self.initial_depth_tree,
self.initial_depth_tree.lvl,
self.params["initial_depth"])
self.set_substrate()
self.set_leaf_names()
def __init__(self, ticker_len, start_amount, histdepth):
self.trade_hist = {}
self.polo = Poloniex()
self.hist_depth = histdepth
self.ticker_len = ticker_len
self.end_ts = datetime.now()+timedelta(seconds=(ticker_len*89))
self.start_amount = start_amount
self.hs = HistWorker()
self.refresh_data()
self.folio = CryptoFolio(start_amount, list(self.hs.currentHists.keys()))
self.inputs = self.hs.hist_shaped.shape[0]*(self.hs.hist_shaped[0].shape[1])
self.outputs = self.hs.hist_shaped.shape[0]
self.leaf_names = []
#num_leafs = 2**(len(self.node_names)-1)//2
self.tree = nDimensionGoldenTree((0.0, 0.0, 0.0), 1.0, 1)
self.tree.divide_childrens()
self.set_substrate()
self.set_leaf_names()
self.epoch_len = histdepth
self.load_net()
print(self.hs.coin_dict)
self.db.purge()
self.poloTrader()
def __init__(self, hist_depth):
self.hs = HistWorker()
self.hd = hist_depth
self.end_idx = len(self.hs.currentHists["XCP"])
self.but_target = .1
self.inputs = self.hs.hist_shaped.shape[0]*(self.hs.hist_shaped[0].shape[1]-1)
self.outputs = self.hs.hist_shaped.shape[0]
sign = 1
for ix in range(1,self.outputs+1):
sign = sign *-1
self.out_shapes.append((sign/ix, .0, 1.0*sign))
for ix2 in range(1,len(self.hs.hist_shaped[0][0])):
self.in_shapes.append((-sign/ix, (sign/ix2), 1.0*sign))
self.subStrate = Substrate(self.in_shapes, self.out_shapes)
self.epoch_len = 360
def __init__(self, hist_depth):
self.hs = HistWorker()
self.hd = hist_depth
self.end_idx = len(self.hs.currentHists["DASH"])
self.but_target = .1
self.inputs = self.hs.hist_shaped.shape[0]*(self.hs.hist_shaped[0].shape[1]-1) * self.hd
self.outputs = self.hs.hist_shaped.shape[0]
sign = 1
for ix in range(self.outputs):
sign = sign *-1
self.out_shapes.append((sign*ix, 1))
for ix2 in range(len(self.hs.hist_shaped[0][0])-1):
for ix3 in range(self.hd):
self.in_shapes.append((sign*ix, ((1+ix2)*.1)))
self.subStrate = Substrate(self.in_shapes, self.out_shapes)
self.epoch_len = 55
class LiveTrader:
params = {
"initial_depth": 3,
"max_depth": 4,
"variance_threshold": 0.00013,
"band_threshold": 0.00013,
"iteration_level": 3,
"division_threshold": 0.00013,
"max_weight": 8.0,
"activation": "tanh"
}
# Config for CPPN.
config = neat.config.Config(neat.genome.DefaultGenome,
neat.reproduction.DefaultReproduction,
neat.species.DefaultSpeciesSet,
neat.stagnation.DefaultStagnation,
'config_trader')
def __init__(self, ticker_len, target_percent, hd, base_sym="BTC"):
self.base_sym = base_sym
self.load_polo_client()
self.hd = hd
self.target_percent = target_percent
self.ticker_len = ticker_len
self.end_ts = datetime.now() + timedelta(seconds=(ticker_len * 55))
self.hs = HistWorker()
self.refresh_data()
self.tickers = self.polo.returnTicker()
self.refresh_balances()
self.sellCoins()
self.set_target()
self.inputs = self.hs.hist_shaped.shape[0] * (
self.hs.hist_shaped[0].shape[1])
self.outputs = self.hs.hist_shaped.shape[0]
self.end_idx = len(self.hs.hist_shaped[0]) - 1
self.make_shapes()
self.load_net()
self.poloTrader()
def load_polo_client(self):
keys = self.get_keys()
self.polo = Poloniex(keys[0], keys[1])
def purge_polo_client(self):
self.polo = None
def load_net(self):
champ_file = open("./champ_data/latest_greatest.pkl", 'rb')
g = pickle.load(champ_file)
#file.close()
the_cppn = neat.nn.FeedForwardNetwork.create(g, self.config)
self.cppn = the_cppn
def refresh_data(self):
try:
self.hs.pull_polo_usd_live(21)
self.hs.combine_live_usd_frames()
except Exception as e:
print(e)
time.sleep(360)
self.refresh_data()
def refresh_balances(self):
try:
self.bal = self.polo.returnCompleteBalances()
except Exception as e:
print(e)
time.sleep(360)
self.refresh_balances()
def get_one_bar_input_2d(self):
master_active = []
try:
for x in range(0, self.hd):
active = []
#print(self.outputs)
for y in range(0, self.outputs):
sym_data = self.hs.hist_shaped[y][self.end_idx - x]
#print(len(sym_data))
active += sym_data.tolist()
master_active.append(active)
except:
print("error getting look back data")
self.refresh_data()
self.get_one_bar_input_2d()
#print(active)
return master_active
def closeOrders(self):
try:
orders = self.polo.returnOpenOrders()
except Exception as e:
print(e)
print('error getting open orers')
time.sleep(360)
self.closeOrders()
for o in orders:
if orders[o] != []:
try:
ordnum = orders[o][0]['orderNumber']
self.polo.cancelOrder(ordnum)
except Exception as e:
print(e)
print('error closing')
def sellCoins(self):
for b in self.tickers:
if (b.split("_")[0] == self.base_sym):
price = self.get_price(b)
price = price - (price * .005)
self.sell_coin(b, price)
def buy_coin(self, coin, price):
amt = self.target / price
if (self.bal[self.base_sym]["available"] > self.target):
try:
self.polo.buy(coin, price, amt)
print("buying: ", coin)
except Exception as e:
print("error buying ", coin)
print(e)
return
def sell_coin(self, coin, price):
if (self.base_sym != "BTC"):
amt = self.bal[coin.split("_")[1]]["available"]
else:
amt = self.bal[coin.split("_")[1]]["btcValue"]
if (amt * price > .0001):
try:
self.polo.sell(coin, price, amt)
print("selling this shit: ", coin)
except Exception as e:
print("error selling ", coin)
print(e)
return
def reset_tickers(self):
try:
self.tickers = self.polo.returnTicker()
self.bal = self.polo.returnCompleteBalances()
except Exception as e:
print(e)
time.sleep(360)
self.reset_tickers()
return
def get_keys(self):
with open("./godsplan.txt") as f:
content = f.readlines()
content[0] = content[0][:-1]
if (content[1][-1:] == "\n"):
content[1] = content[1][:-1]
return content
def make_shapes(self):
sign = 1
self.out_shapes = []
self.in_shapes = []
for ix in range(1, self.outputs + 1):
sign = sign * -1
self.out_shapes.append((0.0 - (sign * .005 * ix), 0.0, -1.0))
for ix2 in range(1, (self.inputs // self.outputs) + 1):
self.in_shapes.append(
(0.0 + (sign * .01 * ix2), 0.0 - (sign * .01 * ix2), 0.0))
self.subStrate = Substrate(self.in_shapes, self.out_shapes)
def get_price(self, coin):
return self.tickers[coin]['last']
def set_target(self):
total = 0
full_bal = self.polo.returnCompleteBalances()
for x in full_bal:
total += full_bal[x]["btcValue"]
if (self.base_sym != "BTC"):
total = total * self.get_price(self.base_sym + "_" +
"BTC") * self.target_percent
print(total)
self.target = total
def poloTrader(self):
self.refresh_balances()
end_prices = {}
active = self.get_one_bar_input_2d()
self.load_net()
network = ESNetwork(self.subStrate, self.cppn, self.params, self.hd)
net = network.create_phenotype_network_nd('paper_net.png')
net.reset()
sell_syms = []
buy_syms = []
buy_signals = []
sell_signals = []
self.closeOrders()
for n in range(1, self.hd):
net.activate(active[self.hd - n])
out = net.activate(active[0])
for x in range(len(out)):
sym = self.hs.coin_dict[x]
end_prices[sym] = self.get_price(self.base_sym + "_" + sym)
if (out[x] > .5):
buy_signals.append(out[x])
buy_syms.append(sym)
if (out[x] < -.5):
sell_signals.append(out[x])
sell_syms.append(sym)
#rng = iter(shuffle(rng))
sorted_buys = np.argsort(buy_signals)[::-1]
sorted_sells = np.argsort(sell_signals)
self.reset_tickers()
for x in sorted_sells:
sym = sell_syms[x]
p = self.get_price(self.base_sym + "_" + sym)
price = p - (p * .005)
self.sell_coin(self.base_sym + "_" + sym, price)
for x in sorted_buys:
sym = buy_syms[x]
self.target_percent = .1 + out[x] - .45
p = self.get_price(self.base_sym + "_" + sym)
price = p * 1.005
self.buy_coin(self.base_sym + "_" + sym, price)
if datetime.now() >= self.end_ts:
return
else:
self.purge_polo_client()
time.sleep(self.ticker_len)
self.load_polo_client()
self.refresh_data()
self.make_shapes()
#self.closeOrders()
self.poloTrader()
class PaperTrader:
params = {
"initial_depth": 3,
"max_depth": 4,
"variance_threshold": 0.00013,
"band_threshold": 0.00013,
"iteration_level": 3,
"division_threshold": 0.00013,
"max_weight": 8.0,
"activation": "tanh"
}
# Config for CPPN.
config = neat.config.Config(neat.genome.DefaultGenome,
neat.reproduction.DefaultReproduction,
neat.species.DefaultSpeciesSet,
neat.stagnation.DefaultStagnation,
'config_trader')
in_shapes = []
out_shapes = []
def __init__(self, ticker_len, start_amount, histdepth, base_sym):
self.trade_hist = {}
self.polo = Poloniex()
self.hd = histdepth
self.ticker_len = ticker_len
self.end_ts = datetime.now() + timedelta(seconds=(ticker_len * 24))
self.start_amount = start_amount
self.hs = HistWorker()
self.hs.combine_live_usd_frames()
print(self.hs.currentHists.keys())
self.end_idx = len(self.hs.hist_shaped[0]) - 1
self.but_target = .1
self.inputs = self.hs.hist_shaped.shape[0] * (
self.hs.hist_shaped[0].shape[1])
self.outputs = self.hs.hist_shaped.shape[0]
self.folio = CryptoFolio(start_amount,
list(self.hs.currentHists.keys()), base_sym)
self.base_sym = base_sym
sign = 1
for ix in range(1, self.outputs + 1):
sign = sign * -1
self.out_shapes.append((0.0 - (sign * .005 * ix), 0.0, -1.0))
for ix2 in range(1, (self.inputs // self.outputs) + 1):
self.in_shapes.append(
(0.0 + (sign * .01 * ix2), 0.0 - (sign * .01 * ix2), 0.0))
self.subStrate = Substrate(self.in_shapes, self.out_shapes)
self.load_net()
print(self.hs.coin_dict)
self.poloTrader()
def refresh_data(self):
try:
self.hs.pull_polo_usd_live(21)
self.hs.combine_live_usd_frames()
self.end_idx = len(self.hs.hist_shaped[0]) - 1
except Exception as e:
print(e)
time.sleep(360)
self.refresh_data()
return
def load_net(self):
champ_file = open("./champ_data/latest_greatest.pkl", 'rb')
g = pickle.load(champ_file)
#file.close()
the_cppn = neat.nn.FeedForwardNetwork.create(g, self.config)
self.cppn = the_cppn
def make_shapes(self):
self.in_shapes = []
self.out_shapes = []
sign = 1
for ix in range(1, self.outputs + 1):
sign = sign * -1
self.out_shapes.append((0.0 - (sign * .005 * ix), -1.0, -1.0))
for ix2 in range(1, (self.inputs // self.outputs) + 1):
self.in_shapes.append(
(0.0 + (sign * .01 * ix2), 0.0 - (sign * .01 * ix2), 1.0))
def reset_tickers(self):
try:
self.tickers = self.polo.returnTicker()
except Exception as e:
time.sleep(360)
self.reset_tickers()
return
def get_price(self, coin):
return self.tickers[coin]['last']
def get_current_balance(self):
#self.refresh_data()
self.reset_tickers()
c_prices = {}
for s in self.hs.currentHists.keys():
if s != self.base_sym:
c_prices[s] = self.get_price(self.base_sym + "_" + s)
return self.folio.get_total_btc_value_no_sell(c_prices)
def get_one_bar_input_2d(self):
master_active = []
try:
for x in range(0, self.hd):
active = []
#print(self.outputs)
for y in range(0, self.outputs):
sym_data = self.hs.hist_shaped[y][self.end_idx - x]
#print(len(sym_data))
active += sym_data.tolist()
master_active.append(active)
except:
self.refresh_data()
self.get_one_bar_input_2d()
#print(active)
return master_active
def poloTrader(self):
try:
trade_df = pd.read_json("./live_hist/json_hist.json")
except Exception as e:
trade_df = pd.DataFrame()
end_prices = {}
active = self.get_one_bar_input_2d()
self.load_net()
sub = Substrate(self.in_shapes, self.out_shapes)
net = ESNetwork(sub, self.cppn, self.params, self.hd)
network = net.create_phenotype_network_nd('paper_net.png')
sell_syms = []
buy_syms = []
buy_signals = []
sell_signals = []
for n in range(1, self.hd):
network.activate(active[self.hd - n])
out = network.activate(active[0])
self.reset_tickers()
for x in range(len(out)):
sym = self.hs.coin_dict[x]
end_prices[sym] = self.get_price(self.base_sym + "_" + sym)
if (out[x] > .5):
buy_signals.append(out[x])
buy_syms.append(sym)
if (out[x] < -.5):
sell_signals.append(out[x])
sell_syms.append(sym)
#rng = iter(shuffle(rng))
sorted_buys = np.argsort(buy_signals)[::-1]
sorted_sells = np.argsort(sell_signals)
for x in sorted_sells:
try:
sym = sell_syms[x]
p = end_prices[sym]
print("selling: ", sym)
self.folio.sell_coin(sym, p)
except Exception as e:
print("error placing order")
for x in sorted_buys:
try:
sym = buy_syms[x]
p = end_prices[sym]
print("buying: ", sym)
self.folio.buy_coin(sym, p)
except Exception as e:
print("error placing order")
'''
self.trade_hist["date"] = datetime.now()
self.trade_hist["portfoliovalue"] = self.folio.get_total_btc_value_no_sell(end_prices)[0]
self.trade_hist["portfolio"] = self.folio.ledger
self.trade_hist["percentchange"] = ((self.trade_hist["portfoliovalue"] - self.folio.start)/self.folio.start)*100
trade_df.append(self.trade_hist)
trade_df.to_json("./live_hist/json_hist.json")
if(self.trade_hist["portfoliovalue"] > self.folio.start *1.1):
self.folio.start = self.folio.get_total_btc_value(end_prices)[0]
'''
if datetime.now() >= self.end_ts:
port_info = self.folio.get_total_btc_value(end_prices)
print("total val: ", port_info[0], "btc balance: ", port_info[1])
return
else:
print(self.get_current_balance())
for t in range(2):
p_vals = self.get_current_balance()
print("current value: ", p_vals[0], "current holdings: ",
p_vals[1])
time.sleep(self.ticker_len / 2)
self.refresh_data()
self.poloTrader()
class PurpleTrader:
#needs to be initialized so as to allow for 62 outputs that return a coordinate
# ES-HyperNEAT specific parameters.
params = {"initial_depth": 3,
"max_depth": 4,
"variance_threshold": 0.00013,
"band_threshold": 0.00013,
"iteration_level": 3,
"division_threshold": 0.00013,
"max_weight": 3.0,
"activation": "tanh"}
# Config for CPPN.
config = neat.config.Config(neat.genome.DefaultGenome, neat.reproduction.DefaultReproduction,
neat.species.DefaultSpeciesSet, neat.stagnation.DefaultStagnation,
'config_trader')
start_idx = 0
highest_returns = 0
portfolio_list = []
rand_start = 0
in_shapes = []
out_shapes = []
def __init__(self, hist_depth):
self.hs = HistWorker()
self.hs.combine_binance_frames_vol_sorted(8)
self.hd = hist_depth
print(self.hs.currentHists.keys())
self.end_idx = len(self.hs.hist_shaped[0])
self.but_target = .1
self.inputs = self.hs.hist_shaped.shape[0]*(self.hs.hist_shaped[0].shape[1])
self.outputs = len(self.hs.coin_dict)
print(self.inputs, self.outputs)
self.epoch_len = 144
self.node_names = ['x1', 'y1', 'z1', 'x2', 'y2', 'z2', 'weight']
self.leaf_names = []
num_leafs = 2**(len(self.node_names)-1)//2
self.tree = nDimensionGoldenTree((0.0, 0.0, 0.0), 1.0, 1)
self.tree.divide_childrens()
self.set_substrate()
self.set_leaf_names()
def set_leaf_names(self):
for l in range(len(self.in_shapes[0])):
self.leaf_names.append('leaf_one_'+str(l))
self.leaf_names.append('leaf_two_'+str(l))
#self.leaf_names.append('bias')
def set_substrate(self):
sign = 1
x_increment = 1.0 / self.outputs
y_increment = 1.0 / len(self.hs.hist_shaped[0][0])
for ix in range(self.outputs):
self.out_shapes.append((1.0-(ix*x_increment), 0.0, -1.0))
for ix2 in range(self.inputs//self.outputs):
if(ix2 >= len(self.tree.cs)-1):
treex = ix2 - len(self.tree.cs)-1
else:
treex = ix2
center = self.tree.cs[treex]
self.in_shapes.append((center.coord[0]+(ix*x_increment), center.coord[1] - (ix2*y_increment), center.coord[2]+.5))
self.subStrate = Substrate(self.in_shapes, self.out_shapes)
def set_portfolio_keys(self, folio):
for k in self.hs.currentHists.keys():
folio.ledger[k] = 0
def get_one_epoch_input(self,end_idx):
master_active = []
for x in range(0, self.hd):
active = []
#print(self.outputs)
for y in range(0, self.outputs):
try:
sym_data = self.hs.hist_shaped[y][end_idx-x]
#print(len(sym_data))
active += sym_data.tolist()
except:
print('error')
master_active.append(active)
#print(active)
return master_active
def evaluate(self, g, config):
# k so here we create a random start index
# then we make our nets, and start fitness evaluation
rand_start = self.rand_start
[cppn] = create_cppn(g, config, self.leaf_names, ['cppn_out'])
net = ESNetwork(self.subStrate, cppn, self.params)
network = net.create_phenotype_network_nd()
portfolio_start = 1.0
key_list = list(self.hs.currentHists.keys())
portfolio = CryptoFolio(portfolio_start, self.hs.coin_dict)
end_prices = {}
buys = 0
sells = 0
if(len(network.node_evals) > 0):
for z in range(rand_start, rand_start+self.epoch_len):
active = self.get_one_epoch_input(z)
signals = []
network.reset()
for n in range(1, self.hd+1):
out = network.activate(active[self.hd-n])
for x in range(len(out)):
signals.append(out[x])
sorted_shit = np.argsort(signals)[::-1]
for x in sorted_shit:
sym = self.hs.coin_dict[x]
#print(out[x])
#try:
if(out[x] < -.5):
#print("selling")
portfolio.sell_coin(sym, self.hs.currentHists[sym]['close'][z])
#print("bought ", sym)
if(out[x] > .5):
#print("buying")
portfolio.target_amount = .1 + (out[x] * .1)
portfolio.buy_coin(sym, self.hs.currentHists[sym]['close'][z])
#print("sold ", sym)
#skip the hold case because we just dont buy or sell hehe
if(z > self.epoch_len+rand_start-2):
end_prices[sym] = self.hs.currentHists[sym]['close'][self.epoch_len+rand_start]
result_val = portfolio.get_total_btc_value(end_prices)
print(result_val[0], "buys: ", result_val[1], "sells: ", result_val[2])
ft = result_val[0]
else:
ft = 0.0
return ft
def eval_fitness(self, genomes, config):
#generate random batch size between 1/8 and 1/5 our whole data set length
min_batch_size = (self.hs.hist_full_size-self.hd) // 8
max_batch_size = (self.hs.hist_full_size-self.hd) // 5
self.epoch_len = randint(min_batch_size, max_batch_size)
#generate random start here too ?
self.rand_start = randint(self.hd, self.hs.hist_full_size - self.epoch_len)
runner = neat.ParallelEvaluator(8, self.evaluate)
runner.evaluate(genomes, config)
from hist_service import HistWorker hs = HistWorker() hs.pull_polo() #hs.write_binance_training_files()
class PurpleTrader:
# ES-HyperNEAT specific parameters.
params = {
"initial_depth": 1,
"max_depth": 3,
"variance_threshold": 0.055,
"band_threshold": 0.034,
"iteration_level": 3,
"division_threshold": 0.021,
"max_weight": 34.55,
"activation": "tanh"
}
# Config for CPPN.
config = neat.config.Config(neat.genome.DefaultGenome,
neat.reproduction.DefaultReproduction,
neat.species.DefaultSpeciesSet,
neat.stagnation.DefaultStagnation,
'config_trader')
start_idx = 0
highest_returns = 0
portfolio_list = []
leaf_names = []
def __init__(self, hist_depth, num_gens, gen_count=1):
self.hd = hist_depth
if gen_count != 1:
self.num_gens = num_gens
else:
self.num_gens = gen_count + num_gens
self.gen_count = gen_count
self.refresh()
def refresh(self):
self.in_shapes = []
self.out_shapes = [(0.0, -1.0, -1.0)]
self.hs = HistWorker()
self.hs.get_binance_train()
print(self.hs.currentHists.keys())
self.end_idx = len(self.hs.hist_shaped[0])
self.but_target = .1
print(self.hs.hist_shaped.shape)
self.num_syms = self.hs.hist_shaped.shape[0]
self.inputs = self.hs.hist_shaped[0].shape[1] + 1
self.outputs = 1
sign = 1
for ix2 in range(1, self.inputs + 1):
sign *= -1
self.in_shapes.append(
(0.0 + (sign * .01 * ix2), 0.0 - (sign * .01 * ix2), 0.0))
self.substrate = Substrate(self.in_shapes, self.out_shapes)
self.set_leaf_names()
# informing the substrate
def reset_substrate(self, input_row):
current_inputs = self.substrate.input_coordinates
new_input = []
for ix, t in enumerate(current_inputs):
t = list(t)
offset = input_row[ix] * .5
t[2] = t[2] + .5
new_input.append(tuple(t))
#self.in_shapes = new_input
self.substrate = Substrate(new_input, self.out_shapes)
def set_portfolio_keys(self, folio):
for k in self.hs.currentHists.keys():
folio.ledger[k] = 0
def set_leaf_names(self):
for l in range(len(self.in_shapes[0])):
self.leaf_names.append('leaf_one_' + str(l))
self.leaf_names.append('leaf_two_' + str(l))
print(self.leaf_names)
def get_one_epoch_input(self, end_idx):
master_active = []
for x in range(1, self.hd + 1):
active = []
#print(self.outputs)
for y in range(0, self.outputs):
sym_data = self.hs.hist_shaped[y][end_idx + x]
#print(len(sym_data))
active += sym_data.tolist()
master_active.append(active)
#print(active)
return master_active
def get_single_symbol_epoch_recurrent(self, end_idx, symbol_idx):
master_active = []
for x in range(0, self.hd):
try:
sym_data = self.hs.hist_shaped[symbol_idx][end_idx - x]
#print(len(sym_data))
master_active.append(sym_data.tolist())
except:
print('error')
return master_active
def get_single_symbol_epoch_recurrent_with_position_size(
self, end_idx, symbol_idx, current_position):
master_active = []
for x in range(0, self.hd):
try:
sym_data = self.hs.hist_shaped[symbol_idx][end_idx - x]
#print(len(sym_data))
sym_data = sym_data.tolist()
sym_data.append(current_position)
master_active.append(sym_data)
except:
print('error')
return master_active
def evaluate_champ(self, builder, rand_start, g, verbose=False):
portfolio_start = 1000.0
portfolio = CryptoFolio(portfolio_start, self.hs.coin_dict, "USDT")
end_prices = {}
phenotypes = {}
buys = 0
sells = 0
with open("./trade_hists/kraken/" + str(g.key) + "_hist.txt",
"w") as ft:
ft.write('date,current_balance \n')
for z_minus in range(0, self.epoch_len - 1):
for x in range(self.num_syms):
z = rand_start - z_minus
sym = self.hs.coin_dict[x]
z = rand_start - z_minus
pos_size = portfolio.ledger[sym]
active = self.get_single_symbol_epoch_recurrent_with_position_size(
z, x, pos_size)
if (z_minus == 0 or (z_minus + 1) % 8 == 0):
self.reset_substrate(active[0])
builder.substrate = self.substrate
phenotypes[sym] = builder.create_phenotype_network_nd()
network = phenotypes[sym]
network.reset()
for n in range(1, self.hd + 1):
network.activate([active[self.hd - n]])
out = network.activate([active[0]])
end_prices[sym] = self.hs.currentHists[sym]['close'][z]
if (out[0] < .5 or (z_minus + 1) % 16 == 0):
portfolio.sell_coin(
sym, self.hs.currentHists[sym]['close'][z])
else:
portfolio.buy_coin(
sym, self.hs.currentHists[sym]['close'][z])
ft.write(str(self.hs.currentHists[sym]['time'][z]) + ",")
ft.write(
str(
portfolio.get_total_btc_value_no_sell(end_prices)
[0]) + " \n")
result_val = portfolio.get_total_btc_value(end_prices)
print("genome id ", g.key, " : ")
print(result_val[0], "buys: ", result_val[1], "sells: ",
result_val[2])
ft = result_val[0]
return ft
def evaluate(self, builder, rand_start, g, verbose=False):
portfolio_start = 1000.0
portfolio = CryptoFolio(portfolio_start, self.hs.coin_dict, "USD")
end_prices = {}
phenotypes = {}
buys = 0
sells = 0
last_val = portfolio_start
ft = 0.0
for z_minus in range(0, self.epoch_len):
for x in range(self.num_syms):
sym = self.hs.coin_dict[x]
z = rand_start - z_minus
pos_size = portfolio.ledger[sym]
active = self.get_single_symbol_epoch_recurrent_with_position_size(
z, x, pos_size)
#print(active)
if (z_minus == 0 or (z_minus + 1) % 8 == 0):
self.reset_substrate(active[0])
builder.substrate = self.substrate
phenotypes[sym] = builder.create_phenotype_network_nd()
network = phenotypes[sym]
network.reset()
for n in range(1, self.hd + 1):
network.activate([active[self.hd - n]])
out = network.activate([active[0]])
if (out[0] < -0.5):
portfolio.sell_coin(sym,
self.hs.currentHists[sym]['close'][z])
#print("bought ", sym)
elif (out[0] > 0.5):
did_buy = portfolio.buy_coin(
sym, self.hs.currentHists[sym]['close'][z])
#rng = iter(shuffle(rng))
end_prices[sym] = self.hs.currentHists[sym]['close'][z]
bal_now = portfolio.get_total_btc_value_no_sell(end_prices)[0]
ft += bal_now - last_val
last_val = bal_now
result_val = portfolio.get_total_btc_value(end_prices)
print(g.key, " : ")
print(result_val[0], "buys: ", result_val[1], "sells: ", result_val[2])
if result_val[0] == portfolio_start:
ft = -.2
return ft
def eval_fitness(self, genomes, config):
self.epoch_len = 16
r_start = randint(0 + self.epoch_len, self.hs.hist_full_size - self.hd)
best_g_fit = 0.0
champ_counter = self.gen_count % 10
#img_count = 0
for idx, g in genomes:
[cppn] = create_cppn(g, config, self.leaf_names, ["cppn_out"])
net_builder = ESNetwork(self.substrate, cppn, self.params)
#cppn = neat.nn.FeedForwardNetwork.create(g, config)
#network = ESNetwork(self.subStrate, cppn, self.params, self.hd)
#net = network.create_phenotype_network_nd()
train_ft = self.evaluate(net_builder, r_start, g)
g.fitness = train_ft
if (g.fitness > best_g_fit):
best_g_fit = g.fitness
with open(
"./champ_data/kraken/latest_greatest" +
str(champ_counter) + ".pkl", 'wb') as output:
pickle.dump(g, output)
#img_count += 1
'''
if(champ_counter == 0):
self.refresh()
self.compare_champs()
'''
self.gen_count += 1
return
def write_hists(self):
self.epoch_len = self.hs.hist_full_size - (self.hd + 1)
r_start = self.epoch_len
champ_current = open("./champ_data/binance/latest_greatest.pkl", 'rb')
g = pickle.load(champ_current)
champ_current.close()
[cppn] = create_cppn(g, self.config, self.leaf_names, ["cppn_out"])
net_builder = ESNetwork(self.substrate, cppn, self.params)
champ_fit = self.evaluate_champ(net_builder, r_start, g)
for f in os.listdir("./champ_data/binance"):
if (f != "lastest_greatest.pkl"):
champ_file = open("./champ_data/binance/" + f, 'rb')
g = pickle.load(champ_file)
champ_file.close()
[cppn] = create_cppn(g, self.config, self.leaf_names,
["cppn_out"])
net_builder = ESNetwork(self.substrate, cppn, self.params)
g.fitness = self.evaluate_champ(net_builder, r_start, g)
if (g.fitness > champ_fit):
with open("./champ_data/binance/latest_greatest.pkl",
'wb') as output:
pickle.dump(g, output)
return
def validate_fitness(self):
config = self.config
genomes = neat.Checkpointer.restore_checkpoint(
"./pkl_pops/pop-checkpoint-27").population
self.epoch_len = 233
r_start = self.hs.hist_full_size - self.epoch_len - 1
best_g_fit = 1.0
for idx in genomes:
g = genomes[idx]
cppn = neat.nn.FeedForwardNetwork.create(g, config)
network = ESNetwork(self.subStrate, cppn, self.params, self.hd)
net = network.create_phenotype_network_nd()
g.fitness = self.evaluate(net, network, r_start, g)
if (g.fitness > best_g_fit):
best_g_fit = g.fitness
with open('./champ_data/binance/latest_greatest.pkl',
'wb') as output:
pickle.dump(g, output)
return
# Create the population and run the XOR task by providing the above fitness function.
def run_pop(self, checkpoint=""):
if (checkpoint == ""):
pop = neat.population.Population(self.config)
else:
pop = neat.Checkpointer.restore_checkpoint(
"./pkl_pops/binance/pop-checkpoint-" + checkpoint)
checkpoints = neat.Checkpointer(
generation_interval=2,
time_interval_seconds=None,
filename_prefix='./pkl_pops/binance/pop-checkpoint-')
stats = neat.statistics.StatisticsReporter()
pop.add_reporter(stats)
pop.add_reporter(checkpoints)
pop.add_reporter(neat.reporting.StdOutReporter(True))
print(self.num_gens)
winner = pop.run(self.eval_fitness, self.num_gens)
return winner, stats
# If run as script.
def run_training(self, checkpoint=""):
#print(task.trial_run())
if checkpoint == "":
winner = self.run_pop()[0]
else:
winner = self.run_pop(checkpoint)[0]
print('\nBest genome:\n{!s}'.format(winner))
checkpoint_string = str(self.num_gens - 1)
self.num_gens += self.num_gens
self.run_training(checkpoint_string)
def run_validation(self):
self.validate_fitness()
class LiveTrader:
params = {"initial_depth": 4,
"max_depth": 4,
"variance_threshold": 0.00013,
"band_threshold": 0.00013,
"iteration_level": 3,
"division_threshold": 0.00013,
"max_weight": 8.0,
"activation": "tanh"}
# Config for CPPN.
config = neat.config.Config(neat.genome.DefaultGenome, neat.reproduction.DefaultReproduction,
neat.species.DefaultSpeciesSet, neat.stagnation.DefaultStagnation,
'config_trader')
def __init__(self, ticker_len, target_percent, hd):
self.polo = Poloniex(key, secret)
self.hist_depth = hd
self.target_percent = target_percent
self.ticker_len = ticker_len
self.end_ts = datetime.now()+timedelta(seconds=(ticker_len*55))
self.hs = HistWorker()
self.refresh_data()
self.tickers = self.polo.returnTicker()
self.bal = self.polo.returnBalances()
self.sellCoins()
self.set_target()
self.inputs = self.hs.hist_shaped.shape[0]*(self.hs.hist_shaped[0].shape[1])
self.outputs = self.hs.hist_shaped.shape[0]
self.make_shapes()
self.leaf_names = []
self.db = tinydb.database("live_hist/memories.json")
for l in range(len(self.in_shapes[0])):
self.leaf_names.append('leaf_one_'+str(l))
self.leaf_names.append('leaf_two_'+str(l))
#self.load_net()
self.poloTrader()
def load_net(self):
#file = open("./champ_gens/thot-checkpoint-13",'rb')
g = neat.Checkpointer.restore_checkpoint("./champ_gens/thot-checkpoint-25")
best_fit = 0.0
for gx in g.population:
if g.population[gx].fitness != None:
if g.population[gx].fitness > best_fit:
bestg = g.population[gx]
g = bestg
#file.close()
[the_cppn] = create_cppn(g, self.config, self.leaf_names, ['cppn_out'])
self.cppn = the_cppn
def refresh_data(self):
self.hs.pull_polo_live(20)
self.hs.combine_live_frames(self.hist_depth)
def make_shapes(self):
self.in_shapes = []
self.out_shapes = []
sign = 1
for ix in range(1,self.outputs+1):
sign = sign *-1
self.out_shapes.append((0.0-(sign*.005*ix), -1.0, -1.0))
for ix2 in range(1,(self.inputs//self.outputs)+1):
self.in_shapes.append((0.0+(sign*.01*ix2), 0.0-(sign*.01*ix2), 1.0))
def get_one_bar_input_2d(self,end_idx=10):
master_active = []
for x in range(0, self.hist_depth):
active = []
#print(self.outputs)
for y in range(0, self.outputs):
sym_data = self.hs.hist_shaped[y][self.hist_depth-x]
#print(len(sym_data))
active += sym_data.tolist()
master_active.append(active)
#print(active)
return master_active
def closeOrders(self):
try:
orders = self.polo.returnOpenOrders()
except:
print('error getting open orers')
time.sleep(360)
self.closeOrder()
for o in orders:
if orders[o] != []:
try:
ordnum = orders[o][0]['orderNumber']
self.polo.cancelOrder(ordnum)
except:
print('error closing')
def sellCoins(self):
for b in self.tickers:
if(b[:3] == "BTC"):
price = self.get_price(b)
price = price - (price * .005)
self.sell_coin(b, price)
def buy_coin(self, coin, price):
amt = self.target / price
if(self.bal['BTC'] > self.target):
self.polo.buy(coin, price, amt, fillOrKill=1)
print("buying: ", coin)
return
def sell_coin(self, coin, price):
amt = self.bal[coin[4:]]
if (amt*price > .0001):
try:
self.polo.sell(coin, price, amt,fillOrKill=1)
print("selling this shit: ", coin)
except:
print('error selling', coin)
return
def reset_tickers(self):
try:
self.tickers = self.polo.returnTicker()
self.bal = self.polo.returnBalances()
except:
time.sleep(360)
self.reset_tickers()
return
def get_price(self, coin):
return self.tickers[coin]['last']
def set_target(self):
total = 0
full_bal = self.polo.returnCompleteBalances()
for x in full_bal:
total += full_bal[x]["btcValue"]
self.target = total*self.target_percent
def poloTrader(self):
end_prices = {}
active = self.get_one_bar_input_2d()
self.load_net()
sub = Substrate(self.in_shapes, self.out_shapes)
network = ESNetwork(sub, self.cppn, self.params)
net = network.create_phenotype_network_nd('paper_net.png')
net.reset()
for n in range(1, self.hist_depth+1):
out = net.activate(active[self.hist_depth-n])
#print(len(out))
rng = len(out)
#rng = iter(shuffle(rng))
self.reset_tickers()
for x in np.random.permutation(rng):
sym = self.hs.coin_dict[x]
#print(out[x])
try:
if(out[x] < -.5):
print("selling: ", sym)
p = self.get_price('BTC_'+sym)
price = p -(p*.01)
self.sell_coin('BTC_'+sym, price)
elif(out[x] > .5):
print("buying: ", sym)
self.target_percent = .1 + out[x] - .45
p = self.get_price('BTC_'+sym)
price = p*1.01
self.buy_coin('BTC_'+sym, price)
except:
print('error', sym)
#skip the hold case because we just dont buy or sell hehe
if datetime.now() >= self.end_ts:
return
else:
time.sleep(self.ticker_len)
self.refresh_data()
#self.closeOrders()
self.poloTrader()
class PaperTrader:
params = {"initial_depth": 3,
"max_depth": 4,
"variance_threshold": 0.00013,
"band_threshold": 0.00013,
"iteration_level": 3,
"division_threshold": 0.00013,
"max_weight": 3.0,
"activation": "tanh"}
in_shapes = []
out_shapes = []
db = TinyDB('./live_hist/memories.json')
# Config for CPPN.
config = neat.config.Config(neat.genome.DefaultGenome, neat.reproduction.DefaultReproduction,
neat.species.DefaultSpeciesSet, neat.stagnation.DefaultStagnation,
'config_trader')
def __init__(self, ticker_len, start_amount, histdepth):
self.trade_hist = {}
self.polo = Poloniex()
self.hist_depth = histdepth
self.ticker_len = ticker_len
self.end_ts = datetime.now()+timedelta(seconds=(ticker_len*89))
self.start_amount = start_amount
self.hs = HistWorker()
self.refresh_data()
self.folio = CryptoFolio(start_amount, list(self.hs.currentHists.keys()))
self.inputs = self.hs.hist_shaped.shape[0]*(self.hs.hist_shaped[0].shape[1])
self.outputs = self.hs.hist_shaped.shape[0]
self.leaf_names = []
#num_leafs = 2**(len(self.node_names)-1)//2
self.tree = nDimensionGoldenTree((0.0, 0.0, 0.0), 1.0, 1)
self.tree.divide_childrens()
self.set_substrate()
self.set_leaf_names()
self.epoch_len = histdepth
self.load_net()
print(self.hs.coin_dict)
self.db.purge()
self.poloTrader()
def set_leaf_names(self):
for l in range(len(self.in_shapes[0])):
self.leaf_names.append('leaf_one_'+str(l))
self.leaf_names.append('leaf_two_'+str(l))
#self.leaf_names.append('bias')
def set_substrate(self):
sign = 1
x_increment = 1.0 / self.outputs
y_increment = 1.0 / len(self.hs.hist_shaped[0][0])
for ix in range(self.outputs):
self.out_shapes.append((1.0-(ix*x_increment), 0.0, -1.0))
for ix2 in range(self.inputs//self.outputs):
if(ix2 >= len(self.tree.cs)-1):
treex = ix2 - len(self.tree.cs)-1
else:
treex = ix2
center = self.tree.cs[treex]
self.in_shapes.append((center.coord[0]+(ix*x_increment), center.coord[1] - (ix2*y_increment), center.coord[2]+.5))
self.subStrate = Substrate(self.in_shapes, self.out_shapes)
def set_portfolio_keys(self, folio):
for k in self.hs.currentHists.keys():
folio.ledger[k] = 0
def refresh_data(self):
self.hs.pull_polo_live(21)
self.hs.combine_live_frames(89)
def load_net(self):
#file = open("./champ_gens/thot-checkpoint-13",'rb')
g = neat.Checkpointer.restore_checkpoint("./binance_champs_2/tradegod-checkpoint-48")
'''
best_fit = 0.0
for gx in g.population:
if g.population[gx].fitness != None:
if g.population[gx].fitness > best_fit:
bestg = g.population[gx]
g = bestg
'''
#file.close()
g = g.population[5910]
[the_cppn] = create_cppn(g, self.config, self.leaf_names, ['cppn_out'])
self.cppn = the_cppn
def make_shapes(self):
self.in_shapes = []
self.out_shapes = []
sign = 1
for ix in range(1,self.outputs+1):
sign = sign *-1
self.out_shapes.append((0.0-(sign*.005*ix), -1.0, -1.0))
for ix2 in range(1,(self.inputs//self.outputs)+1):
self.in_shapes.append((0.0+(sign*.01*ix2), 0.0-(sign*.01*ix2), 1.0))
def reset_tickers(self):
try:
self.tickers = self.polo.returnTicker()
except:
time.sleep(360)
self.reset_tickers()
return
def get_price(self, coin):
return self.tickers[coin]['last']
def get_current_balance(self):
#self.refresh_data()
c_prices = {}
for s in self.hs.currentHists.keys():
if s != 'BTC':
c_prices[s] = self.hs.currentHists[s]['close'].iloc[-1]
return self.folio.get_total_btc_value_no_sell(c_prices)
def get_one_bar_input_2d(self,end_idx=10):
master_active = []
for x in range(0, self.hist_depth):
active = []
#print(self.outputs)
for y in range(0, self.outputs):
sym_data = self.hs.hist_shaped[y][self.hist_depth-x]
#print(len(sym_data))
active += sym_data.tolist()
master_active.append(active)
#print(active)
return master_active
def poloTrader(self):
try:
trade_df = pd.read_json("./live_hist/json_hist.json")
except:
trade_df = pd.DataFrame()
end_prices = {}
active = self.get_one_bar_input_2d()
self.load_net()
sub = Substrate(self.in_shapes, self.out_shapes)
network = ESNetwork(sub, self.cppn, self.params)
net = network.create_phenotype_network_nd()
net.reset()
signals = []
net.reset()
for n in range(1, self.hist_depth+1):
out = net.activate(active[self.hist_depth-n])
for x in range(len(out)):
signals.append(out[x])
#rng = iter(shuffle(rng))
sorted_shit = np.argsort(signals)[::-1]
self.reset_tickers()
sym = ""
for x in sorted_shit:
sym = self.hs.coin_dict[x]
#print(out[x])
try:
if(out[x] < -.5):
p = self.get_price('BTC_'+sym)
print("selling: ", sym)
self.folio.sell_coin(sym, p)
elif(out[x] > .5):
p = self.get_price('BTC_'+sym)
print("buying: ", sym)
self.folio.buy_coin(sym, p)
except:
print("error buying or selling")
#skip the hold case because we just dont buy or sell hehe
end_prices[sym] = self.hs.currentHists[sym]["close"].iloc[-1]
#self.trade_hist["date"] = self.hs.currentHists[sym]["date"].iloc[-1]
self.trade_hist["date"] = time.time()
self.trade_hist["portfoliovalue"] = self.folio.get_total_btc_value_no_sell(end_prices)[0]
self.trade_hist["portfolio"] = self.folio.ledger
self.trade_hist["percentchange"] = ((self.trade_hist["portfoliovalue"] - self.folio.start)/self.folio.start)*100
print(self.trade_hist)
print(self.folio.ledger)
self.db.insert(self.trade_hist)
self.trade_hist = {}
'''
if(self.trade_hist["portfoliovalue"] > self.folio.start *1.1):
self.folio.start = self.folio.get_total_btc_value(end_prices)[0]
'''
if datetime.now() >= self.end_ts:
port_info = self.folio.get_total_btc_value(end_prices)
print("total val: ", port_info[0], "btc balance: ", port_info[1])
return
else:
print(self.get_current_balance())
for t in range(2):
time.sleep(self.ticker_len/2)
#print(self.folio.ledger)
self.refresh_data
self.poloTrader()
class PurpleTrader:
#needs to be initialized so as to allow for 62 outputs that return a coordinate
# ES-HyperNEAT specific parameters.
params = {
"initial_depth": 1,
"max_depth": 3,
"variance_threshold": 0.3,
"band_threshold": 0.3,
"iteration_level": 3,
"division_threshold": 0.5,
"max_weight": 8.0,
"activation": "tanh"
}
# Config for CPPN.
config = neat.config.Config(neat.genome.DefaultGenome,
neat.reproduction.DefaultReproduction,
neat.species.DefaultSpeciesSet,
neat.stagnation.DefaultStagnation,
'config_trader')
start_idx = 0
highest_returns = 0
portfolio_list = []
def __init__(self, hist_depth, num_gens, gen_count=1):
self.hd = hist_depth
if gen_count != 1:
self.num_gens = num_gens
else:
self.num_gens = gen_count + num_gens
self.gen_count = gen_count
self.refresh()
def refresh(self):
self.in_shapes = []
self.out_shapes = []
self.hs = HistWorker()
self.hs.get_kraken_train()
print(self.hs.currentHists.keys())
self.end_idx = len(self.hs.hist_shaped[0])
self.but_target = .1
self.inputs = self.hs.hist_shaped.shape[0] * (
self.hs.hist_shaped[0].shape[1])
self.outputs = self.hs.hist_shaped.shape[0]
sign = 1
for ix in range(1, self.outputs + 1):
sign = sign * -1
self.out_shapes.append((0.0 - (sign * .005 * ix), 0.0, -1.0))
for ix2 in range(1, (self.inputs // self.outputs) + 1):
self.in_shapes.append(
(0.0 + (sign * .01 * ix2), 0.0 - (sign * .01 * ix2), 0.0))
self.subStrate = Substrate(self.in_shapes, self.out_shapes)
#self.leaf_names.append('bias')
def set_portfolio_keys(self, folio):
for k in self.hs.currentHists.keys():
folio.ledger[k] = 0
def get_one_epoch_input(self, end_idx):
master_active = []
for x in range(1, self.hd + 1):
active = []
#print(self.outputs)
for y in range(0, self.outputs):
sym_data = self.hs.hist_shaped[y][end_idx + x]
#print(len(sym_data))
active += sym_data.tolist()
master_active.append(active)
#print(active)
return master_active
def evaluate_champ(self, network, es, rand_start, g, verbose=False):
portfolio_start = 10000.0
portfolio = CryptoFolio(portfolio_start, self.hs.coin_dict, "USDT")
end_prices = {}
buys = 0
sells = 0
with open("./trade_hists/kraken/" + str(g.key) + "_hist.txt",
"w") as ft:
ft.write('date,current_balance \n')
for z_minus in range(0, self.epoch_len - 1):
z = rand_start - z_minus
active = self.get_one_epoch_input(z)
buy_signals = []
buy_syms = []
sell_syms = []
sell_signals = []
network.reset()
for n in range(1, self.hd + 1):
network.activate(active[self.hd - n])
out = network.activate(active[0])
for x in range(len(out)):
sym = self.hs.coin_dict[x]
end_prices[sym] = self.hs.currentHists[sym]['close'][z]
if (out[x] > .5):
buy_signals.append(out[x])
buy_syms.append(sym)
if (out[x] < -.5):
sell_signals.append(out[x])
sell_syms.append(sym)
#rng = iter(shuffle(rng))
sorted_buys = np.argsort(buy_signals)[::-1]
sorted_sells = np.argsort(sell_signals)
#print(len(sorted_shit), len(key_list))
for x in sorted_sells:
sym = sell_syms[x]
portfolio.sell_coin(sym,
self.hs.currentHists[sym]['close'][z])
for x in sorted_buys:
sym = buy_syms[x]
portfolio.buy_coin(sym,
self.hs.currentHists[sym]['close'][z])
ft.write(str(self.hs.currentHists[sym]['time'][z]) + ",")
ft.write(
str(portfolio.get_total_btc_value_no_sell(end_prices)[0]) +
" \n")
result_val = portfolio.get_total_btc_value(end_prices)
print("genome id ", g.key, " : ")
print(result_val[0], "buys: ", result_val[1], "sells: ",
result_val[2])
if result_val[1] == 0:
ft = .7
else:
ft = result_val[0]
return ft
def evaluate(self, network, es, rand_start, g, verbose=False):
portfolio_start = 1.0
portfolio = CryptoFolio(portfolio_start, self.hs.coin_dict, "USDT")
end_prices = {}
buys = 0
sells = 0
last_val = portfolio_start
ft = 0.0
for z_minus in range(0, self.epoch_len):
#TODO add comments to clarify all the
#shit im doing here
z = rand_start - z_minus
active = self.get_one_epoch_input(z)
buy_signals = []
buy_syms = []
sell_syms = []
sell_signals = []
network.reset()
for n in range(1, self.hd + 1):
network.activate(active[self.hd - n])
out = network.activate(active[0])
for x in range(len(out)):
sym = self.hs.coin_dict[x]
end_prices[sym] = self.hs.currentHists[sym]['close'][z]
if (out[x] > .5):
buy_signals.append(out[x])
buy_syms.append(sym)
if (out[x] < -.5):
sell_signals.append(out[x])
sell_syms.append(sym)
#rng = iter(shuffle(rng))
sorted_buys = np.argsort(buy_signals)[::-1]
sorted_sells = np.argsort(sell_signals)
#print(len(sorted_shit), len(key_list))
for x in sorted_sells:
sym = sell_syms[x]
portfolio.sell_coin(sym, self.hs.currentHists[sym]['close'][z])
for x in sorted_buys:
sym = buy_syms[x]
portfolio.buy_coin(sym, self.hs.currentHists[sym]['close'][z])
bal_now = portfolio.get_total_btc_value_no_sell(end_prices)[0]
ft += bal_now - last_val
last_val = bal_now
result_val = portfolio.get_total_btc_value(end_prices)
print(g.key, " : ")
print(result_val[0], "buys: ", result_val[1], "sells: ", result_val[2])
if result_val[0] == portfolio_start:
ft = ft * .9
return ft
def solve(self, network):
return self.evaluate(network) >= self.highest_returns
def trial_run(self):
r_start = 0
file = open("es_trade_god_cppn_3d.pkl", 'rb')
[cppn] = pickle.load(file)
network = ESNetwork(self.subStrate, cppn, self.params)
net = network.create_phenotype_network_nd()
fitness = self.evaluate(net, network, r_start)
return fitness
def eval_fitness(self, genome, config):
self.epoch_len = randint(42, 42 * 4)
r_start = randint(0 + self.epoch_len, self.hs.hist_full_size - self.hd)
#r_start_2 = self.hs.hist_full_size - self.epoch_len-1
best_g_fit = 0.0
champ_counter = self.gen_count % 10
#img_count = 0
for idx, g in genomes:
cppn = neat.nn.FeedForwardNetwork.create(g, config)
network = ESNetwork(self.subStrate, cppn, self.params, self.hd)
net = network.create_phenotype_network_nd()
train_ft = self.evaluate(net, network, r_start, g)
#validate_ft = self.evaluate(net, network, r_start_2, g)
g.fitness = train_ft
if (g.fitness > best_g_fit):
best_g_fit = g.fitness
with open(
"./champ_data/kraken/latest_greatest" +
str(champ_counter) + ".pkl", 'wb') as output:
pickle.dump(g, output)
#img_count += 1
if (champ_counter == 0):
self.refresh()
self.compare_champs()
self.gen_count += 1
return
def compare_champs(self):
self.epoch_len = self.hs.hist_full_size - (self.hd + 1)
r_start = self.epoch_len
champ_current = open("./champ_data/kraken/latest_greatest.pkl", 'rb')
g = pickle.load(champ_current)
champ_current.close()
cppn = neat.nn.FeedForwardNetwork.create(g, self.config)
network = ESNetwork(self.subStrate, cppn, self.params, self.hd)
net = network.create_phenotype_network_nd()
champ_fit = self.evaluate(net, network, r_start, g)
for f in os.listdir("./champ_data/kraken"):
if (f != "lastest_greatest.pkl"):
champ_file = open("./champ_data/kraken/" + f, 'rb')
g = pickle.load(champ_file)
champ_file.close()
cppn = neat.nn.FeedForwardNetwork.create(g, self.config)
network = ESNetwork(self.subStrate, cppn, self.params, self.hd)
net = network.create_phenotype_network_nd()
g.fitness = self.evaluate_champ(net, network, r_start, g)
if (g.fitness > champ_fit):
with open("./champ_data/kraken/latest_greatest.pkl",
'wb') as output:
pickle.dump(g, output)
return
def validate_fitness(self):
config = self.config
genomes = neat.Checkpointer.restore_checkpoint(
"./pkl_pops/pop-checkpoint-27").population
self.epoch_len = 233
r_start = self.hs.hist_full_size - self.epoch_len - 1
best_g_fit = 1.0
for idx in genomes:
g = genomes[idx]
cppn = neat.nn.FeedForwardNetwork.create(g, config)
network = ESNetwork(self.subStrate, cppn, self.params, self.hd)
net = network.create_phenotype_network_nd()
g.fitness = self.evaluate(net, network, r_start, g)
if (g.fitness > best_g_fit):
best_g_fit = g.fitness
with open('./champ_data/kraken/latest_greatest.pkl',
'wb') as output:
pickle.dump(g, output)
return
# Create the population and run the XOR task by providing the above fitness function.
def run_pop(self, checkpoint=""):
if (checkpoint == ""):
pop = neat.population.Population(self.config)
else:
pop = neat.Checkpointer.restore_checkpoint(
"./pkl_pops/kraken/pop-checkpoint-" + checkpoint)
checkpoints = neat.Checkpointer(
generation_interval=2,
time_interval_seconds=None,
filename_prefix='./pkl_pops/kraken/pop-checkpoint-')
stats = neat.statistics.StatisticsReporter()
pop.add_reporter(stats)
pop.add_reporter(checkpoints)
pop.add_reporter(neat.reporting.StdOutReporter(True))
pe = neat.ThreadedEvaluator(4, self.eval_fitness)
winner = pop.run(pe.evaluate, self.num_gens)
return winner, stats
# If run as script.
def run_training(self, checkpoint=""):
#print(task.trial_run())
if checkpoint == "":
winner = self.run_pop()[0]
else:
winner = self.run_pop(checkpoint)[0]
print('\nBest genome:\n{!s}'.format(winner))
checkpoint_string = str(self.num_gens - 1)
self.num_gens += self.num_gens
self.run_training(checkpoint_string)
def run_validation(self):
self.validate_fitness()
class PurpleTrader:
#needs to be initialized so as to allow for 62 outputs that return a coordinate
# ES-HyperNEAT specific parameters.
params = {
"initial_depth": 3,
"max_depth": 4,
"variance_threshold": 0.00013,
"band_threshold": 0.00013,
"iteration_level": 3,
"division_threshold": 0.00013,
"max_weight": 5.0,
"activation": "tanh"
}
# Config for CPPN.
config = neat.config.Config(neat.genome.DefaultGenome,
neat.reproduction.DefaultReproduction,
neat.species.DefaultSpeciesSet,
neat.stagnation.DefaultStagnation,
'config_trader')
start_idx = 0
highest_returns = 0
portfolio_list = []
in_shapes = []
out_shapes = []
def __init__(self, hist_depth):
self.hs = HistWorker()
self.hs.combine_binance_frames()
self.hd = hist_depth
print(self.hs.currentHists.keys())
self.end_idx = len(self.hs.hist_shaped[0])
self.but_target = .1
self.inputs = self.hs.hist_shaped.shape[0] * (
self.hs.hist_shaped[0].shape[1])
self.outputs = self.hs.hist_shaped.shape[0]
sign = 1
for ix in range(1, self.outputs + 1):
sign = sign * -1
self.out_shapes.append((0.0 - (sign * .005 * ix), -1.0, -1.0))
for ix2 in range(1, (self.inputs // self.outputs) + 1):
self.in_shapes.append(
(0.0 + (sign * .01 * ix2), 0.0 - (sign * .01 * ix2), 1.0))
self.subStrate = Substrate(self.in_shapes, self.out_shapes)
self.epoch_len = 144
#self.node_names = ['x1', 'y1', 'z1', 'x2', 'y2', 'z2', 'weight']
self.leaf_names = []
#num_leafs = 2**(len(self.node_names)-1)//2
for l in range(len(self.in_shapes[0])):
self.leaf_names.append('leaf_one_' + str(l))
self.leaf_names.append('leaf_two_' + str(l))
#self.leaf_names.append('bias')
def set_portfolio_keys(self, folio):
for k in self.hs.currentHists.keys():
folio.ledger[k] = 0
def get_one_epoch_input(self, end_idx):
master_active = []
for x in range(0, self.hd):
active = []
#print(self.outputs)
for y in range(0, self.outputs):
try:
sym_data = self.hs.hist_shaped[y][end_idx - x]
#print(len(sym_data))
active += sym_data.tolist()
except:
print('error')
master_active.append(active)
#print(active)
return master_active
def evaluate(self, network, es, rand_start, g, verbose=False):
portfolio_start = .05
portfolio = CryptoFolio(portfolio_start, self.hs.coin_dict)
end_prices = {}
buys = 0
sells = 0
if (len(g.connections) > 0.0):
for z in range(rand_start, rand_start + self.epoch_len):
active = self.get_one_epoch_input(z)
network.reset()
for n in range(1, self.hd + 1):
out = network.activate(active[self.hd - n])
for x in range(len(out)):
signals.append(out[x])
#rng = iter(shuffle(rng))
sorted_shit = np.argsort(signals)[::-1]
#print(sorted_shit, len(sorted_shit))
#print(len(sorted_shit), len(key_list))
for x in sorted_shit:
sym = self.hs.coin_dict[x]
#print(out[x])
#try:
if (out[x] < -.5):
#print("selling")
portfolio.sell_coin(
sym, self.hs.currentHists[sym]['close'][z])
#print("bought ", sym)
if (out[x] > .5):
#print("buying")
portfolio.target_amount = .1 + (out[x] * .1)
portfolio.buy_coin(
sym, self.hs.currentHists[sym]['close'][z])
#print("sold ", sym)
#skip the hold case because we just dont buy or sell hehe
if (z == self.epoch_len + rand_start):
end_prices[sym] = self.hs.currentHists[sym]['close'][
self.epoch_len + rand_start]
result_val = portfolio.get_total_btc_value(end_prices)
print(result_val[0], "buys: ", result_val[1], "sells: ",
result_val[2])
ft = result_val[0]
else:
ft = 0.0
return ft
def solve(self, network):
return self.evaluate(network) >= self.highest_returns
def trial_run(self):
r_start = 0
file = open("es_trade_god_cppn_3d.pkl", 'rb')
[cppn] = pickle.load(file)
network = ESNetwork(self.subStrate, cppn, self.params)
net = network.create_phenotype_network_nd()
fitness = self.evaluate(net, network, r_start)
return fitness
def eval_fitness(self, genomes, config):
self.epoch_len = randint(21, 255)
r_start = randint(0 + self.hd, self.hs.hist_full_size - self.epoch_len)
for idx, g in genomes:
[cppn] = create_cppn(g, config, self.leaf_names, ['cppn_out'])
network = ESNetwork(self.subStrate, cppn, self.params)
net = network.create_phenotype_network_nd('training_net.png')
g.fitness = self.evaluate(net, network, r_start, g)
return
class PurpleTrader:
#needs to be initialized so as to allow for 62 outputs that return a coordinate
# ES-HyperNEAT specific parameters.
params = {"initial_depth": 3,
"max_depth": 4,
"variance_threshold": 0.00013,
"band_threshold": 0.00013,
"iteration_level": 3,
"division_threshold": 0.00013,
"max_weight": 5.0,
"activation": "tanh"}
# Config for CPPN.
config = neat.config.Config(neat.genome.DefaultGenome, neat.reproduction.DefaultReproduction,
neat.species.DefaultSpeciesSet, neat.stagnation.DefaultStagnation,
'config_trader')
start_idx = 0
highest_returns = 0
portfolio_list = []
in_shapes = []
out_shapes = []
def __init__(self, hist_depth):
self.hs = HistWorker()
self.hs.combine_frames()
self.hd = hist_depth
print(self.hs.currentHists.keys())
self.end_idx = len(self.hs.currentHists["ETH"])
self.but_target = .1
self.inputs = self.hs.hist_shaped[0].shape[1]
self.outputs = 1
sign = 1
for ix in range(1,self.inputs+1):
sign = sign *-1
self.in_shapes.append((0.0-(sign*.005*ix), -1.0, 0.0+(sign*.005*ix)))
self.out_shapes.append((0.0, 1.0, 0.0))
self.subStrate = Substrate(self.in_shapes, self.out_shapes)
self.epoch_len = 144
#self.node_names = ['x1', 'y1', 'z1', 'x2', 'y2', 'z2', 'weight']
self.leaf_names = []
#num_leafs = 2**(len(self.node_names)-1)//2
for l in range(len(self.in_shapes[0])):
self.leaf_names.append('leaf_one_'+str(l))
self.leaf_names.append('leaf_two_'+str(l))
#self.leaf_names.append('bias')
def set_portfolio_keys(self, folio):
for k in self.hs.currentHists.keys():
folio.ledger[k] = 0
def get_one_epoch_input(self,end_idx):
master_active = []
for x in range(0, self.hd):
active = []
#print(self.outputs)
for y in range(0, self.outputs):
try:
sym_data = self.hs.hist_shaped[y][end_idx-x]
#print(len(sym_data))
active += sym_data.tolist()
except:
print('error')
master_active.append(active)
#print(active)
return master_active
def get_single_symbol_epoch(self, end_idx, symbol_idx):
master_active = []
for x in range(0, self.hd):
try:
sym_data = self.hs.hist_shaped[symbol_idx][end_idx-x]
#print(len(sym_data))
master_active.append(sym_data.tolist())
except:
print('error')
return master_active
def load_net(self, fname):
f = open(fname,'rb')
g = pickle.load(f)
f.close()
[the_cppn] = create_cppn(g, self.config, self.leaf_names, ['cppn_out'])
self.cppn = the_cppn
def run_champs(self):
genomes = os.listdir(os.path.join(os.path.dirname(__file__), 'champs_d2_single'))
fitness_data = {}
best_fitness = 0.0
for g_ix in range(len(genomes)):
genome = self.load_net('./champs_d2_single/'+genomes[g_ix])
start = self.hs.hist_full_size - self.epoch_len
network = ESNetwork(self.subStrate, self.cppn, self.params)
net = network.create_phenotype_network_nd('./champs_visualized2/genome_'+str(g_ix))
fitness = self.evaluate(net, network, start, g_ix, genomes[g_ix])
if fitness > best_fitness:
best_genome = genome
def evaluate(self, network, es, rand_start, g, p_name):
portfolio_start = 1.0
portfolio = CryptoFolio(portfolio_start, self.hs.coin_dict)
end_prices = {}
buys = 0
sells = 0
th = []
with open('./champs_hist2/trade_hist'+p_name + '.txt', 'w') as ft:
ft.write('date,symbol,type,amnt,price,current_balance \n')
for z in range(self.hd, self.hs.hist_full_size -1):
for x in np.random.permutation(self.outputs):
sym = self.hs.coin_dict[x]
active = self.get_single_symbol_epoch(z, x)
network.reset()
for n in range(1, self.hd+1):
out = network.activate(active[self.hd-n])
end_prices[sym] = self.hs.currentHists[sym]['close'][self.hs.hist_full_size-1]
#rng = iter(shuffle(rng))
#print(out[x])
#try:
if(out[0] < -.5):
#print("selling")
did_sell = portfolio.sell_coin(sym, self.hs.currentHists[sym]['close'][z])
if did_sell:
ft.write(str(self.hs.currentHists[sym]['date'][z]) + ",")
ft.write(sym +",")
ft.write('sell,')
ft.write(str(portfolio.ledger[sym])+",")
ft.write(str(self.hs.currentHists[sym]['close'][z])+",")
ft.write(str(portfolio.get_total_btc_value_no_sell(end_prices)[0])+ " \n")
#print("bought ", sym)
elif(out[0] > .5):
did_buy = portfolio.buy_coin(sym, self.hs.currentHists[sym]['close'][z])
if did_buy:
ft.write(str(self.hs.currentHists[sym]['date'][z]) + ",")
ft.write(sym +",")
ft.write('buy,')
ft.write(str(portfolio.target_amount)+",")
ft.write(str(self.hs.currentHists[sym]['close'][z])+",")
ft.write(str(portfolio.get_total_btc_value_no_sell(end_prices)[0])+ " \n")
#print("sold ", sym)
#skip the hold case because we just dont buy or sell heh
result_val = portfolio.get_total_btc_value(end_prices)
print(result_val[0], "buys: ", result_val[1], "sells: ", result_val[2], p_name)
ft = result_val[0]
return ft
def solve(self, network):
return self.evaluate(network) >= self.highest_returns
def report_back(self, portfolio, prices):
print(portfolio.get_total_btc_value(prices))
def trial_run(self):
r_start = 0
file = open("es_trade_god_cppn_3d.pkl",'rb')
[cppn] = pickle.load(file)
network = ESNetwork(self.subStrate, cppn, self.params)
net = network.create_phenotype_network_nd()
fitness = self.evaluate(net, network, r_start)
return fitness
def eval_fitness(self, genomes, config):
r_start = randint(0+self.hd, self.hs.hist_full_size - self.epoch_len)
fitter = genomes[0]
fitter_val = 0.0
for idx, g in genomes:
[cppn] = create_cppn(g, config, self.leaf_names, ['cppn_out'])
network = ESNetwork(self.subStrate, cppn, self.params)
net = network.create_phenotype_network_nd('current_net.png')
g.fitness = self.evaluate(net, network, r_start)
if(g.fitness > fitter_val):
fitter = g
fitter_val = g.fitness
with open('./champs/perpetual_champion_'+str(fitter.key)+'.pkl', 'wb') as output:
pickle.dump(fitter, output)
print("latest_saved")
from hist_service import HistWorker
hs = HistWorker()
hs.combine_polo_frames_vol_sorted(3)
print(next(iter(hs.currentHists.values())).head())
hs.currentHists = {}
hs.combine_binance_frames_vol_sorted(3)
print(next(iter(hs.currentHists.values())).head())
class PaperTrader:
params = {"initial_depth": 3,
"max_depth": 4,
"variance_threshold": 0.00013,
"band_threshold": 0.0000013,
"iteration_level": 3,
"division_threshold": 0.00013,
"max_weight": 8.0,
"activation": "tanh"}
# Config for CPPN.
config = neat.config.Config(neat.genome.DefaultGenome, neat.reproduction.DefaultReproduction,
neat.species.DefaultSpeciesSet, neat.stagnation.DefaultStagnation,
'config_trader')
def __init__(self, ticker_len, start_amount, histdepth):
self.in_shapes = []
self.out_shapes = []
self.trade_hist = {}
self.polo = Poloniex()
self.hist_depth = histdepth
self.ticker_len = ticker_len
self.end_ts = datetime.now()+timedelta(seconds=(ticker_len*24))
self.start_amount = start_amount
self.hs = HistWorker()
self.refresh_data()
self.inputs = self.hs.hist_shaped.shape[0]*(self.hs.hist_shaped[0].shape[1])
self.outputs = self.hs.hist_shaped.shape[0]
#self.make_shapes()
self.folio = CryptoFolio(start_amount, list(self.hs.currentHists.keys()))
self.leaf_names = []
for ix in range(1,self.inputs+1):
sign = sign *-1
self.in_shapes.append((0.0-(sign*.005*ix), -1.0, 0.0+(sign*.005*ix)))
self.out_shapes.append((0.0, 1.0, 0.0))
self.subStrate = Substrate(self.in_shapes, self.out_shapes)
for l in range(len(self.in_shapes[0])):
self.leaf_names.append('leaf_one_'+str(l))
self.leaf_names.append('leaf_two_'+str(l))
self.load_net()
print(self.hs.coin_dict)
self.poloTrader()
def refresh_data(self):
self.hs.pull_polo_live(12*30)
self.hs.combine_live_frames(12*30)
def load_net(self):
#file = open("./champ_gens/thot-checkpoint-13",'rb')
g = neat.Checkpointer.restore_checkpoint("./champ_gens/thot-checkpoint-25")
best_fit = 0.0
for gx in g.population:
if g.population[gx].fitness != None:
if g.population[gx].fitness > best_fit:
bestg = g.population[gx]
g = bestg
#file.close()
[the_cppn] = create_cppn(g, self.config, self.leaf_names, ['cppn_out'])
self.cppn = the_cppn
def make_shapes(self):
self.in_shapes = []
self.out_shapes = []
sign = 1
for ix in range(1,self.outputs+1):
sign = sign *-1
self.out_shapes.append((0.0-(sign*.005*ix), -1.0, -1.0))
for ix2 in range(1,(self.inputs//self.outputs)+1):
self.in_shapes.append((0.0+(sign*.01*ix2), 0.0-(sign*.01*ix2), 1.0))
def reset_tickers(self):
try:
self.tickers = self.polo.returnTicker()
except:
time.sleep(360)
self.reset_tickers()
return
def get_price(self, coin):
return self.tickers[coin]['last']
def get_current_balance(self):
#self.refresh_data()
c_prices = {}
for s in self.hs.currentHists.keys():
if s != 'BTC':
c_prices[s] = self.hs.currentHists[s]['close'].iloc[-1]
return self.folio.get_total_btc_value_no_sell(c_prices)
def get_one_bar_input_2d(self,end_idx=10):
master_active = []
for x in range(0, self.hist_depth):
active = []
#print(self.outputs)
for y in range(0, self.outputs):
sym_data = self.hs.hist_shaped[y][self.hist_depth-x]
#print(len(sym_data))
active += sym_data.tolist()
master_active.append(active)
#print(active)
return master_active
def poloTrader(self):
try:
trade_df = pd.read_json("./live_hist/json_hist.json")
except:
trade_df = pd.DataFrame()
end_prices = {}
active = self.get_one_bar_input_2d()
self.load_net()
sub = Substrate(self.in_shapes, self.out_shapes)
network = ESNetwork(sub, self.cppn, self.params)
net = network.create_phenotype_network_nd('paper_net.png')
net.reset()
for n in range(1, self.hist_depth+1):
out = net.activate(active[self.hist_depth-n])
#print(len(out))
rng = len(out)
#rng = iter(shuffle(rng))
self.reset_tickers()
for x in np.random.permutation(rng):
sym = self.hs.coin_dict[x]
#print(out[x])
try:
if(out[x] < -.5):
p = self.get_price('BTC_'+sym)
print("selling: ", sym)
self.folio.sell_coin(sym, p)
elif(out[x] > .5):
p = self.get_price('BTC_'+sym)
print("buying: ", sym)
self.folio.buy_coin(sym, p)
else:
except:
print("error buying or selling")
#skip the hold case because we just dont buy or sell hehe
end_prices[sym] = self.hs.currentHists[sym]["close"].iloc[-1]
self.trade_hist["date"] = datetime.now()
self.trade_hist["portfoliovalue"] = self.folio.get_total_btc_value_no_sell(end_prices)[0]
self.trade_hist["portfolio"] = self.folio.ledger
self.trade_hist["percentchange"] = ((self.trade_hist["portfoliovalue"] - self.folio.start)/sefl.folio.start)*100
trade_df.append(self.trade_hist)
trade_df.to_json("./live_hist/json_hist.json")
'''
if(self.trade_hist["portfoliovalue"] > self.folio.start *1.1):
self.folio.start = self.folio.get_total_btc_value(end_prices)[0]
'''
if datetime.now() >= self.end_ts:
port_info = self.folio.get_total_btc_value(end_prices)
print("total val: ", port_info[0], "btc balance: ", port_info[1])
return
else:
print(self.get_current_balance())
for t in range(2):
self.refresh_data
p_vals = self.get_current_balance()
print("current value: ", p_vals[0], "current btc holdings: ", p_vals[1])
time.sleep(self.ticker_len/2)
#print(self.folio.ledger)
self.poloTrader()
class PurpleTrader:
#needs to be initialized so as to allow for 62 outputs that return a coordinate
# ES-HyperNEAT specific parameters.
params = {
"initial_depth": 3,
"max_depth": 4,
"variance_threshold": 0.00013,
"band_threshold": 0.00013,
"iteration_level": 3,
"division_threshold": 0.00013,
"max_weight": 8.0,
"activation": "tanh"
}
# Config for CPPN.
config = neat.config.Config(neat.genome.DefaultGenome,
neat.reproduction.DefaultReproduction,
neat.species.DefaultSpeciesSet,
neat.stagnation.DefaultStagnation,
'./configs/config_trader')
start_idx = 0
highest_returns = 0
portfolio_list = []
in_shapes = []
out_shapes = []
def __init__(self, hist_depth):
self.hs = HistWorker()
self.hs.combine_binance_frames_vol_sorted(3)
self.hd = hist_depth
print(self.hs.currentHists.keys())
self.end_idx = len(self.hs.hist_shaped[0])
self.but_target = .1
self.inputs = self.hs.hist_shaped.shape[0] * (
self.hs.hist_shaped[0].shape[1])
self.outputs = self.hs.hist_shaped.shape[0]
self.leaf_names = []
#num_leafs = 2**(len(self.node_names)-1)//2
#self.tree = nDimensionGoldenTree((0.0, 0.0, 0.0), 1.0, 1)
#self.tree.divide_childrens()
self.set_substrate()
self.set_leaf_names()
self.epoch_len = hist_depth
def set_leaf_names(self):
for l in range(len(self.in_shapes[0])):
self.leaf_names.append('leaf_one_' + str(l))
self.leaf_names.append('leaf_two_' + str(l))
#self.leaf_names.append('bias')
def set_substrate(self):
sign = 1
x_increment = 1.0 / self.outputs
y_increment = 1.0 / len(self.hs.hist_shaped[0][0])
for ix in range(self.outputs):
self.out_shapes.append((1.0 - (ix * x_increment), 0.0, -1.0))
for ix2 in range(len(self.hs.hist_shaped[0][0])):
self.in_shapes.append((-1.0 + (ix * x_increment),
1.0 - (ix2 * y_increment), 1.0))
self.subStrate = Substrate(self.in_shapes, self.out_shapes)
def set_portfolio_keys(self, folio):
for k in self.hs.currentHists.keys():
folio.ledger[k] = 0
def get_one_epoch_input(self, end_idx):
master_active = []
for x in range(0, self.hd):
active = []
#print(self.outputs)
for y in range(0, self.outputs):
try:
sym_data = self.hs.hist_shaped[y][end_idx - x]
#print(len(sym_data))
active += sym_data.tolist()
except:
print('error')
master_active.append(active)
#print(active)
return master_active
def load_net(self, fname):
f = open(fname, 'rb')
g = neat.Checkpointer().restore_checkpoint(fname)
bestFit = 0.0
for gx in g.population:
if g.population[gx].fitness != None:
if g.population[gx].fitness > bestFit:
bestg = g.population[gx]
g = bestg
f.close()
[the_cppn] = create_cppn(g, self.config, self.leaf_names, ['cppn_out'])
self.cppn = the_cppn
def load_net_easy(self, g):
[the_cppn] = create_cppn(g, self.config, self.leaf_names, ['cppn_out'])
self.cppn = the_cppn
def run_champs(self):
genomes = neat.Checkpointer.restore_checkpoint(
"./binance_champs_2/tradegod-checkpoint-70").population
fitness_data = {}
best_fitness = 0.0
best_ix = ""
count = 0
for g_ix in genomes:
self.load_net_easy(genomes[g_ix])
start = self.hs.hist_full_size - self.epoch_len
network = ESNetwork(self.subStrate, self.cppn, self.params)
net = network.create_phenotype_network_nd(
'./champs_visualizedd3/genome_' + str(g_ix))
fitness = self.evaluate(net, network, start, genomes[g_ix], g_ix)
if (fitness > best_fitness):
best_fitness = fitness
best_ix = genomes[g_ix]
count += 1
with open(
'./binance_champs_2/perpetual_champion_' + str(g_ix) + '.pkl',
'wb') as output:
pickle.dump(best_ix, output)
def run_champ(self):
genome = neat.Checkpointer.restore_checkpoint(
"./binance_champs_2/tradegod-checkpoint-48").population[5808]
self.load_net_easy(genome)
start = self.hs.hist_full_size - self.epoch_len
network = ESNetwork(self.subStrate, self.cppn, self.params)
net = network.create_phenotype_network_nd(
'./champs_visualizedd3/genome_' + str(5808))
fitness = self.evaluate(net, network, start, genome, 5808)
with open(
'./binance_champs_2/perpetual_champion_' + str(5808) + '.pkl',
'wb') as output:
pickle.dump(genome, output)
def evaluate(self, network, es, rand_start, g, p_name):
portfolio_start = 1.0
portfolio = CryptoFolio(portfolio_start,
list(self.hs.currentHists.keys()))
end_prices = {}
port_ref = portfolio_start
with open('./champs_histd3/port_hist' + str(p_name) + '.txt',
'w') as pt:
pt.write('date,current_balance \n')
with open('./champs_histd3/trade_hist' + str(p_name) + '.txt',
'w') as ft:
ft.write('date,symbol,type,amnt,price,current_balance \n')
for z in range(34, self.hs.hist_full_size - 1):
active = self.get_one_epoch_input(z)
signals = []
network.reset()
for n in range(1, self.hd + 1):
out = network.activate(active[self.hd - n])
for x in range(len(out)):
signals.append(out[x])
sym2 = list(self.hs.currentHists.keys())[x]
end_prices[sym2] = self.hs.currentHists[sym2]['close'][
z]
sorted_shit = np.argsort(signals)[::-1]
rebalance = portfolio_start
#rng = iter(shuffle(rng))
sym = ""
for x in sorted_shit:
sym = list(self.hs.currentHists.keys())[x]
#print(out[x])
#try:
if (out[x] < -.5):
#print("selling")
did_sell = portfolio.sell_coin(
sym, self.hs.currentHists[sym]['close'][z])
if did_sell:
ft.write(
str(self.hs.currentHists[sym]['date'][z]) +
",")
ft.write(sym + ",")
ft.write('sell,')
ft.write(str(portfolio.ledger[sym]) + ",")
ft.write(
str(self.hs.currentHists[sym]['close'][z])
+ ",")
ft.write(
str(
portfolio.get_total_btc_value_no_sell(
end_prices)[0]) + " \n")
#print("bought ", sym)
elif (out[x] > .5):
did_buy = portfolio.buy_coin(
sym, self.hs.currentHists[sym]['close'][z])
if did_buy:
portfolio.target_amount = .1 + (out[x] * .1)
ft.write(
str(self.hs.currentHists[sym]['date'][z]) +
",")
ft.write(sym + ",")
ft.write('buy,')
ft.write(str(portfolio.target_amount) + ",")
ft.write(
str(self.hs.currentHists[sym]['close'][z])
+ ",")
ft.write(
str(
portfolio.get_total_btc_value_no_sell(
end_prices)[0]) + " \n")
#else:
pt.write(str(self.hs.currentHists[sym]['date'][z]) + ",")
#pt.write(str(self.hs.currentHists[sym]['close'][z])+",")
pt.write(
str(
portfolio.get_total_btc_value_no_sell(end_prices)
[0]) + " \n")
#print("sold ", sym)
new_ref = portfolio.get_total_btc_value_no_sell(
end_prices)[0]
'''
if(new_ref > 1.10 * port_ref or new_ref < .9*port_ref):
port_ref = portfolio.get_total_btc_value_no_sell(end_prices)[0]
portfolio.start = port_ref
#skip the hold case because we just dont buy or sell heh
'''
result_val = portfolio.get_total_btc_value(end_prices)
print(result_val[0], "buys: ", result_val[1], "sells: ", result_val[2],
p_name)
fit = result_val[0]
return fit
def solve(self, network):
return self.evaluate(network) >= self.highest_returns
def report_back(self, portfolio, prices):
print(portfolio.get_total_btc_value(prices))
def trial_run(self):
r_start = 0
file = open("es_trade_god_cppn_3d.pkl", 'rb')
[cppn] = pickle.load(file)
network = ESNetwork(self.subStrate, cppn, self.params)
net = network.create_phenotype_network_nd()
fitness = self.evaluate(net, network, r_start)
return fitness
def eval_fitness(self, genomes, config):
r_start = randint(0 + self.hd, self.hs.hist_full_size - self.epoch_len)
fitter = genomes[0]
fitter_val = 0.0
for idx, g in genomes:
[cppn] = create_cppn(g, config, self.leaf_names, ['cppn_out'])
network = ESNetwork(self.subStrate, cppn, self.params)
net = network.create_phenotype_network_nd('current_net.png')
g.fitness = self.evaluate(net, network, r_start)
if (g.fitness > fitter_val):
fitter = g
fitter_val = g.fitness
with open(
'./binance_champs_2/perpetual_champion_' + str(fitter.key) +
'.pkl', 'wb') as output:
pickle.dump(fitter, output)
print("latest_saved")
def __init__(self):
self.hs = HistWorker()
self.end_idx = len(self.hs.currentHists["DASH"])
self.but_target = .1
self.inputs = self.hs.hist_shaped.shape[0]*self.hs.hist_shaped[0].shape[1]
self.outputs = self.hs.hist_shaped.shape[0] # times by three for buy | sell | hodl(pass)
