class PurpleTrader:
#needs to be initialized so as to allow for 62 outputs that return a coordinate
# ES-HyperNEAT specific parameters.
params = {
"initial_depth": 3,
"max_depth": 4,
"variance_threshold": 0.00013,
"band_threshold": 0.00013,
"iteration_level": 3,
"division_threshold": 0.00013,
"max_weight": 3.0,
"activation": "tanh"
}
# Config for CPPN.
config = neat.config.Config(neat.genome.DefaultGenome,
neat.reproduction.DefaultReproduction,
neat.species.DefaultSpeciesSet,
neat.stagnation.DefaultStagnation,
'config_trader')
start_idx = 0
highest_returns = 0
portfolio_list = []
rand_start = 0
in_shapes = []
out_shapes = []
def __init__(self, hist_depth):
self.hs = HistWorker()
self.hs.combine_polo_usd_frames()
self.hd = hist_depth
print(self.hs.currentHists.keys())
self.end_idx = len(self.hs.hist_shaped[0])
self.but_target = .1
self.inputs = self.hs.hist_shaped.shape[0] * (
self.hs.hist_shaped[0].shape[1])
self.outputs = len(self.hs.coin_dict)
print(self.inputs, self.outputs)
self.epoch_len = 144
#self.node_names = ['x1', 'y1', 'z1', 'x2', 'y2', 'z2', 'weight']
self.leaf_names = []
#num_leafs = 2**(len(self.node_names)-1)//2
self.initial_depth_tree = nDimensionTree([0.0, 0.0, 0.0], 1.0, 0)
self.divide_to_depth(self.initial_depth_tree,
self.initial_depth_tree.lvl,
self.params["initial_depth"])
self.set_substrate()
self.set_leaf_names()
def divide_to_depth(self, tree, current_level, desired_depth):
if current_level == desired_depth:
return
else:
tree.divide_childrens()
current_level += 1
for i in tree.cs:
self.divide_to_depth(i, current_level, desired_depth)
def set_leaf_names(self):
for l in range(len(self.in_shapes[0])):
self.leaf_names.append('leaf_one_' + str(l))
self.leaf_names.append('leaf_two_' + str(l))
#self.leaf_names.append('bias')
def set_substrate(self):
sign = 1
x_increment = 1.0 / self.outputs
y_increment = 1.0 / len(self.hs.hist_shaped[0][0])
for ix in range(self.outputs):
self.out_shapes.append((1.0 - (ix * x_increment), 0.0, -1.0))
for ix2 in range(self.inputs // self.outputs):
if (ix2 >= len(self.initial_depth_tree.cs) - 1):
treex = ix2 - len(self.initial_depth_tree.cs) - 1
else:
treex = ix2
center = self.initial_depth_tree.cs[treex]
self.in_shapes.append((center.coord[0] + (ix * x_increment),
center.coord[1] - (ix2 * y_increment),
center.coord[2] + .5))
self.subStrate = Substrate(self.in_shapes, self.out_shapes)
def set_portfolio_keys(self, folio):
for k in self.hs.currentHists.keys():
folio.ledger[k] = 0
def get_one_epoch_input(self, end_idx):
master_active = []
for x in range(0, self.hd):
active = []
for y in range(0, self.outputs):
try:
sym_data = self.hs.hist_shaped[y][end_idx - x]
active += sym_data.tolist()
except:
print('error')
master_active.append(active)
return master_active
def evaluate(self, g, config):
rand_start = self.rand_start
[cppn] = create_cppn(g, config, self.leaf_names, ['cppn_out'])
net = ESNetwork(self.subStrate, cppn, self.params)
network = net.create_phenotype_network_nd()
portfolio_start = 1.0
key_list = list(self.hs.currentHists.keys())
portfolio = CryptoFolio(portfolio_start, self.hs.coin_dict)
end_prices = {}
buys = 0
sells = 0
if (len(g.connections) > 0.0):
for z in range(rand_start, rand_start + self.epoch_len):
active = self.get_one_epoch_input(z)
signals = []
network.reset()
for n in range(1, self.hd + 1):
out = network.activate(active[self.hd - n])
for x in range(len(out)):
signals.append(out[x])
#rng = iter(shuffle(rng))
sorted_shit = np.argsort(signals)[::-1]
for x in sorted_shit:
sym = self.hs.coin_dict[x]
#print(out[x])
#try:
if (out[x] < -.5):
#print("selling")
portfolio.sell_coin(
sym, self.hs.currentHists[sym]['close'][z])
#print("bought ", sym)
if (out[x] > .5):
#print("buying")
portfolio.target_amount = .1 + (out[x] * .1)
portfolio.buy_coin(
sym, self.hs.currentHists[sym]['close'][z])
#print("sold ", sym)
#skip the hold case because we just dont buy or sell hehe
if (z > self.epoch_len + rand_start - 2):
end_prices[sym] = self.hs.currentHists[sym]['close'][z]
result_val = portfolio.get_total_btc_value(end_prices)
print(result_val[0], "buys: ", result_val[1], "sells: ",
result_val[2])
ft = result_val[0]
else:
ft = 0.0
return ft
def eval_fitness(self, genomes, config):
min_batch_size = (self.hs.hist_full_size - self.hd) // 5
max_batch_size = (self.hs.hist_full_size - self.hd) // 2
self.epoch_len = randint(min_batch_size, max_batch_size)
self.rand_start = randint(0 + self.hd,
self.hs.hist_full_size - self.epoch_len)
runner = neat.ParallelEvaluator(8, self.evaluate)
runner.evaluate(genomes, config)
class PurpleTrader:
#needs to be initialized so as to allow for 62 outputs that return a coordinate
# ES-HyperNEAT specific parameters.
params = {
"initial_depth": 3,
"max_depth": 4,
"variance_threshold": 0.00013,
"band_threshold": 0.00013,
"iteration_level": 3,
"division_threshold": 0.00013,
"max_weight": 8.0,
"activation": "tanh"
}
# Config for CPPN.
config = neat.config.Config(neat.genome.DefaultGenome,
neat.reproduction.DefaultReproduction,
neat.species.DefaultSpeciesSet,
neat.stagnation.DefaultStagnation,
'config_trader')
start_idx = 0
highest_returns = 0
portfolio_list = []
def __init__(self, hist_depth, num_gens, gen_count=1):
self.hd = hist_depth
if gen_count != 1:
self.num_gens = num_gens
else:
self.num_gens = gen_count + num_gens
self.gen_count = gen_count
self.refresh()
def refresh(self):
self.in_shapes = []
self.out_shapes = []
self.hs = HistWorker()
self.hs.pull_polo_usd(144)
self.hs.combine_polo_usd_frames()
print(self.hs.currentHists.keys())
self.end_idx = len(self.hs.hist_shaped[0])
self.but_target = .1
self.inputs = self.hs.hist_shaped.shape[0] * (
self.hs.hist_shaped[0].shape[1])
self.outputs = self.hs.hist_shaped.shape[0]
sign = 1
for ix in range(1, self.outputs + 1):
sign = sign * -1
self.out_shapes.append((0.0 - (sign * .005 * ix), 0.0, -1.0))
for ix2 in range(1, (self.inputs // self.outputs) + 1):
self.in_shapes.append(
(0.0 + (sign * .01 * ix2), 0.0 - (sign * .01 * ix2), 0.0))
self.subStrate = Substrate(self.in_shapes, self.out_shapes)
#self.leaf_names.append('bias')
def set_portfolio_keys(self, folio):
for k in self.hs.currentHists.keys():
folio.ledger[k] = 0
def get_one_epoch_input(self, end_idx):
master_active = []
for x in range(0, self.hd):
active = []
#print(self.outputs)
for y in range(0, self.outputs):
try:
sym_data = self.hs.hist_shaped[y][end_idx - x]
#print(len(sym_data))
active += sym_data.tolist()
except:
print('error')
master_active.append(active)
#print(active)
return master_active
def evaluate(self, network, es, rand_start, g, verbose=False):
portfolio_start = 1.0
portfolio = CryptoFolio(portfolio_start, self.hs.coin_dict, "USDT")
end_prices = {}
buys = 0
sells = 0
for z in range(rand_start, rand_start + self.epoch_len):
#TODO add comments to clarify all the
#shit im doing here
active = self.get_one_epoch_input(z)
buy_signals = []
buy_syms = []
sell_syms = []
sell_signals = []
network.reset()
for n in range(1, self.hd + 1):
network.activate(active[self.hd - n])
out = network.activate(active[0])
for x in range(len(out)):
if (z > (self.epoch_len + rand_start) - 2):
sym = self.hs.coin_dict[x]
end_prices[sym] = self.hs.currentHists[sym]['close'][
self.epoch_len + rand_start]
if (out[x] > .5):
buy_signals.append(out[x])
buy_syms.append(self.hs.coin_dict[x])
if (out[x] < -.5):
sell_signals.append(out[x])
sell_syms.append(self.hs.coin_dict[x])
#rng = iter(shuffle(rng))
sorted_buys = np.argsort(buy_signals)[::-1]
sorted_sells = np.argsort(sell_signals)
#print(len(sorted_shit), len(key_list))
for x in sorted_sells:
sym = sell_syms[x]
portfolio.sell_coin(sym, self.hs.currentHists[sym]['close'][z])
for x in sorted_buys:
sym = buy_syms[x]
portfolio.target_amount = .1 + (out[x] * .1)
portfolio.buy_coin(sym, self.hs.currentHists[sym]['close'][z])
result_val = portfolio.get_total_btc_value(end_prices)
print(g.key, " : ")
print(result_val[0], "buys: ", result_val[1], "sells: ", result_val[2])
if result_val[1] == 0:
ft = .7
else:
ft = result_val[0]
return ft
def solve(self, network):
return self.evaluate(network) >= self.highest_returns
def trial_run(self):
r_start = 0
file = open("es_trade_god_cppn_3d.pkl", 'rb')
[cppn] = pickle.load(file)
network = ESNetwork(self.subStrate, cppn, self.params)
net = network.create_phenotype_network_nd()
fitness = self.evaluate(net, network, r_start)
return fitness
def eval_fitness(self, genomes, config):
self.epoch_len = 89
r_start = randint(0 + self.hd, self.hs.hist_full_size - self.epoch_len)
r_start_2 = self.hs.hist_full_size - self.epoch_len - 1
best_g_fit = 0.0
champ_counter = self.gen_count % 10
#img_count = 0
for idx, g in genomes:
cppn = neat.nn.FeedForwardNetwork.create(g, config)
network = ESNetwork(self.subStrate, cppn, self.params, self.hd)
net = network.create_phenotype_network_nd()
train_ft = self.evaluate(net, network, r_start, g)
validate_ft = self.evaluate(net, network, r_start_2, g)
g.fitness = (train_ft + validate_ft) / 2
if (g.fitness > best_g_fit):
best_g_fit = g.fitness
with open(
"./champ_data/latest_greatest" + str(champ_counter) +
".pkl", 'wb') as output:
pickle.dump(g, output)
#img_count += 1
if (champ_counter == 0):
self.refresh()
self.compare_champs()
self.gen_count += 1
return
def compare_champs(self):
self.epoch_len = 233
r_start = self.hs.hist_full_size - self.epoch_len - 1
champ_current = open("./champ_data/latest_greatest.pkl", 'rb')
g = pickle.load(champ_current)
champ_current.close()
cppn = neat.nn.FeedForwardNetwork.create(g, self.config)
network = ESNetwork(self.subStrate, cppn, self.params, self.hd)
net = network.create_phenotype_network_nd()
champ_fit = self.evaluate(net, network, r_start, g)
for f in os.listdir("./champ_data"):
if (f != "lastest_greatest.pkl"):
champ_file = open("./champ_data/" + f, 'rb')
g = pickle.load(champ_file)
champ_file.close()
cppn = neat.nn.FeedForwardNetwork.create(g, self.config)
network = ESNetwork(self.subStrate, cppn, self.params, self.hd)
net = network.create_phenotype_network_nd()
g.fitness = self.evaluate(net, network, r_start, g)
if (g.fitness > champ_fit):
with open("./champ_data/latest_greatest.pkl",
'wb') as output:
pickle.dump(g, output)
return
def validate_fitness(self):
config = self.config
genomes = neat.Checkpointer.restore_checkpoint(
"./pkl_pops/pop-checkpoint-27").population
self.epoch_len = 233
r_start = self.hs.hist_full_size - self.epoch_len - 1
best_g_fit = 1.0
for idx in genomes:
g = genomes[idx]
cppn = neat.nn.FeedForwardNetwork.create(g, config)
network = ESNetwork(self.subStrate, cppn, self.params, self.hd)
net = network.create_phenotype_network_nd()
g.fitness = self.evaluate(net, network, r_start, g)
if (g.fitness > best_g_fit):
best_g_fit = g.fitness
with open('./champ_data/latest_greatest.pkl', 'wb') as output:
pickle.dump(g, output)
return
# Create the population and run the XOR task by providing the above fitness function.
def run_pop(self, checkpoint=""):
if (checkpoint == ""):
pop = neat.population.Population(self.config)
else:
pop = neat.Checkpointer.restore_checkpoint(
"./pkl_pops/pop-checkpoint-" + checkpoint)
checkpoints = neat.Checkpointer(
generation_interval=2,
time_interval_seconds=None,
filename_prefix='./pkl_pops/pop-checkpoint-')
stats = neat.statistics.StatisticsReporter()
pop.add_reporter(stats)
pop.add_reporter(checkpoints)
pop.add_reporter(neat.reporting.StdOutReporter(True))
winner = pop.run(self.eval_fitness, self.num_gens)
return winner, stats
# If run as script.
def run_training(self, checkpoint=""):
#print(task.trial_run())
if checkpoint == "":
winner = self.run_pop()[0]
else:
winner = self.run_pop(checkpoint)[0]
print('\nBest genome:\n{!s}'.format(winner))
checkpoint_string = str(self.num_gens - 1)
self.num_gens += self.num_gens
self.run_training(checkpoint_string)
def run_validation(self):
self.validate_fitness()
