"""Builds an array of dicts that conform to TForce's network specification (see their docs) by mix-and-matching

different network hypers

"""

net, indicators, arbitrage = Box(hypers['net']), hypers['indicators'], hypers['arbitrage']

dense = {

'type': 'dense',

'activation': net.activation,

'l2_regularization': net.l2,

'l1_regularization': net.l1

}

conv2d = {

'type': 'conv2d',

# 'bias': net.bias,

'l2_regularization': net.l2,

'l1_regularization': net.l1

}

lstm = {'type': 'internal_lstm', 'dropout': net.dropout}

dropout = {'type': 'dropout', 'rate': net.dropout}

arr = []

# Pre-layer (TMK only makes sense for LSTM)

if 'depth_pre' in net:

for i in range(net.depth_pre):

size = int(net.width/(net.depth_pre-i+1)) if net.funnel else net.width

arr.append({'size': size, **dense})

if net.dropout: arr.append({**dropout})

# Mid-layer

# TODO figure out how to use LSTM's internals w/ baseline, currently not series-aware. Update: looks like TForce's

# `memory` branch fixes this, when merged to master we can get rid of this `if not`

if not (net.type == 'lstm' and baseline):

if net.type == 'conv2d':

steps_out = hypers['step_window']

for i in range(net.depth_mid):

if net.type == 'lstm':

# arr.append({'size': net.width, 'return_final_state': (i == net.depth-1), **lstm})

arr.append({'size': net.width, **lstm})

continue

# For each Conv2d layer, the window/stride is a function of the step-window size. So `net.window=1` means

# divide the step-window 10 ways; `net.window=2` means divide it 20 ways. This gives us flexibility to

# define window/stride relative to step_window without having to know either. Note, each layer is reduced

# from the prior, so window/stride gets recalculated

step_window = math.ceil(steps_out / (net.window * 10))

step_stride = math.ceil(step_window / net.stride)

# next = (length - window)/stride + 1

steps_out = (steps_out - step_window)/step_stride + 1

# Ensure there's some minimal amount of reduction at the lower levels (else, we get layers that map 1-1

# to next layer). TODO this is ugly, better way?

min_window, min_stride = 3, 2

step_window = max([step_window, min_window])

step_stride = max([step_stride, min_stride])

# This is just my hunch from CNNs I've seen; the filter sizes are much smaller than the downstream denses

# (like 32-64-64 -> 512-256). If anyone has better intuition...

size = max([32, int(net.width / 4)])

# if i == 0: size = int(size / 2) # Most convs have their first layer smaller... right? just the first, or what?

arr.append({

'size': size,

'window': (step_window, 1),

'stride': (step_stride, 1),

**conv2d

})

if net.type == 'conv2d':

arr.append({'type': 'flatten'})

# Post Dense layers

for i in range(net.depth_post):

size = int(net.width / (i + 1)) if net.funnel else net.width

arr.append({'size': size, **dense})

if net.dropout: arr.append({**dropout})

"""First builds up an array of dicts compatible with TForce's network spec. Then passes off to a custom neural

network architecture, rather than using TForce's default LayeredNetwork. The only reason for this is so we can pipe

in the "stationary" inputs after the LSTM/Conv2d layers. Think about it. LTSM/Conv2d are tracking time-series data

(price actions, volume, etc). We don't necessarily want to track our own USD & BTC balances for every time-step.

We _could_, and it _might_ help the agent (I'm not convinced it would); but it actually causes lots of problems

when we go live (eg, right away we take the last 6k time-steps to have a full window, but we don't have any

BTC/USD history for that window. There are other issues). So instead we pipe the stationary inputs into the neural

network downstream, after the time-series layers. Makes more sense to me that way: imagine the conv layers saying

"the price is right, buy!" and then getting handed a note with "you have $0 USD". "Oh.. nevermind..."

"""

layers_spec = build_net_spec(hypers, baseline)

if print_net: pprint(layers_spec)

class CustomNet(LayeredNetwork):

def __init__(self, **kwargs):

super(CustomNet, self).__init__(layers_spec, **kwargs)

def tf_apply(self, x, internals, update, return_internals=False):

series = x['series']

stationary = x['stationary']

x = series

# Apply stationary to the first Dense after the last LSTM. in the case of Baseline, there's no LSTM,

# so apply it to the start

apply_stationary_here = 0

# Find the last LSTM layer, peg to the next layer (a Dense)

for i, layer in enumerate(self.layers):

if isinstance(layer, TForceLayers.InternalLstm) or isinstance(layer, TForceLayers.Flatten):

apply_stationary_here = i + 1

internal_outputs = list()

index = 0

for i, layer in enumerate(self.layers):

layer_internals = [internals[index + n] for n in range(layer.num_internals)]

index += layer.num_internals

if i == apply_stationary_here:

x = tf.concat([x, stationary], axis=1)

x = layer.apply(x, update, *layer_internals)

if not isinstance(x, tf.Tensor):

internal_outputs.extend(x[1])

x = x[0]

if return_internals:

return x, internal_outputs

else:

return x

"""Returns x or `fallback` if it doesn't meet the threshold. Note, if you want to turn a hyper "off" below,

set it to "outside the threshold", rather than 0.

"""

"""Returns 10**-x, or `fallback` if it doesn't meet the threshold. Note, if you want to turn a hyper "off" below,

set it to "outside the threshold", rather than 0.

"""

return {

False: {'baseline_mode': None},

True: {

'baseline': {'type': 'custom'},

'baseline_mode': 'states',

'baseline_optimizer': {

'type': 'multi_step',

'num_steps': flat['optimization_steps'],

'optimizer': {

'type': flat['step_optimizer.type'],

'learning_rate': 10 ** -flat['step_optimizer.learning_rate']

}

},

}

"""This was once a TensorForce environment of its own, when I was using RL to find the best hyper-combo for RL.

I turned from that to Bayesian Optimization, but that's why this is an awkward class of its own - it should be

merged with the `main()` code below.

TODO only tested with ppo_agent. Test with other agents

"""

def __init__(self, cli_args, agent='ppo_agent'):

net_type = cli_args.net_type

hypers_ = hypers[agent].copy()

hypers_.update(hypers['custom'])

hypers_['net.type'] = net_type # set as hard-coded val

hypers_.update(hypers[net_type])

hardcoded = {}

for k, v in hypers_.copy().items():

if type(v) != dict: hardcoded[k] = v

self.hypers = hypers_

self.agent = agent

self.hardcoded = hardcoded

self.cli_args = cli_args

self.conn = data.engine.connect()

self.conn_runs = data.engine_runs.connect()

def close(self):

self.conn.close()

self.conn_runs.close()

def get_hypers(self, actions):

"""

Bit of confusing logic here where I construct a `flat` dict of hypers from the actions - looks like how hypers

are specified above ('dot.key.str': val). Then from that we hydrate it as a proper config dict (hydrated).

Keeping `flat` around because I save the run to the database so can be analyzed w/ a decision tree

(for feature_importances and the like) and that's a good format, rather than a nested dict.

:param actions: the hyperparamters

"""

self.flat = flat = {}

# Preprocess hypers

for k, v in actions.items():

try: v = v.item() # sometimes primitive, sometimes numpy

except Exception: pass

hyper = self.hypers[k]

if 'pre' in hyper:

v = hyper['pre'](v)

flat[k] = v

flat.update(self.hardcoded)

# Post-process hypers (allow for dependency handling, etc)

for k, v in flat.items():

hyper = self.hypers[k]

if type(hyper) == dict and 'post' in hyper:

flat[k] = hyper['post'](v, flat)

# change all a.b=c to {a:{b:c}} (note DotDict class above, I hate and would rather use an off-the-shelf)

main, custom = utils.DotDict({}), utils.DotDict({})

for k, v in flat.items():

obj = main if k in hypers[self.agent] else custom

try:

v = self.hypers[k]['hydrate'](v, self.flat)

if type(v) == dict: obj.update(v)

else: obj[k] = v

except: obj[k] = v

main, custom = main.to_dict(), custom.to_dict()

network = custom_net(custom, print_net=True)

if flat['baseline_mode']:

if type(self.hypers['baseline_mode']) == bool:

main.update(hydrate_baseline(self.hypers['baseline_mode'], flat))

main['baseline']['network_spec'] = custom_net(custom, baseline=True)

# GPU split

session_config = None

gpu_split = self.cli_args.gpu_split

if gpu_split != 1:

fraction = .9 / gpu_split if gpu_split > 1 else gpu_split

session_config = tf.ConfigProto(gpu_options=tf.GPUOptions(per_process_gpu_memory_fraction=fraction))

main['session_config'] = session_config

print('--- Flat ---')

pprint(flat)

print('--- Hydrated ---')

pprint(main)

return flat, main, network

def execute(self, actions):

flat, hydrated, network = self.get_hypers(actions)

env = BitcoinEnv(flat, name=self.agent)

agent = agents_dict[self.agent](

states_spec=env.states,

actions_spec=env.actions,

network_spec=network,

**hydrated

)

env.train_and_test(agent, self.cli_args.n_steps, self.cli_args.n_tests, -1)

step_acc, ep_acc = env.acc.step, env.acc.episode

adv_avg = utils.calculate_score(ep_acc.advantages)

print(flat, f"\nAdvantage={adv_avg}\n\n")

sql = """

insert into runs (hypers, advantage_avg, advantages, uniques, prices, actions, agent, flag)

values (:hypers, :advantage_avg, :advantages, :uniques, :prices, :actions, :agent, :flag)

returning id;

"""

row = self.conn_runs.execute(

text(sql),

hypers=json.dumps(flat),

advantage_avg=adv_avg,

advantages=list(ep_acc.advantages),

uniques=list(ep_acc.uniques),

prices=list(env.prices),

actions=list(step_acc.signals),

agent=self.agent,

flag=self.cli_args.net_type

).fetchone()

if ep_acc.advantages[-1] > 0:

_id = str(row[0])

directory = os.path.join(os.getcwd(), "saves", _id)

filestar = os.path.join(directory, _id)

os.mkdir(directory)

agent.save_model(filestar)

agent.close()

env.close()

return adv_avg

def get_winner(self, id=None):

if id:

sql = "select id, hypers from runs where id=:id"

winner = self.conn_runs.execute(text(sql), id=id).fetchone()

winner = winner.hypers

print(winner)

else:

winner = {}

for k,v in self.hypers.items():

if k not in self.hardcoded:

winner[k] = v['guess']

winner.update(self.hardcoded)

self.hardcoded = winner

if len(X) < 5: return

model = GradientBoostingRegressor()

model_hypers = {

'max_features': [None, 'sqrt', 'log2'],

'max_depth': [None, 10, 20],

'n_estimators': [100, 200, 300],

}

model = GridSearchCV(model, param_grid=model_hypers, cv=5, scoring='neg_mean_squared_error', n_jobs=-1)

model.fit(X, np.squeeze(Y))

feature_imp = sorted(zip(model.best_estimator_.feature_importances_, feat_names), key=lambda x: x[0],

reverse=True)

print('\n\n--- Feature Importances ---\n')

print('\n'.join([f'{x[1]}: {round(x[0],4)}' for x in feature_imp]))

# Handle any specifically-asked for "guesses" first

for i, v in enumerate(y_list):

if v[0] is None:

print("Running guess values")

y_list[i] = loss_fn(x_list[i])

n_pre_samples -= len(x_list)

if n_pre_samples > 0:

for params in np.random.uniform(bounds[:, 0], bounds[:, 1], (n_pre_samples, bounds.shape[0])):

x_list.append(params)

y_list.append(loss_fn(params))

# generate a big number of random hyper combos, pick the best one. The number of combos

# we generate is smaller early on, larger later on. That way the early runs are more random,

# allowing for better exploration, and the later runs give boost a bigger selection to choose

# from, allowing it to be more spot-on

n_experiments = int(1e4 * len(y_list))

best_params, best_score = None, -1000

for params in np.random.uniform(bounds[:, 0], bounds[:, 1], (n_experiments, bounds.shape[0])):

prediction = model.predict([params])[0]

if prediction > best_score:

best_params = params

best_score = prediction

import gp

from sklearn.feature_extraction import DictVectorizer

parser = argparse.ArgumentParser()

parser.add_argument('--guess', type=int, default=-1, help="Run the hard-coded 'guess' values first before exploring")

parser.add_argument('--boost', action="store_true", default=False, help="Use custom gradient-boosting optimization, or bayesian optimization?")

utils.add_common_args(parser)

args = parser.parse_args()

# Encode features

hsearch = HSearchEnv(cli_args=args)

hypers_, hardcoded = hsearch.hypers, hsearch.hardcoded

hypers_ = {k: v for k, v in hypers_.items() if k not in hardcoded}

hsearch.close()

# Build a matrix of features, length = max feature size

max_num_vals = 0

for v in hypers_.values():

l = len(v['vals'])

if l > max_num_vals: max_num_vals = l

empty_obj = {k: None for k in hypers_}

mat = pd.DataFrame([empty_obj.copy() for _ in range(max_num_vals)])

for k, hyper in hypers_.items():

for i, v in enumerate(hyper['vals']):

mat.loc[i,k] = v

mat.ffill(inplace=True)

# Above is Pandas-friendly stuff, now convert to sklearn-friendly & pipe through OneHotEncoder

vectorizer = DictVectorizer()

vectorizer.fit(mat.T.to_dict().values())

feat_names = vectorizer.get_feature_names()

# Map TensorForce actions to GP-compatible `domain`

# instantiate just to get actions (get them from hypers above?)

bounds = []

for k in feat_names:

hyper = hypers_.get(k, False)

if hyper:

bounded, min_, max_ = hyper['type'] == 'bounded', min(hyper['vals']), max(hyper['vals'])

b = [min_, max_] if bounded else [0, 1]

bounds.append(b)

def hypers2vec(obj):

h = dict()

for k, v in obj.items():

if k in hardcoded: continue

if type(v) == bool: h[k] = float(v)

else: h[k] = v or 0.

return vectorizer.transform(h).toarray()[0]

def vec2hypers(vec):

# Reverse the encoding

# https://stackoverflow.com/questions/22548731/how-to-reverse-sklearn-onehotencoder-transform-to-recover-original-data

# https://github.com/scikit-learn/scikit-learn/issues/4414

reversed = vectorizer.inverse_transform([vec])[0]

obj = {}

for k, v in reversed.items():

if '=' not in k:

obj[k] = v

continue

if k in obj: continue # we already handled this x=y logic (below)

# Find the winner (max) option for this key

score, attr, val = v, k.split('=')[0], k.split('=')[1]

for k2, score2 in reversed.items():

if k2.startswith(attr + '=') and score2 > score:

score, val = score2, k2.split('=')[1]

obj[attr] = val

# Bools come in as floats. Also, if the result is False they don't come in at all! So we start iterate

# hypers now instead of nesting this logic in reversed-iteration above

for k, v in hypers_.items():

if v['type'] == 'bool':

obj[k] = bool(round(obj.get(k, 0.)))

return obj

# Specify the "loss" function (which we'll maximize) as a single rl_hsearch instantiate-and-run

def loss_fn(params):

hsearch = HSearchEnv(cli_args=args)

reward = hsearch.execute(vec2hypers(params))

hsearch.close()

return [reward]

guess_i = 0

while True:

# Every iteration, re-fetch from the database & pre-train new model. Acts same as saving/loading a model to disk,

# but this allows to distribute across servers easily

conn_runs = data.engine_runs.connect()

sql = "select hypers, advantages, advantage_avg from runs where flag=:f"

runs = conn_runs.execute(text(sql), f=args.net_type).fetchall()

conn_runs.close()

X, Y = [], []

for run in runs:

X.append(hypers2vec(run.hypers))

Y.append([utils.calculate_score(run.advantages)])

boost_model = print_feature_importances(X, Y, feat_names)

if args.guess != -1:

guess = {k: v['guess'] for k, v in hypers_.items()}

guess.update(utils.guess_overrides[args.guess][guess_i])

loss_fn(hypers2vec(guess))

guess_i += 1

if guess_i > len(utils.guess_overrides[args.guess])-1:

args.guess = -1 # start on GP

continue

if args.boost:

print('Using gradient-boosting')

boost_optimization(

model=boost_model,

loss_fn=loss_fn,

bounds=np.array(bounds),

x_list=X,

y_list=Y

)

else:

# Evidently duplicate values break GP. Many of these are ints, so they're definite duplicates. Either way,

# tack on some small epsilon to make them different (1e-6 < gp.py's min threshold, make sure that #'s not a

# problem). I'm concerned about this since many hypers can go below that epislon (eg learning-rate).

for x in X:

for i, v in enumerate(x):

x[i] += np.random.random() * 1e-6

gp.bayesian_optimisation2(

loss_fn=loss_fn,

bounds=np.array(bounds),

x_list=X,

y_list=Y