import tensorflow as tf

import random

import ssbm

import ctypes

import tf_lib as tfl

import util

import ctype_util as ct

with tf.name_scope('input'):

input_states = ct.inputCType(ssbm.GameMemory, [None], "states")

# player 2's controls

input_controls = ct.inputCType(ssbm.SimpleControllerState, [None], "controls")

def feedStateActions(states, actions, feed_dict = None):

return feed_dict

embedFloat = lambda t: tf.reshape(t, [-1, 1])

castFloat = lambda t: embedFloat(tf.cast(t, tf.float32))

def one_hot(size):

return lambda t: tf.one_hot(tf.cast(t, tf.int64), size, 1.0, 0.0)

maxAction = 512 # altf4 says 0x017E

actionSpace = 32

maxCharacter = 32 # should be large enough?

maxJumps = 8 # unused

with tf.variable_scope("embed_action"):

actionHelper = tfl.makeAffineLayer(maxAction, actionSpace)

def embedAction(t):

return actionHelper(one_hot(maxAction)(t))

def rescale(a):

return lambda x: a * x

playerEmbedding = [

]

# TODO: give the tensors some names/scopes

def embedStruct(embedding):

return f

embedPlayer = embedStruct(playerEmbedding)

def embedArray(embed, indices=None):

return f

maxStage = 64 # overestimate

stageSpace = 32

with tf.variable_scope("embed_stage"):

stageHelper = tfl.makeAffineLayer(maxStage, stageSpace)

def embedStage(stage):

return stageHelper(one_hot(maxStage)(stage))

gameEmbedding = [

]

embedGame = embedStruct(gameEmbedding)

embedded_states = embedGame(input_states)

state_size = embedded_states.get_shape()[-1].value

stickEmbedding = [

('x', embedFloat),

('y', embedFloat)

]

embedStick = embedStruct(stickEmbedding)

controllerEmbedding = [

]

embedController = embedStruct(controllerEmbedding)

def embedEnum(enum):

return one_hot(len(enum))

simpleControllerEmbedding = [

('button', embedEnum(ssbm.SimpleButton)),

('stick_MAIN', embedEnum(ssbm.SimpleStick)),

]

embedSimpleController = embedStruct(simpleControllerEmbedding)

#embedded_controls = embedController(train_controls)

embedded_controls = embedSimpleController(input_controls)

control_size = embedded_controls.get_shape()[-1].value

assert(control_size == 7)

with tf.variable_scope("q_net"):

q1 = tfl.makeAffineLayer(state_size + control_size, 512, tf.tanh)

q2 = tfl.makeAffineLayer(512, 1)

def q(states, controls):

return tf.squeeze(q2(q1_layer), name='q'), q1_layer

# pre-computed long-term rewards

rewards = tf.placeholder(tf.float32, [None], name='rewards')

global_step = tf.Variable(0, name='global_step', trainable=False)

with tf.name_scope('train_q'):

qPredictions, q1_layer = q(embedded_states, embedded_controls)

qLosses = tf.squared_difference(qPredictions, rewards)

qLoss = tf.reduce_mean(qLosses)

#trainQ = tf.train.RMSPropOptimizer(0.0001).minimize(qLoss)

train_q = tf.train.AdamOptimizer().minimize(qLoss, global_step=global_step)

# opt = tf.train.AdamOptimizer()

# opt = tf.train.GradientDescentOptimizer(0.0)

# grads_and_vars = opt.compute_gradients(qLoss)

# train_q = opt.apply_gradients(grads_and_vars)

with tf.name_scope('epsilon'):

epsilon = tf.maximum(0.05, 0.7 - tf.cast(global_step, tf.float32) / 5000.0)

def getEpsilon():

return sess.run(epsilon)

#with tf.name_scope('temperature'):

# temperature = 20.0 * 0.5 ** (tf.cast(global_step, tf.float32) / 1000.0) + 1.0

""" no more actor

with tf.variable_scope("actor"):

layers = [state_size, 64]

nls = [tf.tanh] * (len(layers) - 1)

zip_layers = zip(layers[:-1], layers[1:])

applyLayers = [tfl.makeAffineLayer(prev, next, nl) for (prev, next), nl in zip(zip_layers, nls)]

button_layer = tfl.makeAffineLayer(layers[-1], len(ssbm.SimpleButton._fields_), util.compose(tf.nn.softmax, lambda x: x / temperature))

stick_layer = tfl.makeAffineLayer(layers[-1], len(ssbm.SimpleStick._fields_), util.compose(tf.nn.softmax, lambda x: x / temperature))

def applyActor(state):

for f in applyLayers:

state = f(state)

button_state = button_layer(state)

stick_state = stick_layer(state)

return tf.concat(1, [button_state, stick_state])

actor_variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='actor')

#print(actor_variables)

with tf.name_scope("actorQ"):

actions = applyActor(embedded_states)

actorQ = tf.reduce_mean(q(embedded_states, actions)[0])

#trainActor = tf.train.RMSPropOptimizer(0.001).minimize(-actorQ)

# FIXME: is this the right sign?

#trainActor = tf.train.RMSPropOptimizer(0.0001).minimize(-actorQ, var_list=actor_variables)

# trainActor = tf.train.AdamOptimizer().minimize(-actorQ, var_list=actor_variables)

actor_opt = tf.train.AdamOptimizer()

actor_grads_and_vars = opt.compute_gradients(-actorQ, var_list=actor_variables)

trainActor = opt.apply_gradients(actor_grads_and_vars, global_step=global_step)

def deepMap(f, obj):

if isinstance(obj, dict):

return {k : deepMap(f, v) for k, v in obj.items()}

if isinstance(obj, list):

return [deepMap(f, x) for x in obj]

return f(obj)

with tf.name_scope('predict'):

predict_state = inputCType(ssbm.GameMemory, [None], 'state')

#reshaped = deepMap(lambda t: tf.reshape(t, [1]), predict_input_state)

embedded_state = embedGame(predict_state)

predict_action = inputCType(ssbm.SimpleControllerState, [None], 'action')

e

"""

sess = tf.Session()

summaryWriter = tf.train.SummaryWriter('logs/', sess.graph)

summaryWriter.flush()

saver = tf.train.Saver(tf.all_variables())

#def predictAction(state):

# feed_dict = tfl.feedCType(ssbm.GameMemory, 'predict/state', state)

# return sess.run('predict/action:0', feed_dict)

def predictQ(states, controls):

return sess.run(qPredictions, feedStateActions(states, controls))

# TODO: do this within the tf model?

# if we do, then we can add in softmax and temperature

def scoreActions(state):

return predictQ([state] * len(ssbm.simpleControllerStates), ssbm.simpleControllerStates)

# see https://docs.google.com/spreadsheets/d/1JX2w-r2fuvWuNgGb6D3Cs4wHQKLFegZe2jhbBuIhCG8/edit#gid=13

dyingActions = set(range(0xA))

def isDying(player):

return player.action_state in dyingActions

# players tend to be dead for many frames in a row

# here we prune all but the first frame of the death

def processDeaths(deaths):

return util.zipWith(lambda prev, next: (not prev) and next, [False] + deaths[:-1] , deaths)

# from player 2's perspective

def computeRewards(states, reward_halflife = 2.0):

# return util.scanr(lambda r1, r2: r1 + discount * r2, lastQ, damage_dealt)[:-1]

def readFile(filename, states=None, controls=None):

return states, controls

def train(filename, steps=1):

print(sess.run(qLoss, feed_dict))

def save(filename='saves/simpleDQN'):

print("Saving to", filename)

saver.save(sess, filename)

def restore(filename='saves/simpleDQN'):

saver.restore(sess, filename)

def writeGraph():

os.rename('models/simpleDQN.pb.temp', 'models/simpleDQN.pb')

def init():

sess.run(tf.initialize_all_variables())

#train('testRecord0')

#saver.restore(sess, 'saves/simpleDQN')

#writeGraph()