print("complie: _train")
_train = K.function(train_ins, [train_loss], updates=updates)
print("complie: _train_with_acc")
_train_with_acc = K.function(train_ins, [train_loss, train_accuracy], updates=updates)
print("complie: _predict")
_predict = K.function(predict_ins, [y_test], updates=state_updates)
print("complie: _test")
_test = K.function(test_ins, [test_loss])
print("complie: _test_with_acc")
_test_with_acc = K.function(test_ins, [test_loss, test_accuracy])
model = Sequential()
model.class_mode = "categorical"
model._train = _train
model._train_with_acc = _train_with_acc
model._predict = _predict
model._test = _test
model._test_with_acc = _test_with_acc
# Train the model each generation and show predictions against the validation dataset
for iteration in range(1, 200):
print()
print("-" * 50)
print("Iteration", iteration)
model.fit(
D_X_train, D_y_train, batch_size=BATCH_SIZE, nb_epoch=1, validation_data=(D_X_val, D_y_val), show_accuracy=True
)
###
# Select 10 samples from the validation set at random so we can visualize errors
for i in range(10):
nb_epoch = 125
model = Sequential()
model.add(Dense(25, input_shape=(13, )))
model.add(Activation('linear'))
model.add(Dropout(0.10))
model.add(Dense(40))
model.add(Activation('relu'))
model.add(Dropout(0.05))
# model.add(Dense(25))
# model.add(Activation('linear'))
# model.add(Dropout(0.05)) # this layer doesn't add much
model.add(Dense(1))
model.add(Activation('relu'))
rms = RMSprop()
# ada = Adagrad()
model.compile(loss='mean_squared_error', optimizer=rms)
model.fit(X,
y,
batch_size=batch_size,
nb_epoch=nb_epoch,
verbose=2,
validation_data=(X_test, y_test))
for i in range(X_test.shape[0]):
xn = X_test[i]
yn = y_test[i]
print yn, model._predict([xn])[0][0]
X_test, y_test = getTestData()
batch_size = 8
nb_epoch = 125
model = Sequential()
model.add(Dense(25, input_shape=(13,)))
model.add(Activation('linear'))
model.add(Dropout(0.10))
model.add(Dense(40))
model.add(Activation('relu'))
model.add(Dropout(0.05))
# model.add(Dense(25))
# model.add(Activation('linear'))
# model.add(Dropout(0.05)) # this layer doesn't add much
model.add(Dense(1))
model.add(Activation('relu'))
rms = RMSprop()
# ada = Adagrad()
model.compile(loss='mean_squared_error', optimizer=rms)
model.fit(X, y, batch_size=batch_size, nb_epoch=nb_epoch, verbose=2, validation_data=(X_test, y_test))
for i in range(X_test.shape[0]):
xn = X_test[i]
yn = y_test[i]
print yn, model._predict([xn])[0][0]
model.load_weights('/Users/Aran/Downloads/cross_validation/1000_timestep_chunks/gru_my_weights_iter1000')
#loads the training data
X_train = pickle.load(open("/Users/Aran/Downloads/cross_validation/1000_timestep_chunks/vsoma_train_x.p","rb"))
print (X_train.shape)
print (X_train[:3].shape) #shape will be (3, 1000, 2022)
#this just picks the first 3000 timesteps (assuming the data
#has been processed where each training example consists of 1000 timesteps), which is arbitrarily chosen,
#but if you want to change it to just 1000 timesteps simply replace :3 with :1, or if you want to do
#2000 timesteps replace :3 with :2, etc.
startSeed = X_train[:3]
#reshapes it to be a 1 by 3000 (num_timesteps) by 2022 (num_neurons) tensor (Keras requires its inputs to be a tensor).
#If you wanted the number of timesteps to be 1000, 2000, 4000 etc, simply change the 3000 to your desired number
#of timesteps (has to be a multiple of 1000).
seedSeq = np.reshape(startSeed, (1, 3000, 2022))
print (seedSeq.shape)
#if we consider at each timestep t as a vector of 2022 (num_neurons) by 1 (representing the voltages of the 2022 neurons at timestep t),
#then, for each such vector at any timestep t (where in our case t goes from timestep 0 to 3000), then this predicts the voltages
#for timesteps t+1, given the ground truth voltages for timesteps t, t-1, ..., 0. Thus, if t goes from
#0 to 3000, then this predicts the voltages of all 2022 neurons from timesteps 1 to 3001, where at any timestep.
seedSeqNew = model._predict(seedSeq)
print (seedSeqNew.shape)
print (seedSeqNew[0].shape)
output = seedSeqNew[0]
output = output.T
print (output.shape)
pickle.dump(output, open("gru_output.p", "wb")) #saves the predicted voltages as a 2022 (num_neurons) by num_timesteps array
print ('Finished output eval.')
class RL_NN:
model = None
def __init__(self, configuration: Dictionary, env: Environment):
self.params = configuration
self.log = self.params.log
self.env = env
self.model = None
# This structure is used by the onehot encoder.
all_substates = list(itertools.chain(*self.params.states_list))
self.position = {}
for idx, element in enumerate(all_substates):
self.position[element] = idx
self.callback_args = {}
if self.params.tensorboard is True:
self.log.info('Using TensorBoard')
self.tensorboard = TensorBoard(
log_dir=self.params.tbdir,
histogram_freq=0, write_graph=True, write_images=False)
self.callback_args = {'callbacks': self.tensorboard}
def create_model(self) -> Sequential:
self.model = Sequential()
# Input layer
self.model.add(
InputLayer(batch_input_shape=(None, self.params.num_substates)))
first_layer = True
# Create all the layers
for num_cells in self.params.deep_qnet.hidden_layers:
if first_layer:
self.model.add(Dense(
num_cells,
input_shape=(self.params.num_substates,),
activation=self.params.deep_qnet.activation))
first_layer = False
else:
self.model.add(Dense(num_cells,
activation=self.params.deep_qnet.activation))
# Output Layer
last_layer_cells = self.params.deep_qnet.hidden_layers[-1]
self.model.add(
Dense(self.params.num_actions, input_shape=(last_layer_cells,),
activation='linear'))
self.compile_model()
if self.params.debug and self.params.log_level > 2:
self.log.debug('Model Summary')
self.model.summary()
return self.model
def compile_model(self):
self.model.compile(
loss=self.params.deep_qnet.loss,
optimizer=Adam(lr=self.params.deep_qnet.lr),
metrics=self.params.deep_qnet.metrics)
return self.model
def do_learn(self, episode, episode_step, memory) -> (float, float):
""" perform minibatch learning or experience replay """
self.log.debug('Time to learn')
loss = 0.
mae = 0.
if self.params.experience_replay is True:
loss, mae = self.experience_replay(memory)
else:
loss, mae = self.minibatch_learn(memory)
return loss, mae
def minibatch_learn(self, memory):
"""
MiniBatch Learning routine.
:param memory:
:return: loss and mae
"""
mem_size = len(memory)
if mem_size < self.params.batch_size:
self.log.debug('Not enough samples for minibatch learn, skipping')
return 0.0, 0.0
self.log.debug('Minibatch learn')
mini_batch = np.empty(shape=(0, 5), dtype=np.int32)
for i in range(mem_size - self.params.batch_size - 1, mem_size - 1):
mini_batch = np.append(
mini_batch,
np.asarray(memory[i]).astype(int).reshape(1, -1),
axis=0)
nn_input, nn_output = self.prepare_nn_data(mini_batch)
h = self.model.fit(
nn_input, nn_output,
epochs=1, verbose=0, batch_size=self.params.batch_size,
**self.callback_args)
return h.history['loss'][0], h.history['mae'][0]
def experience_replay(self, memory):
"""
Primarily from: https://github.com/edwardhdlu/q-trader
:param memory:
:return: loss and mae.
"""
if len(memory) <= self.params.exp_batch_size:
self.log.debug(' Not enough samples in experience memory')
return 0., 0.
mini_batch = random.sample(memory,
self.params.exp_batch_size)
nn_input, nn_output = self.prepare_nn_data(mini_batch)
h = self.model.fit(
nn_input, nn_output,
epochs=1, verbose=0, batch_size=self.params.exp_batch_size,
**self.callback_args)
self.log.debug(' Learnt exp. batch, loss/mae: {:.2f}/{:.2f}'.format(
h.history['loss'][0], h.history['mae'][0]))
return h.history['loss'][0], h.history['mae'][0]
def prepare_nn_data(self, mini_batch):
"""
Shape the input and output (supervised labels) to the network from a
minibatch o previous experiences.
:param mini_batch: array of tuples with states, actions, rewards,
next states and done values.
:return: input and output to the network.
"""
nn_input = np.empty((0, self.params.num_substates), dtype=np.int32)
nn_output = np.empty((0, self.params.num_actions))
for state, action, reward, next_state, done in mini_batch:
y = self.prepare_nn_output(state, action, reward, next_state, done)
nn_input = np.append(
nn_input,
self.onehot(
self.env.states.name(state),
self.env.states.state_list),
axis=0)
nn_output = np.append(nn_output, y, axis=0)
return nn_input, nn_output
def prepare_nn_output(self, state, action, reward, next_state, done):
"""
Feed forward the current state to the network to get the output and
reformat it to set the reward associated to the action taken and thus
learn that state -> reward association for that action.
:param state: the state
:param action: action
:param reward: reward
:param next_state: next state
:param done: if loop has ended
:return: an array of `n` values, where `n` is the nr of actions.
"""
target = reward
if not done:
target = reward + self.params.gamma * self.predict_value(
next_state)
labeled_output = self.model._predict(
self.onehot(
self.env.states.name(state),
self.env.states.state_list))[0]
labeled_output[action] = target
y = labeled_output.reshape(-1, self.params.num_actions)
return y
def infer_strategy(self) -> list:
"""
Get the defined strategy from the weights of the model.
:return: strategy matrix
"""
strategy = [
np.argmax(
self.model._predict(
self.onehot(
self.env.states.name(state),
self.env.states.state_list))[0])
for state in range(self.params.num_states)
]
return strategy
def onehot(self, state_name: str, state_list: List[str]):
"""
OneHot (special) encoding.
Considering the total number of substates to represent (the nr. of
possible values that each `state` can be assigned to, like, for
example: the state `GAIN` reflects whether I'm currently gaining money
or not, can be in substates `YES` or `NOT`), this one hot encoding
assigns a position to each of them. The value will be 1 if it's set
or 0 if it's not set.
A two-states scenario with
- `State1` valued as `A` or `B`, and
- `State2` valued as `C` or `D`,
will generate 2^2 possible combinations:
- A_C,
- A_D,
- B_C and
- B_D.
To encode this situation 4 bits will be used and the
encodings will be:
- [1010],
- [1001],
- [0110],
- [0101].
Where first position says if substate `A` is set or not, and so on...
"""
enc = [0] * self.params.num_substates
for substate in state_name.split('_'):
enc[self.position[substate]] = 1
self.log.debug(' state: {}'.format(state_name))
self.log.debug(' encod: {}'.format(enc))
return np.array(enc).reshape(1, -1)
def predict(self, state, relax=True) -> int:
"""
Produces a prediction.
Relax controls whether to allow random output instead of MAX value
at output layer. Relax MUST set to FALSE when called once TRAINED.
"""
prediction = int(np.argmax(
self.model._predict(
self.onehot(
self.env.states.name(state),
self.env.states.state_list))))
if relax is not True:
return prediction
elif np.random.random() > self.params.rnd_output_prob:
return prediction
else:
return np.random.randint(0, self.params.num_actions)
def predict_value(self, state):
return np.max(
self.model._predict(
self.onehot(
self.env.states.name(state),
self.env.states.state_list)))
#
# Saving and loading the model
#
def save_model(self, model, results):
self.log.info('Saving model, weights and results.')
if self.params.output is not None:
fname = self.params.output
else:
fname = 'rl_model_' + splitext(
basename(self.params.forecast_file))[0]
model_name = valid_output_name(fname, self.params.models_dir, 'json')
# serialize model to JSON
model_json = model.to_json()
with open(model_name, 'w') as json_file:
json_file.write(model_json)
self.log.info(' Model: {}'.format(model_name))
# Serialize weights to HDF5
weights_name = model_name.replace('.json', '.h5')
model.save_weights(weights_name)
self.log.info(' Weights: {}'.format(weights_name))
# Save also the results table
results_name = model_name.replace('.json', '.csv')
results.to_csv(results_name,
sep=',',
index=False,
header=True,
float_format='%.2f')
self.log.info(' Results: {}'.format(results_name))
def load_model(self, model_basename):
"""
load json and create model
:param model_basename: model basename without extension. 'h5' and
'json' will be added
:return: the loaded model
"""
json_file = open('{}.json'.format(model_basename), 'r')
loaded_model_json = json_file.read()
json_file.close()
self.model = model_from_json(loaded_model_json)
self.log.info("Loaded model from disk: {}.json".format(model_basename))
# load weights into new model
self.model.load_weights('{}.h5'.format(model_basename))
self.log.info("Loaded weights from disk: {}.h5".format(model_basename))
return self.model
import numpy as np from keras.layers.convolutional import Convolution1D, MaxPooling1D, Convolution2D, MaxPooling2D from keras.models import Sequential # model = Sequential() # model.add(Convolution1D(input_dim=32, nb_filter=8, filter_length=3, subsample_length=2)) # model.compile(loss='mse', optimizer='sgd') # X = np.random.random((100, 20, 32)) # out = model._predict(X) # print out.shape # model = Sequential() # model.add(Convolution1D(input_dim=32, nb_filter=8, filter_length=3)) # model.add(MaxPooling1D()) # model.compile(loss='mse', optimizer='sgd') # X = np.random.random((100, 20, 32)) # out = model._predict(X) # print out.shape model = Sequential() model.add(Convolution2D(nb_filter=8, stack_size=3, nb_row=3, nb_col=3, border_mode='same')) # model.add(MaxPooling2D()) model.compile(loss='mse', optimizer='sgd') X = np.random.random((20, 3, 32, 1)) out = model._predict(X) print out.shape
model.add(Dropout(0.15))
# model.add(Dense(30, init='uniform'))
# model.add(Activation('sigmoid'))
# model.add(Dropout(0.15))
model.add(Dense(2))
model.add(Activation('softmax'))
rms = RMSprop()
model.compile(loss='binary_crossentropy', optimizer=rms)
model.fit(X, y, batch_size=batch_size, nb_epoch=nb_epoch, verbose=2, validation_data=(X_test, y_test))
count = 0
for i in range(X_test.shape[0]):
ny = 0 if y_test[i][0] > y_test[i][1] else 1
ny_test = model._predict([X_test[i]])[0]
ny_test = 0 if ny_test[0] > ny_test[1] else 1
if(ny == ny_test):
count += 1
print count / X_test.shape[0]
predictions = pd.read_csv('data/test.csv', header=0)
predictions['Survived'] = 0
predictions = process(predictions)
predictions = predictions[predictions.columns[1:]]
predictions = predictions.fillna(predictions.mean())
f = open('data/predictions.csv', 'wb')
for index, row in predictions.iterrows():
f.write(str(int(row['PassengerId'])))
