def vectorize(questions, answers, chars=None):
"""Vectorize the questions and expected answers"""
print('Vectorization...')
chars = chars or CHARS
x_maxlen = max(len(question) for question in questions)
y_maxlen = max(len(answer) for answer in answers)
# print (len(questions), x_maxlen, len(chars))
len_of_questions = len(questions)
ctable = CharacterTable(chars)
print("X = np_zeros")
X = np_zeros((len_of_questions, x_maxlen, len(chars)), dtype=np.bool)
print("for i, sentence in enumerate(questions):")
for i in xrange(len(questions)):
sentence = questions.pop()
for j, c in enumerate(sentence):
X[i, j, ctable.char_indices[c]] = 1
print("y = np_zeros")
y = np_zeros((len_of_questions, y_maxlen, len(chars)), dtype=np.bool)
print("for i, sentence in enumerate(answers):")
for i in xrange(len(answers)):
sentence = answers.pop()
for j, c in enumerate(sentence):
y[i, j, ctable.char_indices[c]] = 1
# Explicitly set apart 10% for validation data that we never train over
split_at = int(len(X) - len(X) / 10)
(X_train, X_val) = (slice_X(X, 0, split_at), slice_X(X, split_at))
(y_train, y_val) = (y[:split_at], y[split_at:])
print(X_train.shape)
print(y_train.shape)
return X_train, X_val, y_train, y_val, y_maxlen, ctable
def train(self, data_iterator):
'''
Train a keras model on a worker and send asynchronous updates
to parameter server
'''
feature_iterator, label_iterator = tee(data_iterator, 2)
x_train = np.asarray([x for x, y in feature_iterator])
y_train = np.asarray([y for x, y in label_iterator])
if x_train.size == 0:
return
model = model_from_yaml(self.yaml)
print(self.optimizer)
print(self.loss)
model.compile(optimizer=self.optimizer,loss=self.loss)
nb_epoch = self.train_config['nb_epoch']
batch_size = self.train_config.get('batch_size')
nb_train_sample = len(x_train[0])
nb_batch = int(np.ceil(nb_train_sample/float(batch_size)))
index_array = np.arange(nb_train_sample)
batches = [(i*batch_size, min(nb_train_sample, (i+1)*batch_size)) for i in range(0, nb_batch)]
if self.frequency == 'epoch':
for epoch in range(nb_epoch):
weights_before_training = get_server_weights(self.master_url)
model.set_weights(weights_before_training)
self.train_config['nb_epoch'] = 1
if x_train.shape[0] > batch_size:
model.fit(x_train, y_train, show_accuracy=True, **self.train_config)
weights_after_training = model.get_weights()
deltas = subtract_params(weights_before_training, weights_after_training)
put_deltas_to_server(deltas, self.master_url)
elif self.frequency == 'batch':
for epoch in range(nb_epoch):
if x_train.shape[0] > batch_size:
for (batch_start, batch_end) in batches:
weights_before_training = get_server_weights(self.master_url)
#sometimes weights_before_training ans model.set_weights(params) type not match.
#at first weights_before_training type is scala, but model.set_weights() need numpy array
#and sometimes weight_before_training method does not work well.
if(len(weights_before_training)>0 ):
model.set_weights(weights_before_training)
batch_ids = index_array[batch_start:batch_end]
X = slice_X(x_train, batch_ids)
y = slice_X(y_train, batch_ids)
model.train_on_batch(X, y)
weights_after_training = model.get_weights()
if( len(weights_before_training) == len(weights_after_training) ):
deltas = subtract_params(weights_before_training, weights_after_training)
else:
deltas = weights_after_training
print(len(deltas))
put_deltas_to_server(deltas, self.master_url)
else:
print('Choose frequency to be either batch or epoch')
yield []
def f(data_iterator):
'''
Train a keras model on a worker and send asynchronous updates
to parameter server
'''
feature_iterator, label_iterator = tee(data_iterator, 2)
x_train = np.asarray([x for x, y in feature_iterator])
y_train = np.asarray([y for x, y in label_iterator])
if x_train.size == 0:
return
global curr_worker
if None == curr_worker:
return
model = model_from_yaml(curr_worker.yaml, curr_worker.custom_objects)
model.compile(optimizer=curr_worker.master_optimizer,
loss=curr_worker.master_loss,
metrics=curr_worker.master_metrics)
nb_epoch = curr_worker.train_config['nb_epoch']
batch_size = curr_worker.train_config.get('batch_size')
nb_train_sample = x_train.shape[0]
nb_batch = int(np.ceil(nb_train_sample / float(batch_size)))
index_array = np.arange(nb_train_sample)
batches = [(i * batch_size, min(nb_train_sample, (i + 1) * batch_size))
for i in range(0, nb_batch)]
if curr_worker.frequency == 'epoch':
for epoch in range(nb_epoch):
weights_before_training = get_server_weights(curr_workermaster_url)
model.set_weights(weights_before_training)
curr_worker.train_config['nb_epoch'] = 1
if x_train.shape[0] > batch_size:
model.fit(x_train, y_train, **self.train_config)
weights_after_training = model.get_weights()
deltas = subtract_params(weights_before_training,
weights_after_training)
put_deltas_to_server(deltas, curr_worker.master_url)
elif curr_worker.frequency == 'batch':
from keras.engine.training import slice_X
for epoch in range(nb_epoch):
if x_train.shape[0] > batch_size:
for (batch_start, batch_end) in batches:
weights_before_training = get_server_weights(
curr_worker.master_url)
model.set_weights(weights_before_training)
batch_ids = index_array[batch_start:batch_end]
X = slice_X(x_train, batch_ids)
y = slice_X(y_train, batch_ids)
model.train_on_batch(X, y)
weights_after_training = model.get_weights()
deltas = subtract_params(weights_before_training,
weights_after_training)
put_deltas_to_server(deltas, curr_worker.master_url)
else:
print('Choose frequency to be either batch or epoch')
yield []
def train(self, partition_id, data_iterator):
'''
Train a keras model on a worker and send asynchronous updates
to parameter server
'''
import time
feature_iterator, label_iterator = tee(data_iterator, 2)
x_train = np.asarray([x for x, y in feature_iterator])
y_train = np.asarray([y for x, y in label_iterator])
if x_train.size == 0:
print("Empty Partition!")
return
model = model_from_yaml(self.yaml)
model.compile(loss=self.keras_loss, optimizer=self.keras_optimizer, metrics=['accuracy'])
nb_epoch = self.train_config['nb_epoch']
batch_size = self.train_config.get('batch_size')
nb_train_sample = x_train.shape[0]
nb_batch = int(np.ceil(nb_train_sample/float(batch_size)))
index_array = np.arange(nb_train_sample)
batches = [(i*batch_size, min(nb_train_sample, (i+1)*batch_size)) for i in range(0, nb_batch)]
if self.frequency == 'epoch':
for epoch in range(nb_epoch):
print('-'*40)
print('Partition %d/%d: Epoch %d' %(partition_id+1,self.num_workers,epoch))
print('-'*40)
weights_before_training = get_server_weights(self.master_url)
model.set_weights(weights_before_training)
self.train_config['nb_epoch'] = 1
if x_train.shape[0] > batch_size:
model.fit(x_train, y_train, **self.train_config)
weights_after_training = model.get_weights()
deltas = subtract_params(weights_before_training, weights_after_training)
put_deltas_to_server(deltas, self.master_url)
elif self.frequency == 'batch':
for epoch in range(nb_epoch):
print('-'*40)
print('Partition %d/%d: Epoch %d' %(partition_id+1,self.num_workers,epoch))
print('-'*40)
if x_train.shape[0] > batch_size:
for (batch_start, batch_end) in batches:
weights_before_training = get_server_weights(self.master_url)
model.set_weights(weights_before_training)
batch_ids = index_array[batch_start:batch_end]
X = slice_X(x_train, batch_ids)
y = slice_X(y_train, batch_ids)
model.train_on_batch(X, y)
weights_after_training = model.get_weights()
deltas = subtract_params(weights_before_training, weights_after_training)
put_deltas_to_server(deltas, self.master_url)
else:
print('Choose frequency to be either batch or epoch')
yield []
def train(self, data_iterator):
'''
Train a keras model on a worker and send asynchronous updates
to parameter server
'''
feature_iterator, label_iterator = tee(data_iterator, 2)
x_train = np.asarray([x for x, y in feature_iterator])
y_train = np.asarray([y for x, y in label_iterator])
if x_train.size == 0:
return
model = model_from_yaml(self.yaml, self.custom_objects)
model.compile(optimizer=self.master_optimizer, loss=self.master_loss, metrics=self.master_metrics)
nb_epoch = self.train_config['nb_epoch']
batch_size = self.train_config.get('batch_size')
nb_train_sample = len(x_train[0])
nb_batch = int(np.ceil(nb_train_sample/float(batch_size)))
index_array = np.arange(nb_train_sample)
batches = [(i*batch_size, min(nb_train_sample, (i+1)*batch_size)) for i in range(0, nb_batch)]
if self.frequency == 'epoch':
for epoch in range(nb_epoch):
weights_before_training = get_server_weights(self.master_url)
model.set_weights(weights_before_training)
self.train_config['nb_epoch'] = 1
if x_train.shape[0] > batch_size:
model.fit(x_train, y_train, **self.train_config)
weights_after_training = model.get_weights()
deltas = subtract_params(weights_before_training, weights_after_training)
put_deltas_to_server(deltas, self.master_url)
elif self.frequency == 'batch':
from keras.engine.training import slice_X
for epoch in range(nb_epoch):
if x_train.shape[0] > batch_size:
for (batch_start, batch_end) in batches:
weights_before_training = get_server_weights(self.master_url)
model.set_weights(weights_before_training)
batch_ids = index_array[batch_start:batch_end]
X = slice_X(x_train, batch_ids)
y = slice_X(y_train, batch_ids)
model.train_on_batch(X, y)
weights_after_training = model.get_weights()
deltas = subtract_params(weights_before_training, weights_after_training)
put_deltas_to_server(deltas, self.master_url)
else:
print('Choose frequency to be either batch or epoch')
yield []
def split_train_and_valid(cls,
data,
mask,
validation_split,
shuffle=False):
print('Shuffle & split...')
if shuffle:
data, mask = cls.shuffle_train(data, mask)
split_at = int(len(data) * (1. - validation_split))
x_train, x_valid = (slice_X(data, 0,
split_at), slice_X(data, split_at))
y_train, y_valid = (slice_X(mask, 0,
split_at), slice_X(mask, split_at))
cls.save_valid_idx(range(len(data))[split_at:])
return (x_train, y_train), (x_valid, y_valid)
def _predict_loop(self, f, ins, batch_size=128, verbose=0):
'''
Abstract method to loop over some data in batches.
'''
nb_sample = len(ins[0])
outs = []
if verbose == 1:
progbar = Progbar(target=nb_sample)
batches = make_batches(nb_sample, batch_size)
index_array = np.arange(nb_sample)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
ins_batch = slice_X(ins, batch_ids)
batch_outs = f(self, ins_batch)
if type(batch_outs) != list:
batch_outs = [batch_outs]
if batch_index == 0:
for batch_out in batch_outs:
shape = (nb_sample, ) + batch_out.shape[1:]
outs.append(np.zeros(shape))
for i, batch_out in enumerate(batch_outs):
outs[i][batch_start:batch_end] = batch_out
if verbose == 1:
progbar.update(batch_end)
return outs
def _test_loop(self, f, ins, batch_size=128, verbose=0):
'''
Abstract method to loop over some data in batches.
'''
nb_sample = len(ins[0])
outs = []
if verbose == 1:
progbar = Progbar(target=nb_sample)
batches = make_batches(nb_sample, batch_size)
index_array = np.arange(nb_sample)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
ins_batch = slice_X(ins, batch_ids)
batch_outs = f(self, ins_batch)
if type(batch_outs) == list:
if batch_index == 0:
for batch_out in enumerate(batch_outs):
outs.append(0.)
for i, batch_out in enumerate(batch_outs):
outs[i] += batch_out * len(batch_ids)
else:
if batch_index == 0:
outs.append(0.)
outs[0] += batch_outs * len(batch_ids)
if verbose == 1:
progbar.update(batch_end)
for i, out in enumerate(outs):
outs[i] /= nb_sample
return outs
def _predict_loop(self, f, ins, batch_size=128, verbose=0):
'''
Abstract method to loop over some data in batches.
'''
nb_sample = len(ins[0])
outs = []
if verbose == 1:
progbar = Progbar(target=nb_sample)
batches = make_batches(nb_sample, batch_size)
index_array = np.arange(nb_sample)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
ins_batch = slice_X(ins, batch_ids)
batch_outs = f(self, ins_batch)
if type(batch_outs) != list:
batch_outs = [batch_outs]
if batch_index == 0:
for batch_out in batch_outs:
shape = (nb_sample,) + batch_out.shape[1:]
outs.append(np.zeros(shape))
for i, batch_out in enumerate(batch_outs):
outs[i][batch_start:batch_end] = batch_out
if verbose == 1:
progbar.update(batch_end)
return outs
def _test_loop(self, f, ins, batch_size=128, verbose=0):
'''
Abstract method to loop over some data in batches.
'''
nb_sample = len(ins[0])
outs = []
if verbose == 1:
progbar = Progbar(target=nb_sample)
batches = make_batches(nb_sample, batch_size)
index_array = np.arange(nb_sample)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
ins_batch = slice_X(ins, batch_ids)
batch_outs = f(self, ins_batch)
if type(batch_outs) == list:
if batch_index == 0:
for batch_out in enumerate(batch_outs):
outs.append(0.)
for i, batch_out in enumerate(batch_outs):
outs[i] += batch_out * len(batch_ids)
else:
if batch_index == 0:
outs.append(0.)
outs[0] += batch_outs * len(batch_ids)
if verbose == 1:
progbar.update(batch_end)
for i, out in enumerate(outs):
outs[i] /= nb_sample
return outs
def vectorizeData(input_seq, target_seq):
X = np.zeros((len(input_seq), MAXLEN, len(chars)), dtype=np.bool)
y = np.zeros((len(target_seq), MAXLEN, len(chars)), dtype=np.bool)
for i, password in enumerate(input_seq):
X[i] = ctable.encode(password, maxlen=MAXLEN)
for i, password in enumerate(target_seq):
y[i] = ctable.encode(password, maxlen=MAXLEN)
indices = np.arange(len(y))
np.random.shuffle(indices)
X = X[indices]
y = y[indices]
# Explicitly set apart 10% for validation data that we never train over
split_at = len(X) - len(X) / 10
(X_train, X_val) = (slice_X(X, 0, split_at), slice_X(X, split_at))
(y_train, y_val) = (y[:split_at], y[split_at:])
return X_train, y_train, X_val, y_val
def _eval_loop(f, ins, batch_size=32):
'''Abstract method to loop over some data in batches.
# Arguments
f: Keras function returning a list of tensors.
ins: list of tensors to be fed to `f`.
batch_size: integer batch size.
verbose: verbosity mode.
# Returns
Scalar loss (if the model has a single output and no metrics)
or list of scalars (if the model has multiple outputs
and/or metrics). The attribute `model.metrics_names` will give you
the display labels for the scalar outputs.
'''
nb_sample = ins[0].shape[0]
outs = []
batches = make_batches(nb_sample, batch_size)
index_array = np.arange(nb_sample)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
if isinstance(ins[-1], float):
# do not slice the training phase flag
ins_batch = slice_X(ins[:-1], batch_ids) + [ins[-1]]
else:
ins_batch = slice_X(ins, batch_ids)
batch_outs = f(ins_batch)
if isinstance(batch_outs, list):
if batch_index == 0:
for batch_out in enumerate(batch_outs):
outs.append(np.zeros((nb_sample, )))
for i, batch_out in enumerate(batch_outs):
outs[i][batch_ids] = batch_out
else:
if batch_index == 0:
outs.append(np.zeros((nb_sample, )))
outs[i][batch_ids] = batch_out
if len(outs) == 1:
return outs[0]
return outs
def test_model(model): # 测试全部样本
X, y = prepare_data(TRAINING_SIZE)
# Explicitly set apart 10% for validation data that we never train over
split_at = len(X) - len(X) / 10
(X_train, X_val) = (slice_X(X, 0, split_at), slice_X(X, split_at))
(y_train, y_val) = (y[:split_at], y[split_at:])
X = X_val
y = y_val
#X = X_train
#y = y_train
print('Guessing ... ')
b = b2 = 0
for i in range(len(X)):
rowX, rowy = np.array([X[i]]), np.array([y[i]])
preds = model.predict_classes(rowX, verbose=0)
q = rowX[0]
correct = rowy[0]
guess = preds[0]
print('Q', q)
print('T', correct)
for xx in range(MAXW_a):
if correct[0][xx] > 0:
g2 = xx
break
print(
colors.ok + '☑' +
colors.close if correct[0][guess[0]] > 0 else colors.fail + '☒' +
colors.close, guess, g2)
b += 1 if correct[0][guess[0]] > 0 else 0
b2 += 1 if (guess[0] < 10 and g2 < 10) or (
guess[0] > 10 and g2 > 10) or (guess[0] == 10 and g2 == 10) else 0
print('---')
print('t=', len(X))
print('b=', b)
print('b/t= %.4f' % (b * 1.0 / len(X)))
print('b2=', b2)
print('b2/t= %.4f' % (b2 * 1.0 / len(X)))
def _fit_loop(self, f, ins, discrete_actions, out_labels=[],
batch_size=32, nb_epoch=100, verbose=1, callbacks=[],
val_f=None, val_ins=None, shuffle=True, callback_metrics=[],
theta_metrics={}):
do_validation = False
if val_f and val_ins:
do_validation = True
if verbose:
print('Train on %d samples, validate on %d samples' %
(ins[0].shape[0], val_ins[0].shape[0]))
nb_train_sample = ins[0].shape[0]
print(nb_train_sample)
index_array = np.arange(nb_train_sample)
history = PBOHistory()
callbacks = [history] + callbacks
if verbose:
callbacks += [cbks.ProgbarLogger()]
callbacks = cbks.CallbackList(callbacks)
callback_model = self
callbacks._set_model(callback_model)
callbacks._set_params({
'batch_size': batch_size,
'nb_epoch': nb_epoch,
'nb_sample': nb_train_sample,
'verbose': verbose,
'do_validation': do_validation,
'metrics': callback_metrics + [el for el in theta_metrics.keys()],
'theta': self.q_model.trainable_weights[0].eval()
})
callbacks.on_train_begin()
callback_model.stop_training = False
for epoch in range(nb_epoch):
callbacks.on_epoch_begin(epoch)
if shuffle == 'batch':
index_array = batch_shuffle(index_array, batch_size)
elif shuffle:
np.random.shuffle(index_array)
batches = make_batches(nb_train_sample, batch_size)
epoch_logs = {}
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
try:
if type(ins[-1]) is float:
# do not slice the training phase flag
ins_batch = slice_X(ins[:-1], batch_ids) + [ins[-1]]
else:
ins_batch = slice_X(ins, batch_ids)
except TypeError:
raise Exception('TypeError while preparing batch. '
'If using HDF5 input data, '
'pass shuffle="batch".')
batch_logs = {}
batch_logs['batch'] = batch_index
batch_logs['size'] = len(batch_ids)
batch_logs['theta'] = self.q_model.trainable_weights[0].eval()
for k in theta_metrics.keys():
batch_logs[k] = theta_metrics[k](self.q_model.trainable_weights)
callbacks.on_batch_begin(batch_index, batch_logs)
inp = ins_batch + discrete_actions
outs = f(*inp)
if type(outs) != list:
outs = [outs]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
if batch_index == len(batches) - 1: # last batch
# validation
if do_validation:
# replace with self._evaluate
val_outs = self._test_loop(val_f, val_ins,
batch_size=batch_size,
verbose=0)
if type(val_outs) != list:
val_outs = [val_outs]
# same labels assumed
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_epoch_end(epoch, epoch_logs)
callbacks.on_train_end()
return history
ct = lnCounts[ii]
soFar += ct
print(ii, ct, "\t", soFar / tot)
print('Vectorization...')
Xpseud = np.zeros((len(pseudWords), maxlen, len(chars)), dtype=np.bool)
ypseud = np.zeros((len(pseudWords), maxlen, len(chars)), dtype=np.bool)
for ii, word in enumerate(pseudWords):
#print("encoding", word, "at row", ii)
Xpseud[ii] = ctable.encode(pad(word, maxlen, "X"), maxlen)
ypseud[ii] = ctable.encode(pad(word, maxlen, "X"), maxlen)
# Explicitly set apart 10% for validation data that we never train over.
split_at = len(Xpseud) - len(Xpseud) // 10
(X_p_train, X_p_val) = (slice_X(Xpseud, 0, split_at),
slice_X(Xpseud, split_at))
(y_p_train, y_p_val) = (ypseud[:split_at], ypseud[split_at:])
RNN = recurrent.LSTM
BATCH_SIZE = 128
LAYERS = 1
hidden = 40
modPseud = buildCharEncDec(hidden, RNN, LAYERS, maxlen, chars, dropout=.75)
for iteration in range(1, 20):
print()
print('-' * 50)
print('Iteration', iteration)
modPseud.fit(X_p_train, y_p_train, batch_size=BATCH_SIZE,
def fit(self,
s,
a,
s_next,
r,
absorbing,
theta,
batch_size=32,
nb_epoch=10,
shuffle=True,
theta_metrics={}):
"""
Args:
s (numpy.array): the samples of the state (nsamples, state_dim)
a (numpy.array): the samples of the state (nsamples, action_dim)
s_next (numpy.array): the samples of the next (reached) state (nsamples, state_dim)
r (numpy.array): the sample of the reward (nsamples, )
theta (numpy.array): the sample of the Q-function parameters (1, n_params)
batch_size (int): dimension of the batch used for a single step of the gradient
nb_epoch (int): number of epochs
verbose (int): 0 or 1. Verbosity mode. 0 = silent, 1 = verbose.
callbacks (list): list of callbacks to be called during training.
See [Keras Callbacks](https://keras.io/callbacks/).
validation_split (float): float between 0 and 1:
fraction of the training data to be used as validation data.
The model will set apart this fraction of the training data,
will not train on it, and will evaluate the loss and any model metrics
on this data at the end of each epoch.
validation_data (tuple): data on which to evaluate the loss and any model metrics
at the end of each epoch. The model will not be trained on this data.
This could be a tuple (val_s, val_a, val_s_next, val_r) or a tuple
(val_s, val_a, val_s_next, val_r, val_theta).
shuffle (boolean): whether to shuffle the training data before each epoch.
theta_metrics (dict): dictionary storing the pairs (name: callable object).
The callable object/function is used to evaluate the Q-function parameters
at each iteration. The signature of the callable is simple: f(theta)
e.g.: theta_metrics={'k': lambda theta: evaluate(theta)})
Returns:
A PBOHistory instance storing train information
"""
s, a, s_next, r, absorbing, theta = self._standardize_user_data(
s, a, s_next, r, absorbing, theta, check_batch_dim=False)
all_actions = standardize_input_data(
self.discrete_actions, ['all_actions'],
[(None, self.action_dim)] if self.action_dim is not None else None,
exception_prefix='discrete_actions')
n_updates = 0
history = {"theta": [], 'rho': []}
for k in theta_metrics.keys():
history.update({k: []})
ins = s + a + s_next + [r, absorbing]
self._make_train_function()
f = self.train_function
nb_train_sample = ins[0].shape[0]
index_array = np.arange(nb_train_sample)
# append evolution of theta for independent case
for _ in range(len(self.theta_list) - 1):
if self.incremental:
tmp = theta[-1] + self.bellman_model.predict(theta[-1])
else:
tmp = self.bellman_model.predict(theta[-1])
theta += [tmp]
term_condition = self.term_condition
stop = False
old_theta = theta
for epoch in range(nb_epoch):
if stop:
break
if shuffle == 'batch':
index_array = batch_shuffle(index_array, batch_size)
elif shuffle:
np.random.shuffle(index_array)
batches = make_batches(nb_train_sample, batch_size)
for batch_index, (batch_start, batch_end) in enumerate(batches):
history["theta"].append(theta[0])
if hasattr(self.bellman_model, '_model'):
history["rho"].append(
self.bellman_model._model.get_weights())
else:
history["rho"].append(self.bellman_model.get_weights())
for k, v in iteritems(theta_metrics):
history[k].append(v(theta))
batch_ids = index_array[batch_start:batch_end]
try:
if type(ins[-1]) is float:
# do not slice the training phase flag
ins_batch = slice_X(ins[:-1], batch_ids) + [ins[-1]]
else:
ins_batch = slice_X(ins, batch_ids)
except TypeError:
raise Exception('TypeError while preparing batch. '
'If using HDF5 input data, '
'pass shuffle="batch".')
inp = ins_batch + theta + all_actions
outs = f(*inp)
n_updates += 1
if self.update_theta_every > 0 and n_updates % self.update_theta_every == 0:
tmp = self.apply_bo(theta[0],
n_times=self.steps_per_theta_update)
theta = [tmp]
for _ in range(len(self.theta_list) - 1):
if self.incremental:
tmp = tmp + self.bellman_model.predict(tmp)
else:
tmp = self.bellman_model.predict(tmp)
theta += [tmp]
if term_condition is not None:
stop = term_condition(old_theta, theta)
if stop:
break
old_theta = theta
# finally apply the bellman operator K-times to get the final point
self.learned_theta_value = self.apply_bo(theta[0], n_times=100)
if self.verbose > 1:
print('learned theta: {}'.format(self.learned_theta_value))
self.history = history
return history
y = np.zeros((len(questions), DIGITS + 1, len(chars)), dtype=np.bool)
for i, sentence in enumerate(questions):
X[i] = ctable.encode(sentence, MAXLEN)
for i, sentence in enumerate(expected):
y[i] = ctable.encode(sentence, DIGITS + 1)
# Shuffle (X, y) in unison as the later parts of X will almost all be larger
# digits.
indices = np.arange(len(y))
np.random.shuffle(indices)
X = X[indices]
y = y[indices]
# Explicitly set apart 10% for validation data that we never train over.
split_at = len(X) - len(X) // 10
(X_train, X_val) = (slice_X(X, 0, split_at), slice_X(X, split_at))
(y_train, y_val) = (y[:split_at], y[split_at:])
print('Training Data:')
print(X_train.shape)
print(y_train.shape)
print('Validation Data:')
print(X_val.shape)
print(y_val.shape)
# Try replacing GRU, or SimpleRNN.
RNN = recurrent.LSTM
HIDDEN_SIZE = 128
BATCH_SIZE = 128
LAYERS = 1
def fit(self,
sast,
r,
batch_size=32,
nb_epoch=10,
shuffle=True,
theta_metrics={}):
"""
Args:
s (numpy.array): the samples of the state (nsamples, state_dim)
a (numpy.array): the samples of the state (nsamples, action_dim)
s_next (numpy.array): the samples of the next (reached) state (nsamples, state_dim)
r (numpy.array): the sample of the reward (nsamples, )
theta (numpy.array): the sample of the Q-function parameters (1, n_params)
batch_size (int): dimension of the batch used for a single step of the gradient
nb_epoch (int): number of epochs
verbose (int): 0 or 1. Verbosity mode. 0 = silent, 1 = verbose.
callbacks (list): list of callbacks to be called during training.
See [Keras Callbacks](https://keras.io/callbacks/).
validation_split (float): float between 0 and 1:
fraction of the training data to be used as validation data.
The model will set apart this fraction of the training data,
will not train on it, and will evaluate the loss and any model metrics
on this data at the end of each epoch.
validation_data (tuple): data on which to evaluate the loss and any model metrics
at the end of each epoch. The model will not be trained on this data.
This could be a tuple (val_s, val_a, val_s_next, val_r) or a tuple
(val_s, val_a, val_s_next, val_r, val_theta).
shuffle (boolean): whether to shuffle the training data before each epoch.
theta_metrics (dict): dictionary storing the pairs (name: callable object).
The callable object/function is used to evaluate the Q-function parameters
at each iteration. The signature of the callable is simple: f(theta)
e.g.: theta_metrics={'k': lambda theta: evaluate(theta)})
Returns:
A PBOHistory instance storing train information
"""
sast = standardize_input_data(
sast, ['sast'], (None, 2 * self.state_dim + self.action_dim + 1),
exception_prefix='sast')[0]
next_states_idx = self.state_dim + self.action_dim
sa = sast[:, :next_states_idx]
s_next = sast[:, next_states_idx:-1]
absorbing = sast[:, -1]
n_updates = 0
maxq, maxa = self.maxQA(s_next, absorbing)
if hasattr(self._estimator, 'adapt'):
# update estimator structure
self._estimator.adapt(iteration=self._iteration)
# y = np.reshape(r + self.gamma * maxq, (-1, 1))
y = r + self.gamma * maxq
ins = [sa, y]
self._make_train_function()
f = self.train_function
nb_train_sample = sa.shape[0]
index_array = np.arange(nb_train_sample)
history = {"theta": []}
for k in theta_metrics.keys():
history.update({k: []})
for epoch in range(nb_epoch):
if shuffle == 'batch':
index_array = batch_shuffle(index_array, batch_size)
elif shuffle:
np.random.shuffle(index_array)
batches = make_batches(nb_train_sample, batch_size)
for batch_index, (batch_start, batch_end) in enumerate(batches):
if hasattr(self._estimator, '_model'):
ltheta = self._model.get_weights()
history["theta"].append(ltheta)
else:
ltheta = self._estimator.get_weights()
history["theta"].append(ltheta)
for k, v in iteritems(theta_metrics):
history[k].append(v(ltheta))
batch_ids = index_array[batch_start:batch_end]
try:
if type(ins[-1]) is float:
# do not slice the training phase flag
ins_batch = slice_X(ins[:-1], batch_ids) + [ins[-1]]
else:
ins_batch = slice_X(ins, batch_ids)
except TypeError:
raise Exception('TypeError while preparing batch. '
'If using HDF5 input data, '
'pass shuffle="batch".')
outs = f(*ins_batch)
n_updates += 1
if self.update_theta_every > 0 \
and n_updates % self.update_theta_every == 0:
maxq, maxa = self.maxQA(s_next, absorbing)
if hasattr(self._estimator, 'adapt'):
# update estimator structure
self._estimator.adapt(iteration=self._iteration)
# y = np.reshape(r + self.gamma * maxq, (-1, 1))
y = r + self.gamma * maxq
ins = [ins[0], y]
if self._verbose > 1:
print('learned theta: {}'.format(self._estimator.get_weights()))
return history
def fit(model, x, y, batch_size=32, nb_epoch=10, verbose=1, callbacks=[],
validation_split=0., validation_data=None, shuffle=True,
class_weight=None, sample_weight=None):
'''Trains the model for a fixed number of epochs (iterations on a dataset).
# Arguments
x: Numpy array of training data,
or list of Numpy arrays if the model has multiple inputs.
If all inputs in the model are named, you can also pass a dictionary
mapping input names to Numpy arrays.
y: Numpy array of target data,
or list of Numpy arrays if the model has multiple outputs.
If all outputs in the model are named, you can also pass a dictionary
mapping output names to Numpy arrays.
batch_size: integer. Number of samples per gradient update.
nb_epoch: integer, the number of times to iterate over the training data arrays.
verbose: 0, 1, or 2. Verbosity mode. 0 = silent, 1 = verbose, 2 = one log line per epoch.
callbacks: list of callbacks to be called during training.
See [callbacks](/callbacks).
validation_split: float between 0 and 1:
fraction of the training data to be used as validation data.
The model will set apart this fraction of the training data,
will not train on it, and will evaluate the loss and any model metrics
on this data at the end of each epoch.
validation_data: data on which to evaluate the loss and any model metrics
at the end of each epoch. The model will not be trained on this data.
This could be a tuple (x_val, y_val) or a tuple (val_x, val_y, val_sample_weights).
shuffle: boolean, whether to shuffle the training data before each epoch.
class_weight: optional dictionary mapping class indices (integers) to
a weight (float) to apply to the model's loss for the samples
from this class during training.
This can be useful to tell the model to "pay more attention" to
samples from an under-represented class.
sample_weight: optional array of the same length as x, containing
weights to apply to the model's loss for each sample.
In the case of temporal data, you can pass a 2D array
with shape (samples, sequence_length),
to apply a different weight to every timestep of every sample.
In this case you should make sure to specify sample_weight_mode="temporal" in compile().
# Returns
A `History` instance. Its `history` attribute contains
all information collected during training.
'''
# validate user data
'''
We need to use the custom standardize_user_data function defined above
to skip checking for the array_length. Everything else is the same as the original fit code
Note: We did not use model._standardize_user_data(...) but instead used the above
standardize_user_data(...) method to bypass the original standardize code in keras.
'''
x, y, sample_weights = _standardize_user_data(model, x, y,
sample_weight=sample_weight,
class_weight=class_weight,
check_batch_dim=False,
batch_size=batch_size)
# prepare validation data
if validation_data:
do_validation = True
if len(validation_data) == 2:
val_x, val_y = validation_data
val_sample_weight = None
elif len(validation_data) == 3:
val_x, val_y, val_sample_weight = validation_data
else:
raise
val_x, val_y, val_sample_weights = model._standardize_user_data(val_x, val_y,
sample_weight=val_sample_weight,
check_batch_dim=False,
batch_size=batch_size)
model._make_test_function()
val_f = model.test_function
if model.uses_learning_phase and type(K.learning_phase()) is not int:
val_ins = val_x + val_y + val_sample_weights + [0.]
else:
val_ins = val_x + val_y + val_sample_weights
elif validation_split and 0. < validation_split < 1.:
do_validation = True
split_at = int(len(x[0]) * (1. - validation_split))
x, val_x = (slice_X(x, 0, split_at), slice_X(x, split_at))
y, val_y = (slice_X(y, 0, split_at), slice_X(y, split_at))
sample_weights, val_sample_weights = (
slice_X(sample_weights, 0, split_at), slice_X(sample_weights, split_at))
model._make_test_function()
val_f = model.test_function
if model.uses_learning_phase and type(K.learning_phase()) is not int:
val_ins = val_x + val_y + val_sample_weights + [0.]
else:
val_ins = val_x + val_y + val_sample_weights
else:
do_validation = False
val_f = None
val_ins = None
# prepare input arrays and training function
if model.uses_learning_phase and type(K.learning_phase()) is not int:
ins = x + y + sample_weights + [1.]
else:
ins = x + y + sample_weights
model._make_train_function()
f = model.train_function
# prepare display labels
out_labels = model.metrics_names
# rename duplicated metrics name
# (can happen with an output layer shared among multiple dataflows)
deduped_out_labels = []
for i, label in enumerate(out_labels):
new_label = label
if out_labels.count(label) > 1:
dup_idx = out_labels[:i].count(label)
new_label += '_' + str(dup_idx + 1)
deduped_out_labels.append(new_label)
out_labels = deduped_out_labels
if do_validation:
callback_metrics = copy.copy(out_labels) + ['val_' + n for n in out_labels]
else:
callback_metrics = copy.copy(out_labels)
# delegate logic to _fit_loop
return model._fit_loop(f, ins, out_labels=out_labels,
batch_size=batch_size, nb_epoch=nb_epoch,
verbose=verbose, callbacks=callbacks,
val_f=val_f, val_ins=val_ins, shuffle=shuffle,
callback_metrics=callback_metrics)
def _fit_loop(self,
f,
ins,
discrete_actions,
out_labels=[],
batch_size=32,
nb_epoch=100,
verbose=1,
callbacks=[],
val_f=None,
val_ins=None,
shuffle=True,
callback_metrics=[],
theta_metrics={}):
do_validation = False
if val_f and val_ins:
do_validation = True
if verbose:
print('Train on %d samples, validate on %d samples' %
(ins[0].shape[0], val_ins[0].shape[0]))
nb_train_sample = ins[0].shape[0]
print(nb_train_sample)
index_array = np.arange(nb_train_sample)
history = PBOHistory()
callbacks = [history] + callbacks
if verbose:
callbacks += [cbks.ProgbarLogger()]
callbacks = cbks.CallbackList(callbacks)
callback_model = self
callbacks._set_model(callback_model)
callbacks._set_params({
'batch_size':
batch_size,
'nb_epoch':
nb_epoch,
'nb_sample':
nb_train_sample,
'verbose':
verbose,
'do_validation':
do_validation,
'metrics':
callback_metrics + [el for el in theta_metrics.keys()],
'theta':
self.q_model.trainable_weights[0].eval()
})
callbacks.on_train_begin()
callback_model.stop_training = False
for epoch in range(nb_epoch):
callbacks.on_epoch_begin(epoch)
if shuffle == 'batch':
index_array = batch_shuffle(index_array, batch_size)
elif shuffle:
np.random.shuffle(index_array)
batches = make_batches(nb_train_sample, batch_size)
epoch_logs = {}
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
try:
if type(ins[-1]) is float:
# do not slice the training phase flag
ins_batch = slice_X(ins[:-1], batch_ids) + [ins[-1]]
else:
ins_batch = slice_X(ins, batch_ids)
except TypeError:
raise Exception('TypeError while preparing batch. '
'If using HDF5 input data, '
'pass shuffle="batch".')
batch_logs = {}
batch_logs['batch'] = batch_index
batch_logs['size'] = len(batch_ids)
batch_logs['theta'] = self.q_model.trainable_weights[0].eval()
for k in theta_metrics.keys():
batch_logs[k] = theta_metrics[k](
self.q_model.trainable_weights)
callbacks.on_batch_begin(batch_index, batch_logs)
inp = ins_batch + discrete_actions
outs = f(*inp)
if type(outs) != list:
outs = [outs]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
if batch_index == len(batches) - 1: # last batch
# validation
if do_validation:
# replace with self._evaluate
val_outs = self._test_loop(val_f,
val_ins,
batch_size=batch_size,
verbose=0)
if type(val_outs) != list:
val_outs = [val_outs]
# same labels assumed
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_epoch_end(epoch, epoch_logs)
callbacks.on_train_end()
return history
def train(self, data_iterator):
'''
Train a keras model on a worker and send asynchronous updates
to parameter server
'''
feature_iterator, label_iterator = tee(data_iterator, 2)
x_train = np.asarray([x for x, y in feature_iterator])
y_train = np.asarray([y for x, y in label_iterator])
if x_train.size == 0:
return
model = model_from_yaml(self.yaml)
print(self.optimizer)
print(self.loss)
model.compile(optimizer=self.optimizer, loss=self.loss)
nb_epoch = self.train_config['nb_epoch']
batch_size = self.train_config.get('batch_size')
nb_train_sample = len(x_train[0])
nb_batch = int(np.ceil(nb_train_sample / float(batch_size)))
index_array = np.arange(nb_train_sample)
batches = [(i * batch_size, min(nb_train_sample, (i + 1) * batch_size))
for i in range(0, nb_batch)]
if self.frequency == 'epoch':
for epoch in range(nb_epoch):
weights_before_training = get_server_weights(self.master_url)
model.set_weights(weights_before_training)
self.train_config['nb_epoch'] = 1
if x_train.shape[0] > batch_size:
model.fit(x_train,
y_train,
show_accuracy=True,
**self.train_config)
weights_after_training = model.get_weights()
deltas = subtract_params(weights_before_training,
weights_after_training)
put_deltas_to_server(deltas, self.master_url)
elif self.frequency == 'batch':
for epoch in range(nb_epoch):
if x_train.shape[0] > batch_size:
for (batch_start, batch_end) in batches:
weights_before_training = get_server_weights(
self.master_url)
#sometimes weights_before_training ans model.set_weights(params) type not match.
#at first weights_before_training type is scala, but model.set_weights() need numpy array
#and sometimes weight_before_training method does not work well.
if (len(weights_before_training) > 0):
model.set_weights(weights_before_training)
batch_ids = index_array[batch_start:batch_end]
X = slice_X(x_train, batch_ids)
y = slice_X(y_train, batch_ids)
model.train_on_batch(X, y)
weights_after_training = model.get_weights()
if (len(weights_before_training) == len(
weights_after_training)):
deltas = subtract_params(weights_before_training,
weights_after_training)
else:
deltas = weights_after_training
print(len(deltas))
put_deltas_to_server(deltas, self.master_url)
else:
print('Choose frequency to be either batch or epoch')
yield []
def _fit_loop(self, f, ins, out_labels=None, batch_size=32,
nb_epoch=100, verbose=1, callbacks=None,
val_f=None, val_ins=None, shuffle=True,
callback_metrics=None, initial_epoch=0):
"""Abstract fit function for f(ins).
Assume that f returns a list, labeled by out_labels.
# Arguments
f: Keras function returning a list of tensors
ins: list of tensors to be fed to `f`
out_labels: list of strings, display names of
the outputs of `f`
batch_size: integer batch size
nb_epoch: number of times to iterate over the data
verbose: verbosity mode, 0, 1 or 2
callbacks: list of callbacks to be called during training
val_f: Keras function to call for validation
val_ins: list of tensors to be fed to `val_f`
shuffle: whether to shuffle the data at the beginning of each epoch
callback_metrics: list of strings, the display names of the metrics
passed to the callbacks. They should be the
concatenation of list the display names of the outputs of
`f` and the list of display names of the outputs of `f_val`.
initial_epoch: epoch at which to start training
(useful for resuming a previous training run)
# Returns
`History` object.
[A tweaked version.]
"""
do_validation = False
if val_f and val_ins:
do_validation = True
if verbose:
print('Train on %d samples, validate on %d samples' %
(ins[0].shape[0], val_ins[0].shape[0]))
nb_train_sample = ins[0].shape[0]
index_array = np.arange(nb_train_sample)
self.history = cbks.History()
callbacks = [cbks.BaseLogger()] + (callbacks or []) + [self.history]
if verbose:
callbacks += [cbks.ProgbarLogger()]
callbacks = cbks.CallbackList(callbacks)
out_labels = out_labels or []
# it's possible to callback a different model than self
# (used by Sequential models)
if hasattr(self, 'callback_model') and self.callback_model:
callback_model = self.callback_model
else:
callback_model = self
callbacks.set_model(callback_model)
callbacks.set_params({
'batch_size': batch_size,
'nb_epoch': nb_epoch,
'nb_sample': nb_train_sample,
'verbose': verbose,
'do_validation': do_validation,
'metrics': callback_metrics or [],
})
callbacks.on_train_begin()
callback_model.stop_training = False
self.validation_data = val_ins
for epoch in range(initial_epoch, nb_epoch):
callbacks.on_epoch_begin(epoch)
if shuffle == 'batch':
index_array = batch_shuffle(index_array, batch_size)
elif shuffle:
np.random.shuffle(index_array)
batches = make_batches(nb_train_sample, batch_size)
epoch_logs = {}
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
try:
if isinstance(ins[-1], float):
# do not slice the training phase flag
ins_batch = slice_X(ins[:-1], batch_ids) + [ins[-1]]
else:
ins_batch = slice_X(ins, batch_ids)
except TypeError:
raise TypeError('TypeError while preparing batch. '
'If using HDF5 input data, '
'pass shuffle="batch".')
batch_logs = {}
batch_logs['batch'] = batch_index
batch_logs['size'] = len(batch_ids)
batch_logs['ids'] = batch_ids
callbacks.on_batch_begin(batch_index, batch_logs)
outs = f(ins_batch)
if not isinstance(outs, list):
outs = [outs]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
if batch_index == len(batches) - 1: # last batch
# validation
if do_validation:
# replace with self._evaluate
val_outs = self._test_loop(val_f, val_ins,
batch_size=batch_size,
verbose=0)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# same labels assumed
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_epoch_end(epoch, epoch_logs)
if callback_model.stop_training:
break
callbacks.on_train_end()
return self.history
def fit(model, x, y, batch_size=32, nb_epoch=10, verbose=1, callbacks=[],
validation_split=0., validation_data=None, shuffle=True,
class_weight=None, sample_weight=None):
'''Trains the model for a fixed number of epochs (iterations on a dataset).
# Arguments
x: Numpy array of training data,
or list of Numpy arrays if the model has multiple inputs.
If all inputs in the model are named, you can also pass a dictionary
mapping input names to Numpy arrays.
y: Numpy array of target data,
or list of Numpy arrays if the model has multiple outputs.
If all outputs in the model are named, you can also pass a dictionary
mapping output names to Numpy arrays.
batch_size: integer. Number of samples per gradient update.
nb_epoch: integer, the number of times to iterate over the training data arrays.
verbose: 0, 1, or 2. Verbosity mode. 0 = silent, 1 = verbose, 2 = one log line per epoch.
callbacks: list of callbacks to be called during training.
See [callbacks](/callbacks).
validation_split: float between 0 and 1:
fraction of the training data to be used as validation data.
The model will set apart this fraction of the training data,
will not train on it, and will evaluate the loss and any model metrics
on this data at the end of each epoch.
validation_data: data on which to evaluate the loss and any model metrics
at the end of each epoch. The model will not be trained on this data.
This could be a tuple (x_val, y_val) or a tuple (val_x, val_y, val_sample_weights).
shuffle: boolean, whether to shuffle the training data before each epoch.
class_weight: optional dictionary mapping class indices (integers) to
a weight (float) to apply to the model's loss for the samples
from this class during training.
This can be useful to tell the model to "pay more attention" to
samples from an under-represented class.
sample_weight: optional array of the same length as x, containing
weights to apply to the model's loss for each sample.
In the case of temporal data, you can pass a 2D array
with shape (samples, sequence_length),
to apply a different weight to every timestep of every sample.
In this case you should make sure to specify sample_weight_mode="temporal" in compile().
# Returns
A `History` instance. Its `history` attribute contains
all information collected during training.
'''
# validate user data
'''
We need to use the custom standardize_user_data function defined above
to skip checking for the array_length. Everything else is the same as the original fit code
Note: We did not use model._standardize_user_data(...) but instead used the above
standardize_user_data(...) method to bypass the original standardize code in keras.
'''
x, y, sample_weights = _standardize_user_data(model, x, y,
sample_weight=sample_weight,
class_weight=class_weight,
check_batch_dim=False,
batch_size=batch_size)
# prepare validation data
if validation_data:
do_validation = True
if len(validation_data) == 2:
val_x, val_y = validation_data
val_sample_weight = None
elif len(validation_data) == 3:
val_x, val_y, val_sample_weight = validation_data
else:
raise
val_x, val_y, val_sample_weights = model._standardize_user_data(val_x, val_y,
sample_weight=val_sample_weight,
check_batch_dim=False,
batch_size=batch_size)
model._make_test_function()
val_f = model.test_function
if model.uses_learning_phase and type(K.learning_phase()) is not int:
val_ins = val_x + val_y + val_sample_weights + [0.]
else:
val_ins = val_x + val_y + val_sample_weights
elif validation_split and 0. < validation_split < 1.:
do_validation = True
split_at = int(len(x[0]) * (1. - validation_split))
x, val_x = (slice_X(x, 0, split_at), slice_X(x, split_at))
y, val_y = (slice_X(y, 0, split_at), slice_X(y, split_at))
sample_weights, val_sample_weights = (
slice_X(sample_weights, 0, split_at), slice_X(sample_weights, split_at))
model._make_test_function()
val_f = model.test_function
if model.uses_learning_phase and type(K.learning_phase()) is not int:
val_ins = val_x + val_y + val_sample_weights + [0.]
else:
val_ins = val_x + val_y + val_sample_weights
else:
do_validation = False
val_f = None
val_ins = None
# prepare input arrays and training function
if model.uses_learning_phase and type(K.learning_phase()) is not int:
ins = x + y + sample_weights + [1.]
else:
ins = x + y + sample_weights
model._make_train_function()
f = model.train_function
# prepare display labels
out_labels = model.metrics_names
# rename duplicated metrics name
# (can happen with an output layer shared among multiple dataflows)
deduped_out_labels = []
for i, label in enumerate(out_labels):
new_label = label
if out_labels.count(label) > 1:
dup_idx = out_labels[:i].count(label)
new_label += '_' + str(dup_idx + 1)
deduped_out_labels.append(new_label)
out_labels = deduped_out_labels
if do_validation:
callback_metrics = copy.copy(out_labels) + ['val_' + n for n in out_labels]
else:
callback_metrics = copy.copy(out_labels)
# delegate logic to _fit_loop
return model._fit_loop(f, ins, out_labels=out_labels,
batch_size=batch_size, nb_epoch=nb_epoch,
verbose=verbose, callbacks=callbacks,
val_f=val_f, val_ins=val_ins, shuffle=shuffle,
callback_metrics=callback_metrics)
def main():
questions = []
expected = []
seen = set()
print('Generating data...')
train_data = load_data('2017-01-12.dat')
train_data = train_data[:TRAINING_SIZE] # for test
for x in train_data:
#query = ','.join(['%04d+%05d'%(i[0],i[1]) for i in x[0]][:CART_SIZE])
query = ','.join(['%04d'%i for i in x[0]][:CART_SIZE]) # 不含价格信息
ans = ','.join(['%04d'%i for i in x[1]][:CART_SIZE])
questions.append(query)
expected.append(ans)
print('Total addition questions:', len(questions))
#print(questions)
#print(expected)
print('Vectorization...')
X = np.zeros((len(questions), MAXLEN_q, len(chars), 1))
y = np.zeros((len(questions), MAXLEN_a, len(chars)))
for i, sentence in enumerate(questions):
X[i] = ctable.encode(sentence, maxlen=MAXLEN_q)
for i, sentence in enumerate(expected):
y[i] = ctable.encode(sentence, maxlen=MAXLEN_a)
# Shuffle (X, y) in unison as the later parts of X will almost all be larger digits
indices = np.arange(len(y))
np.random.shuffle(indices)
X = X[indices]
y = y[indices]
# Explicitly set apart 10% for validation data that we never train over
split_at = len(X) - len(X) / 10
(X_train, X_val) = (slice_X(X, 0, split_at), slice_X(X, split_at))
(y_train, y_val) = (y[:split_at], y[split_at:])
print(X_train.shape)
print(y_train.shape)
#return X_train, y_train
print('Build model...')
model = Sequential()
model.add(Convolution2D(nb_filters, kernel_size[0], kernel_size[1],
border_mode='valid',
input_shape=(MAXLEN_q, len(chars), 1) ))
model.add(Activation('relu'))
model.add(Convolution2D(nb_filters, kernel_size[0], kernel_size[1]))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=pool_size))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(128))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(len(chars)))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adadelta',
metrics=['accuracy'])
model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch,
verbose=1, validation_data=(X_val, y_val))
score = model.evaluate(X_val, y_val, verbose=0)
print('Test score:', score[0])
print('Test accuracy:', score[1])
X = np.zeros((len(questions), MAXLEN, len(chars)), dtype=np.bool)
y = np.zeros((len(questions), DIGITS + 1, len(chars)), dtype=np.bool)
for i, sentence in enumerate(questions):
X[i] = ctable.encode(sentence, maxlen=MAXLEN)
for i, sentence in enumerate(expected):
y[i] = ctable.encode(sentence, maxlen=DIGITS + 1)
# Shuffle (X, y) in unison as the later parts of X will almost all be larger digits
indices = np.arange(len(y))
np.random.shuffle(indices)
X = X[indices]
y = y[indices]
# Explicitly set apart 10% for validation data that we never train over
split_at = len(X) - len(X) / 10
(X_train, X_val) = (slice_X(X, 0, split_at), slice_X(X, split_at))
(y_train, y_val) = (y[:split_at], y[split_at:])
print(X_train.shape)
print(y_train.shape)
print('Build model...')
model = Sequential()
# "Encode" the input sequence using an RNN, producing an output of HIDDEN_SIZE
# note: in a situation where your input sequences have a variable length,
# use input_shape=(None, nb_feature).
model.add(RNN(HIDDEN_SIZE, input_shape=(MAXLEN, len(chars))))
# For the decoder's input, we repeat the encoded input for each time step
model.add(RepeatVector(DIGITS + 1))
# The decoder RNN could be multiple layers stacked or a single layer
for _ in range(LAYERS):
def main(model=None):
X, y = prepare_data(TRAINING_SIZE)
print('Split data ... ')
# Shuffle (X, y)
#indices = np.arange(len(y))
#np.random.shuffle(indices)
#X = X[indices]
#y = y[indices]
# Explicitly set apart 10% for validation data that we never train over
split_at = len(X) - len(X) / 10
(X_train, X_val) = (slice_X(X, 0, split_at), slice_X(X, split_at))
(y_train, y_val) = (y[:split_at], y[split_at:])
print(X_train.shape)
print(y_train.shape)
#print(X_train)
#print(y_train)
#return X_train, y_train
if model is None:
print('Build model...')
model = Sequential()
model.add(RNN(HIDDEN_SIZE, input_shape=(MAXLEN_q, MAXW_q)))
model.add(RepeatVector(MAXLEN_a))
for _ in range(LAYERS):
model.add(RNN(HIDDEN_SIZE, return_sequences=True))
model.add(TimeDistributed(Dense(MAXW_a)))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
else:
print('Use existed model...')
# Train the model each generation and show predictions against the validation dataset
for iteration in range(1, 50):
print()
print('-' * 50)
print('Iteration', iteration)
model.fit(
X_train,
y_train,
batch_size=BATCH_SIZE,
nb_epoch=10, #validation_split=0.1,
validation_data=(X_val, y_val))
###
# Select samples from the validation set at random so we can visualize errors
for i in range(5):
ind = np.random.randint(0, len(X_val))
rowX, rowy = X_val[np.array([ind])], y_val[np.array([ind])]
preds = model.predict_classes(rowX, verbose=0)
#print(rowX)
#print(rowy)
#print(preds)
q = rowX[0]
correct = rowy[0]
guess = preds[0]
#print('Q', q)
print('T', correct)
#print('G', preds)
print(
colors.ok + '☑' +
colors.close if correct[0][guess[0]] > 0 else colors.fail +
'☒' + colors.close, guess)
print('---')
print('Save model ... ')
model.save('stk_rnn2.h5')
def fit(self, sast, r, batch_size=32, nb_epoch=10, shuffle=True, theta_metrics={}):
"""
Args:
s (numpy.array): the samples of the state (nsamples, state_dim)
a (numpy.array): the samples of the state (nsamples, action_dim)
s_next (numpy.array): the samples of the next (reached) state (nsamples, state_dim)
r (numpy.array): the sample of the reward (nsamples, )
theta (numpy.array): the sample of the Q-function parameters (1, n_params)
batch_size (int): dimension of the batch used for a single step of the gradient
nb_epoch (int): number of epochs
verbose (int): 0 or 1. Verbosity mode. 0 = silent, 1 = verbose.
callbacks (list): list of callbacks to be called during training.
See [Keras Callbacks](https://keras.io/callbacks/).
validation_split (float): float between 0 and 1:
fraction of the training data to be used as validation data.
The model will set apart this fraction of the training data,
will not train on it, and will evaluate the loss and any model metrics
on this data at the end of each epoch.
validation_data (tuple): data on which to evaluate the loss and any model metrics
at the end of each epoch. The model will not be trained on this data.
This could be a tuple (val_s, val_a, val_s_next, val_r) or a tuple
(val_s, val_a, val_s_next, val_r, val_theta).
shuffle (boolean): whether to shuffle the training data before each epoch.
theta_metrics (dict): dictionary storing the pairs (name: callable object).
The callable object/function is used to evaluate the Q-function parameters
at each iteration. The signature of the callable is simple: f(theta)
e.g.: theta_metrics={'k': lambda theta: evaluate(theta)})
Returns:
A PBOHistory instance storing train information
"""
sast = standardize_input_data(
sast, ["sast"], (None, 2 * self.state_dim + self.action_dim + 1), exception_prefix="sast"
)[0]
next_states_idx = self.state_dim + self.action_dim
sa = sast[:, :next_states_idx]
s_next = sast[:, next_states_idx:-1]
absorbing = sast[:, -1]
n_updates = 0
maxq, maxa = self.maxQA(s_next, absorbing)
if hasattr(self._estimator, "adapt"):
# update estimator structure
self._estimator.adapt(iteration=self._iteration)
# y = np.reshape(r + self.gamma * maxq, (-1, 1))
y = r + self.gamma * maxq
ins = [sa, y]
self._make_train_function()
f = self.train_function
nb_train_sample = sa.shape[0]
index_array = np.arange(nb_train_sample)
history = {"theta": []}
for k in theta_metrics.keys():
history.update({k: []})
for epoch in range(nb_epoch):
if shuffle == "batch":
index_array = batch_shuffle(index_array, batch_size)
elif shuffle:
np.random.shuffle(index_array)
batches = make_batches(nb_train_sample, batch_size)
for batch_index, (batch_start, batch_end) in enumerate(batches):
if hasattr(self._estimator, "_model"):
ltheta = self._model.get_weights()
history["theta"].append(ltheta)
else:
ltheta = self._estimator.get_weights()
history["theta"].append(ltheta)
for k, v in iteritems(theta_metrics):
history[k].append(v(ltheta))
batch_ids = index_array[batch_start:batch_end]
try:
if type(ins[-1]) is float:
# do not slice the training phase flag
ins_batch = slice_X(ins[:-1], batch_ids) + [ins[-1]]
else:
ins_batch = slice_X(ins, batch_ids)
except TypeError:
raise Exception(
"TypeError while preparing batch. " "If using HDF5 input data, " 'pass shuffle="batch".'
)
outs = f(*ins_batch)
n_updates += 1
if self.update_theta_every > 0 and n_updates % self.update_theta_every == 0:
maxq, maxa = self.maxQA(s_next, absorbing)
if hasattr(self._estimator, "adapt"):
# update estimator structure
self._estimator.adapt(iteration=self._iteration)
# y = np.reshape(r + self.gamma * maxq, (-1, 1))
y = r + self.gamma * maxq
ins = [ins[0], y]
if self._verbose > 1:
print("learned theta: {}".format(self._estimator.get_weights()))
return history
def main():
questions = []
expected = []
seen = set()
print('Generating data...')
train_data = load_data('000333.csv')
train_data = train_data[:TRAINING_SIZE] # for test
for x in train_data:
#print(x)
X = np.zeros((MAXLEN_q, MAXW_q))
for i, c in enumerate(x[0]):
X[i] = c
questions.append(X)
Y = np.zeros((MAXLEN_a, MAXW_a))
Y[0, x[1]] = 1
expected.append(Y)
print('Total addition questions:', len(questions))
#print(questions)
#print(expected)
print('Vectorization...')
X = np.zeros((len(questions), MAXLEN_q, MAXW_q))
y = np.zeros((len(questions), MAXW_a))
for i, sentence in enumerate(questions):
X[i] = sentence
for i, sentence in enumerate(expected):
y[i] = sentence
# Shuffle (X, y)
indices = np.arange(len(y))
np.random.shuffle(indices)
X = X[indices]
y = y[indices]
# Explicitly set apart 10% for validation data that we never train over
split_at = len(X) - len(X) / 10
(X_train, X_val) = (slice_X(X, 0, split_at), slice_X(X, split_at))
(y_train, y_val) = (y[:split_at], y[split_at:])
print(X_train.shape)
print(y_train.shape)
#print(X_train)
#print(y_train)
#return X_train, y_train
print('Build model...')
model = Sequential()
model.add(RNN(HIDDEN_SIZE, input_shape=(MAXLEN_q, MAXW_q)))
model.add(Dropout(0.5))
model.add(Dense(HIDDEN_SIZE, activation='tanh'))
model.add(Dense(MAXW_a, activation='softmax'))
sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
# Train the model each generation and show predictions against the validation dataset
for iteration in range(1, 10):
print()
print('-' * 50)
print('Iteration', iteration)
model.fit(X_train,
y_train,
batch_size=BATCH_SIZE,
nb_epoch=10,
validation_data=(X_val, y_val))
###
# Select samples from the validation set at random so we can visualize errors
for i in range(5):
ind = np.random.randint(0, len(X_val))
rowX, rowy = X_val[np.array([ind])], y_val[np.array([ind])]
preds = model.predict_classes(rowX, verbose=0)
#print(rowX)
#print(rowy)
#print(preds)
q = rowX[0]
correct = rowy[0]
guess = preds[0]
print('Q', q)
print('T', correct)
print('G', preds)
print(
colors.ok + '☑' +
colors.close if correct[guess] > 0 else colors.fail + '☒' +
colors.close, guess)
print('---')
model.save('stk_rnn.h5')
def main():
questions = []
expected = []
seen = set()
print('Generating data...')
train_data = load_data('2017-01-12.dat')
train_data = train_data[:TRAINING_SIZE] # for test
for x in train_data:
#query = ','.join(['%04d+%05d'%(i[0],i[1]) for i in x[0]][:CART_SIZE])
query = ','.join(['%04d' % i for i in x[0]][:CART_SIZE]) # 不含价格信息
ans = ','.join(['%04d' % i for i in x[1]][:CART_SIZE])
questions.append(query)
expected.append(ans)
print('Total addition questions:', len(questions))
print('Vectorization...')
X = np.zeros((len(questions), MAXLEN_q, len(chars)))
y = np.zeros((len(questions), MAXLEN_a, len(chars)))
for i, sentence in enumerate(questions):
X[i] = ctable.encode(sentence, maxlen=MAXLEN_q)
for i, sentence in enumerate(expected):
y[i] = ctable.encode(sentence, maxlen=MAXLEN_a)
# Shuffle (X, y) in unison as the later parts of X will almost all be larger digits
indices = np.arange(len(y))
np.random.shuffle(indices)
X = X[indices]
y = y[indices]
# Explicitly set apart 10% for validation data that we never train over
split_at = len(X) - len(X) / 10
(X_train, X_val) = (slice_X(X, 0, split_at), slice_X(X, split_at))
(y_train, y_val) = (y[:split_at], y[split_at:])
print(X_train.shape)
print(y_train.shape)
#print(questions)
#print(expected)
#print(X_train)
#print(y_train)
#return X_train, y_train
print('Build model...')
model = Sequential()
#model.add(Reshape((CART_SIZE, MAXLEN, len(chars)), input_shape=(CART_SIZE*MAXLEN*len(chars),)))
# "Encode" the input sequence using an RNN, producing an output of HIDDEN_SIZE
# note: in a situation where your input sequences have a variable length,
# use input_shape=(None, nb_feature).
model.add(RNN(HIDDEN_SIZE, input_shape=(MAXLEN_q, len(chars))))
# For the decoder's input, we repeat the encoded input for each time step
model.add(RepeatVector(MAXLEN_a))
# The decoder RNN could be multiple layers stacked or a single layer
for _ in range(LAYERS):
model.add(RNN(HIDDEN_SIZE, return_sequences=True))
# For each of step of the output sequence, decide which character should be chosen
model.add(TimeDistributed(Dense(len(chars))))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
# Train the model each generation and show predictions against the validation dataset
for iteration in range(1, 2):
print()
print('-' * 50)
print('Iteration', iteration)
model.fit(X_train,
y_train,
batch_size=BATCH_SIZE,
nb_epoch=1,
validation_data=(X_val, y_val))
###
# Select samples from the validation set at random so we can visualize errors
for i in range(5):
ind = np.random.randint(0, len(X_val))
rowX, rowy = X_val[np.array([ind])], y_val[np.array([ind])]
preds = model.predict_classes(rowX, verbose=0)
print(rowX)
print(rowy)
print(preds)
q = ctable.decode(rowX[0])
correct = ctable.decode(rowy[0])
guess = ctable.decode(preds[0], calc_argmax=False)
print('Q', q)
print('T', correct)
print(
colors.ok + '☑' +
colors.close if correct == guess else colors.fail + '☒' +
colors.close, guess)
print('---')
model.save('my_model_rnn.h5')
ct = lnCounts[ii]
soFar += ct
print(ii, ct, "\t", soFar / tot)
print('Vectorization...')
Xreal = np.zeros((len(words), maxlen, len(chars)), dtype=np.bool)
yreal = np.zeros((len(words), maxlen, len(chars)), dtype=np.bool)
for ii, word in enumerate(words):
#print("encoding", word, "at row", ii)
Xreal[ii] = ctable.encode(pad(word, maxlen, "X"), maxlen)
yreal[ii] = ctable.encode(pad(word, maxlen, "X"), maxlen)
# Explicitly set apart 10% for validation data that we never train over.
split_at = len(Xreal) - len(Xreal) // 10
(X_r_train, X_r_val) = (slice_X(Xreal, 0,
split_at), slice_X(Xreal, split_at))
(y_r_train, y_r_val) = (yreal[:split_at], yreal[split_at:])
Xpseud = np.zeros((len(pseudWords), maxlen, len(chars)), dtype=np.bool)
ypseud = np.zeros((len(pseudWords), maxlen, len(chars)), dtype=np.bool)
for ii, word in enumerate(pseudWords):
#print("encoding", word, "at row", ii)
Xpseud[ii] = ctable.encode(pad(word, maxlen, "X"), maxlen)
ypseud[ii] = ctable.encode(pad(word, maxlen, "X"), maxlen)
# Explicitly set apart 10% for validation data that we never train over.
split_at = len(Xpseud) - len(Xpseud) // 10
(X_p_train, X_p_val) = (slice_X(Xpseud, 0,
split_at), slice_X(Xpseud, split_at))
(y_p_train, y_p_val) = (ypseud[:split_at], ypseud[split_at:])
def fit(self, s, a, s_next, r, absorbing, theta,
batch_size=32, nb_epoch=10, shuffle=True,
theta_metrics={}):
"""
Args:
s (numpy.array): the samples of the state (nsamples, state_dim)
a (numpy.array): the samples of the state (nsamples, action_dim)
s_next (numpy.array): the samples of the next (reached) state (nsamples, state_dim)
r (numpy.array): the sample of the reward (nsamples, )
theta (numpy.array): the sample of the Q-function parameters (1, n_params)
batch_size (int): dimension of the batch used for a single step of the gradient
nb_epoch (int): number of epochs
verbose (int): 0 or 1. Verbosity mode. 0 = silent, 1 = verbose.
callbacks (list): list of callbacks to be called during training.
See [Keras Callbacks](https://keras.io/callbacks/).
validation_split (float): float between 0 and 1:
fraction of the training data to be used as validation data.
The model will set apart this fraction of the training data,
will not train on it, and will evaluate the loss and any model metrics
on this data at the end of each epoch.
validation_data (tuple): data on which to evaluate the loss and any model metrics
at the end of each epoch. The model will not be trained on this data.
This could be a tuple (val_s, val_a, val_s_next, val_r) or a tuple
(val_s, val_a, val_s_next, val_r, val_theta).
shuffle (boolean): whether to shuffle the training data before each epoch.
theta_metrics (dict): dictionary storing the pairs (name: callable object).
The callable object/function is used to evaluate the Q-function parameters
at each iteration. The signature of the callable is simple: f(theta)
e.g.: theta_metrics={'k': lambda theta: evaluate(theta)})
Returns:
A PBOHistory instance storing train information
"""
s, a, s_next, r, absorbing, theta = self._standardize_user_data(
s, a, s_next, r, absorbing, theta,
check_batch_dim=False
)
all_actions = standardize_input_data(
self.discrete_actions, ['all_actions'],
[(None, self.action_dim)] if self.action_dim is not None else None,
exception_prefix='discrete_actions')
n_updates = 0
history = {"theta": [], 'rho': []}
for k in theta_metrics.keys():
history.update({k: []})
ins = s + a + s_next + [r, absorbing]
self._make_train_function()
f = self.train_function
nb_train_sample = ins[0].shape[0]
index_array = np.arange(nb_train_sample)
# append evolution of theta for independent case
for _ in range(len(self.theta_list) - 1):
if self.incremental:
tmp = theta[-1] + self.bellman_model.predict(theta[-1])
else:
tmp = self.bellman_model.predict(theta[-1])
theta += [tmp]
term_condition = self.term_condition
stop = False
old_theta = theta
for epoch in range(nb_epoch):
if stop:
break
if shuffle == 'batch':
index_array = batch_shuffle(index_array, batch_size)
elif shuffle:
np.random.shuffle(index_array)
batches = make_batches(nb_train_sample, batch_size)
for batch_index, (batch_start, batch_end) in enumerate(batches):
history["theta"].append(theta[0])
if hasattr(self.bellman_model, '_model'):
history["rho"].append(
self.bellman_model._model.get_weights())
else:
history["rho"].append(self.bellman_model.get_weights())
for k, v in iteritems(theta_metrics):
history[k].append(v(theta))
batch_ids = index_array[batch_start:batch_end]
try:
if type(ins[-1]) is float:
# do not slice the training phase flag
ins_batch = slice_X(ins[:-1], batch_ids) + [ins[-1]]
else:
ins_batch = slice_X(ins, batch_ids)
except TypeError:
raise Exception('TypeError while preparing batch. '
'If using HDF5 input data, '
'pass shuffle="batch".')
inp = ins_batch + theta + all_actions
outs = f(*inp)
n_updates += 1
if self.update_theta_every > 0 and n_updates % self.update_theta_every == 0:
tmp = self.apply_bo(theta[0],
n_times=self.steps_per_theta_update)
theta = [tmp]
for _ in range(len(self.theta_list) - 1):
if self.incremental:
tmp = tmp + self.bellman_model.predict(tmp)
else:
tmp = self.bellman_model.predict(tmp)
theta += [tmp]
if term_condition is not None:
stop = term_condition(old_theta, theta)
if stop:
break
old_theta = theta
# finally apply the bellman operator K-times to get the final point
self.learned_theta_value = self.apply_bo(theta[0], n_times=100)
if self.verbose > 1:
print('learned theta: {}'.format(self.learned_theta_value))
self.history = history
return history
print('Generating data')
datagen = DataGenerator(FLAGS.TRAINING_SIZE, 0, FLAGS.DIGITS,
FLAGS.DATASET_TYPE)
#print( 'Validating data' )
#datagen.validate()
X, y = datagen.train_X, datagen.train_y
print(X.shape)
print(y.shape)
# Explicitly set apart 10% for validation data that we never train over
split_at = len(X) - len(X) / 10
X_train, X_val = slice_X(X, 0, split_at), slice_X(X, split_at)
y_train, y_val = y[:split_at], y[split_at:]
print('Build model')
model = Sequential()
# "Encode" the input sequence using an RNN, producing an output of HIDDEN_SIZE
# note: in a situation where your input sequences have a variable length,
# use input_shape=(None, nb_feature).
left = Sequential()
left.add(
RNN(FLAGS.HIDDEN_SIZE,
input_shape=[datagen.sent_maxlen, datagen.alphabet.maxlen],
return_sequences=True))
if FLAGS.BIDIRECTIONAL:
right = Sequential()
