def _predict_loop(self, f, ins, batch_size=128, verbose=0):
'''
Abstract method to loop over some data in batches.
'''
nb_sample = len(ins[0])
outs = []
if verbose == 1:
progbar = Progbar(target=nb_sample)
batches = make_batches(nb_sample, batch_size)
index_array = np.arange(nb_sample)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
ins_batch = slice_X(ins, batch_ids)
batch_outs = f(self, ins_batch)
if type(batch_outs) != list:
batch_outs = [batch_outs]
if batch_index == 0:
for batch_out in batch_outs:
shape = (nb_sample, ) + batch_out.shape[1:]
outs.append(np.zeros(shape))
for i, batch_out in enumerate(batch_outs):
outs[i][batch_start:batch_end] = batch_out
if verbose == 1:
progbar.update(batch_end)
return outs
def _test_loop(self, f, ins, batch_size=128, verbose=0):
'''
Abstract method to loop over some data in batches.
'''
nb_sample = len(ins[0])
outs = []
if verbose == 1:
progbar = Progbar(target=nb_sample)
batches = make_batches(nb_sample, batch_size)
index_array = np.arange(nb_sample)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
ins_batch = slice_X(ins, batch_ids)
batch_outs = f(self, ins_batch)
if type(batch_outs) == list:
if batch_index == 0:
for batch_out in enumerate(batch_outs):
outs.append(0.)
for i, batch_out in enumerate(batch_outs):
outs[i] += batch_out * len(batch_ids)
else:
if batch_index == 0:
outs.append(0.)
outs[0] += batch_outs * len(batch_ids)
if verbose == 1:
progbar.update(batch_end)
for i, out in enumerate(outs):
outs[i] /= nb_sample
return outs
def _predict_loop(self, f, ins, batch_size=128, verbose=0):
'''
Abstract method to loop over some data in batches.
'''
nb_sample = len(ins[0])
outs = []
if verbose == 1:
progbar = Progbar(target=nb_sample)
batches = make_batches(nb_sample, batch_size)
index_array = np.arange(nb_sample)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
ins_batch = slice_X(ins, batch_ids)
batch_outs = f(self, ins_batch)
if type(batch_outs) != list:
batch_outs = [batch_outs]
if batch_index == 0:
for batch_out in batch_outs:
shape = (nb_sample,) + batch_out.shape[1:]
outs.append(np.zeros(shape))
for i, batch_out in enumerate(batch_outs):
outs[i][batch_start:batch_end] = batch_out
if verbose == 1:
progbar.update(batch_end)
return outs
def _test_loop(self, f, ins, batch_size=128, verbose=0):
'''
Abstract method to loop over some data in batches.
'''
nb_sample = len(ins[0])
outs = []
if verbose == 1:
progbar = Progbar(target=nb_sample)
batches = make_batches(nb_sample, batch_size)
index_array = np.arange(nb_sample)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
ins_batch = slice_X(ins, batch_ids)
batch_outs = f(self, ins_batch)
if type(batch_outs) == list:
if batch_index == 0:
for batch_out in enumerate(batch_outs):
outs.append(0.)
for i, batch_out in enumerate(batch_outs):
outs[i] += batch_out * len(batch_ids)
else:
if batch_index == 0:
outs.append(0.)
outs[0] += batch_outs * len(batch_ids)
if verbose == 1:
progbar.update(batch_end)
for i, out in enumerate(outs):
outs[i] /= nb_sample
return outs
def data_generator(x, y, batch_size=50):
index_array = np.arange(len(x))
while 1:
batches = make_batches(len(X_test), batch_size)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
x_batch = x[batch_ids]
y_batch = y[batch_ids]
yield (x_batch, y_batch)
def data_generator(x, y, batch_size=50):
index_array = np.arange(len(x))
while 1:
batches = make_batches(len(X_test), batch_size)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
x_batch = x[batch_ids]
y_batch = y[batch_ids]
yield (x_batch, y_batch)
def mix_training_inputs(x_train, y_train, pu_x_train, pu_y_train, pu_aux_train, discr_pt_cut=14., tile=15):
# Apply veto on PU events with a muon with pT > 14 GeV
pu_x_train_tmp = ~(pu_aux_train[:,2] > discr_pt_cut)
pu_x_train = pu_x_train[pu_x_train_tmp]
pu_y_train = [pu_y_train[0][pu_x_train_tmp], pu_y_train[1][pu_x_train_tmp]]
# Put together x_train & pu_x_train, y_train & pu_y_train
assert(len(pu_y_train) == 2)
assert(pu_x_train.shape[0] == pu_y_train[0].shape[0])
assert(pu_x_train.shape[0] == pu_y_train[1].shape[0])
num_samples = pu_x_train.shape[0]
index_array = np.arange(num_samples)
index_array_ext = np.tile(index_array, tile) # choose tile to make sure pu_x_train_ext has more entries than x_train
pu_x_train_ext = pu_x_train[index_array_ext]
pu_y_train_ext = [pu_y_train[0][index_array_ext], pu_y_train[1][index_array_ext]]
assert(len(y_train) == 2)
assert(x_train.shape[0] == y_train[0].shape[0])
assert(x_train.shape[0] == y_train[1].shape[0])
if not (pu_x_train_ext.shape[0] >= x_train.shape[0]):
raise Exception('pu_x_train_ext is required to have more entries than x_train. Make sure {0} >= {1}'.format(pu_x_train_ext.shape[0], x_train.shape[0]))
num_samples = x_train.shape[0]
index_array = np.arange(num_samples)
#np.random.shuffle(index_array)
try:
from keras.engine.training import _make_batches as make_batches
except ImportError:
from keras.engine.training_utils import make_batches
sample_batch_size = 128
batches = make_batches(num_samples, sample_batch_size)
x_train_new = np.zeros((num_samples*2, x_train.shape[1]), dtype=np.float32)
y_train_new = [np.zeros((num_samples*2,), dtype=np.float32), np.zeros((num_samples*2,), dtype=np.float32)]
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
x_train_new[batch_start*2:batch_start*2 + (batch_end-batch_start)] = x_train[batch_ids]
x_train_new[batch_start*2 + (batch_end-batch_start):batch_end*2] = pu_x_train_ext[batch_ids]
y_train_new[0][batch_start*2:batch_start*2 + (batch_end-batch_start)] = y_train[0][batch_ids]
y_train_new[0][batch_start*2 + (batch_end-batch_start):batch_end*2] = pu_y_train_ext[0][batch_ids]
y_train_new[1][batch_start*2:batch_start*2 + (batch_end-batch_start)] = y_train[1][batch_ids]
y_train_new[1][batch_start*2 + (batch_end-batch_start):batch_end*2] = pu_y_train_ext[1][batch_ids]
logger.info('Mixed muon data with pileup data. x_train_new has shape {0}, y_train_new has shape {1},{2}'.format(x_train_new.shape, y_train_new[0].shape, y_train_new[1].shape))
return x_train_new, y_train_new
def _fit_loop(self,
f,
ins,
discrete_actions,
out_labels=[],
batch_size=32,
nb_epoch=100,
verbose=1,
callbacks=[],
val_f=None,
val_ins=None,
shuffle=True,
callback_metrics=[],
theta_metrics={}):
do_validation = False
if val_f and val_ins:
do_validation = True
if verbose:
print('Train on %d samples, validate on %d samples' %
(ins[0].shape[0], val_ins[0].shape[0]))
nb_train_sample = ins[0].shape[0]
print(nb_train_sample)
index_array = np.arange(nb_train_sample)
history = PBOHistory()
callbacks = [history] + callbacks
if verbose:
callbacks += [cbks.ProgbarLogger()]
callbacks = cbks.CallbackList(callbacks)
callback_model = self
callbacks._set_model(callback_model)
callbacks._set_params({
'batch_size':
batch_size,
'nb_epoch':
nb_epoch,
'nb_sample':
nb_train_sample,
'verbose':
verbose,
'do_validation':
do_validation,
'metrics':
callback_metrics + [el for el in theta_metrics.keys()],
'theta':
self.q_model.trainable_weights[0].eval()
})
callbacks.on_train_begin()
callback_model.stop_training = False
for epoch in range(nb_epoch):
callbacks.on_epoch_begin(epoch)
if shuffle == 'batch':
index_array = batch_shuffle(index_array, batch_size)
elif shuffle:
np.random.shuffle(index_array)
batches = make_batches(nb_train_sample, batch_size)
epoch_logs = {}
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
try:
if type(ins[-1]) is float:
# do not slice the training phase flag
ins_batch = slice_X(ins[:-1], batch_ids) + [ins[-1]]
else:
ins_batch = slice_X(ins, batch_ids)
except TypeError:
raise Exception('TypeError while preparing batch. '
'If using HDF5 input data, '
'pass shuffle="batch".')
batch_logs = {}
batch_logs['batch'] = batch_index
batch_logs['size'] = len(batch_ids)
batch_logs['theta'] = self.q_model.trainable_weights[0].eval()
for k in theta_metrics.keys():
batch_logs[k] = theta_metrics[k](
self.q_model.trainable_weights)
callbacks.on_batch_begin(batch_index, batch_logs)
inp = ins_batch + discrete_actions
outs = f(*inp)
if type(outs) != list:
outs = [outs]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
if batch_index == len(batches) - 1: # last batch
# validation
if do_validation:
# replace with self._evaluate
val_outs = self._test_loop(val_f,
val_ins,
batch_size=batch_size,
verbose=0)
if type(val_outs) != list:
val_outs = [val_outs]
# same labels assumed
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_epoch_end(epoch, epoch_logs)
callbacks.on_train_end()
return history
def predict(model, data, *args, **kwargs):
"""Make predictions given a model and data
Args:
model(dict): a serialized keras models
data(list, dict, np.array): data to be passed as a dictionary mapping
inputs names to np.arrays or a list of arrays or an arrays
Returns:
an np.array of predictions
"""
from keras.engine.training import make_batches
if kwargs.get("batch_size") is None: # pragma: no cover
kwargs['batch_size'] = 32
batch_size = kwargs['batch_size']
custom_objects = kwargs.get('custom_objects')
model_name = model['model_arch']['config'].get('class_name')
# check if the predict function is already compiled
if model['mod_id'] in COMPILED_MODELS:
pred_function = COMPILED_MODELS[model['mod_id']]['pred']
model_k = COMPILED_MODELS[model['mod_id']]['model']
learning_phase = COMPILED_MODELS[model['mod_id']]['learning_phase']
output_shape = COMPILED_MODELS[model['mod_id']]['model'].output_shape
else:
# get the model arch
model_dict = model['model_arch']
model_dict.pop('optimizer')
# load model
model_k = model_from_dict_w_opt(model_dict,
custom_objects=custom_objects)
# load the weights
model_k.load_weights(model['params_dump'])
# build the prediction function
pred_function = build_predict_func(model_k)
COMPILED_MODELS[model['mod_id']] = dict()
COMPILED_MODELS[model['mod_id']]['pred'] = pred_function
COMPILED_MODELS[model['mod_id']]['model'] = model_k
learning_phase = model_k.uses_learning_phase
COMPILED_MODELS[model['mod_id']]['learning_phase'] = learning_phase
output_shape = model_k.output_shape
# predict according to the input/output type
if model_name == 'Sequential':
if not isinstance(data, list):
data = [data]
elif model_name == 'Model':
if isinstance(data, dict):
data = [data[k] for k in model_k.input_names]
elif not isinstance(data, list):
data = [data]
else:
raise NotImplementedError(
'{}: This type of model is not supported'.format(model_name))
# Predict by batch to control GPU memory
len_data = len(data[0])
batches = make_batches(len_data, batch_size)
index_array = np.arange(len_data)
results_array = np.empty((len_data, ) + output_shape[1:])
for batch_start, batch_end in batches:
batch_ids = index_array[batch_start:batch_end]
data_b = [d[batch_ids] for d in data]
if learning_phase:
data_b.append(0.)
batch_prediction = pred_function(data_b)
if isinstance(batch_prediction, list): # pragma: no cover
batch_prediction = batch_prediction[0]
results_array[batch_ids] = batch_prediction
return results_array
def fit(self,
s,
a,
s_next,
r,
absorbing,
theta,
batch_size=32,
nb_epoch=10,
shuffle=True,
theta_metrics={}):
"""
Args:
s (numpy.array): the samples of the state (nsamples, state_dim)
a (numpy.array): the samples of the state (nsamples, action_dim)
s_next (numpy.array): the samples of the next (reached) state (nsamples, state_dim)
r (numpy.array): the sample of the reward (nsamples, )
theta (numpy.array): the sample of the Q-function parameters (1, n_params)
batch_size (int): dimension of the batch used for a single step of the gradient
nb_epoch (int): number of epochs
verbose (int): 0 or 1. Verbosity mode. 0 = silent, 1 = verbose.
callbacks (list): list of callbacks to be called during training.
See [Keras Callbacks](https://keras.io/callbacks/).
validation_split (float): float between 0 and 1:
fraction of the training data to be used as validation data.
The model will set apart this fraction of the training data,
will not train on it, and will evaluate the loss and any model metrics
on this data at the end of each epoch.
validation_data (tuple): data on which to evaluate the loss and any model metrics
at the end of each epoch. The model will not be trained on this data.
This could be a tuple (val_s, val_a, val_s_next, val_r) or a tuple
(val_s, val_a, val_s_next, val_r, val_theta).
shuffle (boolean): whether to shuffle the training data before each epoch.
theta_metrics (dict): dictionary storing the pairs (name: callable object).
The callable object/function is used to evaluate the Q-function parameters
at each iteration. The signature of the callable is simple: f(theta)
e.g.: theta_metrics={'k': lambda theta: evaluate(theta)})
Returns:
A PBOHistory instance storing train information
"""
s, a, s_next, r, absorbing, theta = self._standardize_user_data(
s, a, s_next, r, absorbing, theta, check_batch_dim=False)
all_actions = standardize_input_data(
self.discrete_actions, ['all_actions'],
[(None, self.action_dim)] if self.action_dim is not None else None,
exception_prefix='discrete_actions')
n_updates = 0
history = {"theta": [], 'rho': []}
for k in theta_metrics.keys():
history.update({k: []})
ins = s + a + s_next + [r, absorbing]
self._make_train_function()
f = self.train_function
nb_train_sample = ins[0].shape[0]
index_array = np.arange(nb_train_sample)
# append evolution of theta for independent case
for _ in range(len(self.theta_list) - 1):
if self.incremental:
tmp = theta[-1] + self.bellman_model.predict(theta[-1])
else:
tmp = self.bellman_model.predict(theta[-1])
theta += [tmp]
term_condition = self.term_condition
stop = False
old_theta = theta
for epoch in range(nb_epoch):
if stop:
break
if shuffle == 'batch':
index_array = batch_shuffle(index_array, batch_size)
elif shuffle:
np.random.shuffle(index_array)
batches = make_batches(nb_train_sample, batch_size)
for batch_index, (batch_start, batch_end) in enumerate(batches):
history["theta"].append(theta[0])
if hasattr(self.bellman_model, '_model'):
history["rho"].append(
self.bellman_model._model.get_weights())
else:
history["rho"].append(self.bellman_model.get_weights())
for k, v in iteritems(theta_metrics):
history[k].append(v(theta))
batch_ids = index_array[batch_start:batch_end]
try:
if type(ins[-1]) is float:
# do not slice the training phase flag
ins_batch = slice_X(ins[:-1], batch_ids) + [ins[-1]]
else:
ins_batch = slice_X(ins, batch_ids)
except TypeError:
raise Exception('TypeError while preparing batch. '
'If using HDF5 input data, '
'pass shuffle="batch".')
inp = ins_batch + theta + all_actions
outs = f(*inp)
n_updates += 1
if self.update_theta_every > 0 and n_updates % self.update_theta_every == 0:
tmp = self.apply_bo(theta[0],
n_times=self.steps_per_theta_update)
theta = [tmp]
for _ in range(len(self.theta_list) - 1):
if self.incremental:
tmp = tmp + self.bellman_model.predict(tmp)
else:
tmp = self.bellman_model.predict(tmp)
theta += [tmp]
if term_condition is not None:
stop = term_condition(old_theta, theta)
if stop:
break
old_theta = theta
# finally apply the bellman operator K-times to get the final point
self.learned_theta_value = self.apply_bo(theta[0], n_times=100)
if self.verbose > 1:
print('learned theta: {}'.format(self.learned_theta_value))
self.history = history
return history
def batch_generator(dataset, indexes, indexes_size, arg1_len, arg2_len, conn_len, punc_len, batch_size, random_per_sample=1):
"""Batch generator where each sample represents a different discourse relation."""
batch_size = int(batch_size / (1.0 + random_per_sample))
rel_ids = list(dataset['rel_ids']) # copy list
while True:
# shuffle relations on each epoch
random.shuffle(rel_ids)
for batch_start, batch_end in make_batches(len(rel_ids), batch_size):
# prepare batch data
_rel_id = []
data_in = {}
data_out = {}
for rel_id in rel_ids[batch_start:batch_end]:
# choose sample
doc_id = dataset['rel_parts'][rel_id]['DocID']
words_len = len(dataset['words'][doc_id])
_rel_id.append(rel_id)
def tokens_np(token_ids, max_len):
words_slice = [ dataset['words'][doc_id][i] for i in token_ids ]
ids = map_sequence(words_slice, indexes['words2id'])
x_np = pad_sequence(ids, max_len, value=0)
return x_np
# arg1/2/conn/punc/rand inputs
arg1_np = tokens_np(dataset['rel_parts'][rel_id]['Arg1'], arg1_len)
try:
data_in['arg1_ids'].append(arg1_np)
except KeyError:
data_in['arg1_ids'] = [arg1_np]
arg2_np = tokens_np(dataset['rel_parts'][rel_id]['Arg2'], arg2_len)
try:
data_in['arg2_ids'].append(arg2_np)
except KeyError:
data_in['arg2_ids'] = [arg2_np]
conn_np = tokens_np(dataset['rel_parts'][rel_id]['Connective'], conn_len)
try:
data_in['conn_ids'].append(conn_np)
except KeyError:
data_in['conn_ids'] = [conn_np]
punc_np = tokens_np(dataset['rel_parts'][rel_id]['Punctuation'], punc_len)
try:
data_in['punc_ids'].append(punc_np)
except KeyError:
data_in['punc_ids'] = [punc_np]
# relation senses output
def rsenses_np(cat, oov_key=""):
try:
i = indexes['rel_senses2id'][cat]
except KeyError: # missing in vocabulary
i = indexes['rel_senses2id'][oov_key]
x_np = np.zeros((indexes_size['rel_senses2id'],), dtype=np.float32)
x_np[i] = 1
return x_np
if dataset['rel_senses']:
rsenses = rsenses_np(dataset['rel_senses'][rel_id])
else:
rsenses = rsenses_np("")
try:
data_out['rsenses'].append(rsenses)
except KeyError:
data_out['rsenses'] = [rsenses]
# random noise for each sample
for _ in range(random_per_sample):
j = np.random.randint(3)
if j == 0:
# random arg1
arg1_np = np.random.randint(1, 1 + np.max(arg1_np), size=arg1_np.shape)
elif j == 1:
# random arg2
arg2_np = np.random.randint(1, 1 + np.max(arg2_np), size=arg2_np.shape)
else:
# random arg1 and arg2
arg1_np = np.random.randint(1, 1 + np.max(arg1_np), size=arg1_np.shape)
arg2_np = np.random.randint(1, 1 + np.max(arg2_np), size=arg2_np.shape)
# random connective
if np.max(conn_np) > 0:
conn_np = np.random.randint(1, 1 + np.max(conn_np), size=conn_np.shape)
# random punctuation
if np.max(punc_np) > 0:
punc_np = np.random.randint(1, 1 + np.max(punc_np), size=punc_np.shape)
# mark as out-of-vocabulary in rsenses
rsenses = rsenses_np("")
_rel_id.append(rel_id)
data_in['arg1_ids'].append(arg1_np)
data_in['arg2_ids'].append(arg2_np)
data_in['conn_ids'].append(conn_np)
data_in['punc_ids'].append(punc_np)
data_out['rsenses'].append(rsenses)
# convert to NumPy array
for k, v in data_in.items():
data_in[k] = np.asarray(v)
for k, v in data_out.items():
data_out[k] = np.asarray(v)
# append meta data
data_in['_rel_id'] = _rel_id
# yield batch
yield (data_in, data_out)
# print('Evaluating target samples on source-only model')
# print('Accuracy: ', src_model.evaluate(XT_test, y_test)[1])
# Broken out training loop for a DANN model.
src_index_arr = np.arange(X_train.shape[0])
target_index_arr = np.arange(XT_train.shape[0])
batches_per_epoch = len(X_train) / batch_size
num_steps = nb_epoch * batches_per_epoch
j = 0
print('Training DANN model')
for i in range(nb_epoch):
batches = make_batches(X_train.shape[0], batch_size / 2)
target_batches = make_batches(XT_train.shape[0], batch_size / 2)
src_gen = batch_gen(batches, src_index_arr, X_train, y_train)
target_gen = batch_gen(target_batches, target_index_arr, XT_train, None)
losses = list()
acc = list()
print('Epoch ', i)
for (xb, yb) in src_gen:
# Update learning rate and gradient multiplier as described in
# the paper.
p = float(j) / num_steps
def fit(self, sast, r, batch_size=32, nb_epoch=10, shuffle=True, theta_metrics={}):
"""
Args:
s (numpy.array): the samples of the state (nsamples, state_dim)
a (numpy.array): the samples of the state (nsamples, action_dim)
s_next (numpy.array): the samples of the next (reached) state (nsamples, state_dim)
r (numpy.array): the sample of the reward (nsamples, )
theta (numpy.array): the sample of the Q-function parameters (1, n_params)
batch_size (int): dimension of the batch used for a single step of the gradient
nb_epoch (int): number of epochs
verbose (int): 0 or 1. Verbosity mode. 0 = silent, 1 = verbose.
callbacks (list): list of callbacks to be called during training.
See [Keras Callbacks](https://keras.io/callbacks/).
validation_split (float): float between 0 and 1:
fraction of the training data to be used as validation data.
The model will set apart this fraction of the training data,
will not train on it, and will evaluate the loss and any model metrics
on this data at the end of each epoch.
validation_data (tuple): data on which to evaluate the loss and any model metrics
at the end of each epoch. The model will not be trained on this data.
This could be a tuple (val_s, val_a, val_s_next, val_r) or a tuple
(val_s, val_a, val_s_next, val_r, val_theta).
shuffle (boolean): whether to shuffle the training data before each epoch.
theta_metrics (dict): dictionary storing the pairs (name: callable object).
The callable object/function is used to evaluate the Q-function parameters
at each iteration. The signature of the callable is simple: f(theta)
e.g.: theta_metrics={'k': lambda theta: evaluate(theta)})
Returns:
A PBOHistory instance storing train information
"""
sast = standardize_input_data(
sast, ["sast"], (None, 2 * self.state_dim + self.action_dim + 1), exception_prefix="sast"
)[0]
next_states_idx = self.state_dim + self.action_dim
sa = sast[:, :next_states_idx]
s_next = sast[:, next_states_idx:-1]
absorbing = sast[:, -1]
n_updates = 0
maxq, maxa = self.maxQA(s_next, absorbing)
if hasattr(self._estimator, "adapt"):
# update estimator structure
self._estimator.adapt(iteration=self._iteration)
# y = np.reshape(r + self.gamma * maxq, (-1, 1))
y = r + self.gamma * maxq
ins = [sa, y]
self._make_train_function()
f = self.train_function
nb_train_sample = sa.shape[0]
index_array = np.arange(nb_train_sample)
history = {"theta": []}
for k in theta_metrics.keys():
history.update({k: []})
for epoch in range(nb_epoch):
if shuffle == "batch":
index_array = batch_shuffle(index_array, batch_size)
elif shuffle:
np.random.shuffle(index_array)
batches = make_batches(nb_train_sample, batch_size)
for batch_index, (batch_start, batch_end) in enumerate(batches):
if hasattr(self._estimator, "_model"):
ltheta = self._model.get_weights()
history["theta"].append(ltheta)
else:
ltheta = self._estimator.get_weights()
history["theta"].append(ltheta)
for k, v in iteritems(theta_metrics):
history[k].append(v(ltheta))
batch_ids = index_array[batch_start:batch_end]
try:
if type(ins[-1]) is float:
# do not slice the training phase flag
ins_batch = slice_X(ins[:-1], batch_ids) + [ins[-1]]
else:
ins_batch = slice_X(ins, batch_ids)
except TypeError:
raise Exception(
"TypeError while preparing batch. " "If using HDF5 input data, " 'pass shuffle="batch".'
)
outs = f(*ins_batch)
n_updates += 1
if self.update_theta_every > 0 and n_updates % self.update_theta_every == 0:
maxq, maxa = self.maxQA(s_next, absorbing)
if hasattr(self._estimator, "adapt"):
# update estimator structure
self._estimator.adapt(iteration=self._iteration)
# y = np.reshape(r + self.gamma * maxq, (-1, 1))
y = r + self.gamma * maxq
ins = [ins[0], y]
if self._verbose > 1:
print("learned theta: {}".format(self._estimator.get_weights()))
return history
def _fit_loop(self, f, ins, out_labels=None, batch_size=32,
nb_epoch=100, verbose=1, callbacks=None,
val_f=None, val_ins=None, shuffle=True,
callback_metrics=None, initial_epoch=0):
"""Abstract fit function for f(ins).
Assume that f returns a list, labeled by out_labels.
# Arguments
f: Keras function returning a list of tensors
ins: list of tensors to be fed to `f`
out_labels: list of strings, display names of
the outputs of `f`
batch_size: integer batch size
nb_epoch: number of times to iterate over the data
verbose: verbosity mode, 0, 1 or 2
callbacks: list of callbacks to be called during training
val_f: Keras function to call for validation
val_ins: list of tensors to be fed to `val_f`
shuffle: whether to shuffle the data at the beginning of each epoch
callback_metrics: list of strings, the display names of the metrics
passed to the callbacks. They should be the
concatenation of list the display names of the outputs of
`f` and the list of display names of the outputs of `f_val`.
initial_epoch: epoch at which to start training
(useful for resuming a previous training run)
# Returns
`History` object.
[A tweaked version.]
"""
do_validation = False
if val_f and val_ins:
do_validation = True
if verbose:
print('Train on %d samples, validate on %d samples' %
(ins[0].shape[0], val_ins[0].shape[0]))
nb_train_sample = ins[0].shape[0]
index_array = np.arange(nb_train_sample)
self.history = cbks.History()
callbacks = [cbks.BaseLogger()] + (callbacks or []) + [self.history]
if verbose:
callbacks += [cbks.ProgbarLogger()]
callbacks = cbks.CallbackList(callbacks)
out_labels = out_labels or []
# it's possible to callback a different model than self
# (used by Sequential models)
if hasattr(self, 'callback_model') and self.callback_model:
callback_model = self.callback_model
else:
callback_model = self
callbacks.set_model(callback_model)
callbacks.set_params({
'batch_size': batch_size,
'nb_epoch': nb_epoch,
'nb_sample': nb_train_sample,
'verbose': verbose,
'do_validation': do_validation,
'metrics': callback_metrics or [],
})
callbacks.on_train_begin()
callback_model.stop_training = False
self.validation_data = val_ins
for epoch in range(initial_epoch, nb_epoch):
callbacks.on_epoch_begin(epoch)
if shuffle == 'batch':
index_array = batch_shuffle(index_array, batch_size)
elif shuffle:
np.random.shuffle(index_array)
batches = make_batches(nb_train_sample, batch_size)
epoch_logs = {}
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
try:
if isinstance(ins[-1], float):
# do not slice the training phase flag
ins_batch = slice_X(ins[:-1], batch_ids) + [ins[-1]]
else:
ins_batch = slice_X(ins, batch_ids)
except TypeError:
raise TypeError('TypeError while preparing batch. '
'If using HDF5 input data, '
'pass shuffle="batch".')
batch_logs = {}
batch_logs['batch'] = batch_index
batch_logs['size'] = len(batch_ids)
batch_logs['ids'] = batch_ids
callbacks.on_batch_begin(batch_index, batch_logs)
outs = f(ins_batch)
if not isinstance(outs, list):
outs = [outs]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
if batch_index == len(batches) - 1: # last batch
# validation
if do_validation:
# replace with self._evaluate
val_outs = self._test_loop(val_f, val_ins,
batch_size=batch_size,
verbose=0)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# same labels assumed
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_epoch_end(epoch, epoch_logs)
if callback_model.stop_training:
break
callbacks.on_train_end()
return self.history
def fit(self,
sast,
r,
batch_size=32,
nb_epoch=10,
shuffle=True,
theta_metrics={}):
"""
Args:
s (numpy.array): the samples of the state (nsamples, state_dim)
a (numpy.array): the samples of the state (nsamples, action_dim)
s_next (numpy.array): the samples of the next (reached) state (nsamples, state_dim)
r (numpy.array): the sample of the reward (nsamples, )
theta (numpy.array): the sample of the Q-function parameters (1, n_params)
batch_size (int): dimension of the batch used for a single step of the gradient
nb_epoch (int): number of epochs
verbose (int): 0 or 1. Verbosity mode. 0 = silent, 1 = verbose.
callbacks (list): list of callbacks to be called during training.
See [Keras Callbacks](https://keras.io/callbacks/).
validation_split (float): float between 0 and 1:
fraction of the training data to be used as validation data.
The model will set apart this fraction of the training data,
will not train on it, and will evaluate the loss and any model metrics
on this data at the end of each epoch.
validation_data (tuple): data on which to evaluate the loss and any model metrics
at the end of each epoch. The model will not be trained on this data.
This could be a tuple (val_s, val_a, val_s_next, val_r) or a tuple
(val_s, val_a, val_s_next, val_r, val_theta).
shuffle (boolean): whether to shuffle the training data before each epoch.
theta_metrics (dict): dictionary storing the pairs (name: callable object).
The callable object/function is used to evaluate the Q-function parameters
at each iteration. The signature of the callable is simple: f(theta)
e.g.: theta_metrics={'k': lambda theta: evaluate(theta)})
Returns:
A PBOHistory instance storing train information
"""
sast = standardize_input_data(
sast, ['sast'], (None, 2 * self.state_dim + self.action_dim + 1),
exception_prefix='sast')[0]
next_states_idx = self.state_dim + self.action_dim
sa = sast[:, :next_states_idx]
s_next = sast[:, next_states_idx:-1]
absorbing = sast[:, -1]
n_updates = 0
maxq, maxa = self.maxQA(s_next, absorbing)
if hasattr(self._estimator, 'adapt'):
# update estimator structure
self._estimator.adapt(iteration=self._iteration)
# y = np.reshape(r + self.gamma * maxq, (-1, 1))
y = r + self.gamma * maxq
ins = [sa, y]
self._make_train_function()
f = self.train_function
nb_train_sample = sa.shape[0]
index_array = np.arange(nb_train_sample)
history = {"theta": []}
for k in theta_metrics.keys():
history.update({k: []})
for epoch in range(nb_epoch):
if shuffle == 'batch':
index_array = batch_shuffle(index_array, batch_size)
elif shuffle:
np.random.shuffle(index_array)
batches = make_batches(nb_train_sample, batch_size)
for batch_index, (batch_start, batch_end) in enumerate(batches):
if hasattr(self._estimator, '_model'):
ltheta = self._model.get_weights()
history["theta"].append(ltheta)
else:
ltheta = self._estimator.get_weights()
history["theta"].append(ltheta)
for k, v in iteritems(theta_metrics):
history[k].append(v(ltheta))
batch_ids = index_array[batch_start:batch_end]
try:
if type(ins[-1]) is float:
# do not slice the training phase flag
ins_batch = slice_X(ins[:-1], batch_ids) + [ins[-1]]
else:
ins_batch = slice_X(ins, batch_ids)
except TypeError:
raise Exception('TypeError while preparing batch. '
'If using HDF5 input data, '
'pass shuffle="batch".')
outs = f(*ins_batch)
n_updates += 1
if self.update_theta_every > 0 \
and n_updates % self.update_theta_every == 0:
maxq, maxa = self.maxQA(s_next, absorbing)
if hasattr(self._estimator, 'adapt'):
# update estimator structure
self._estimator.adapt(iteration=self._iteration)
# y = np.reshape(r + self.gamma * maxq, (-1, 1))
y = r + self.gamma * maxq
ins = [ins[0], y]
if self._verbose > 1:
print('learned theta: {}'.format(self._estimator.get_weights()))
return history
def fit(self, s, a, s_next, r, absorbing, theta,
batch_size=32, nb_epoch=10, shuffle=True,
theta_metrics={}):
"""
Args:
s (numpy.array): the samples of the state (nsamples, state_dim)
a (numpy.array): the samples of the state (nsamples, action_dim)
s_next (numpy.array): the samples of the next (reached) state (nsamples, state_dim)
r (numpy.array): the sample of the reward (nsamples, )
theta (numpy.array): the sample of the Q-function parameters (1, n_params)
batch_size (int): dimension of the batch used for a single step of the gradient
nb_epoch (int): number of epochs
verbose (int): 0 or 1. Verbosity mode. 0 = silent, 1 = verbose.
callbacks (list): list of callbacks to be called during training.
See [Keras Callbacks](https://keras.io/callbacks/).
validation_split (float): float between 0 and 1:
fraction of the training data to be used as validation data.
The model will set apart this fraction of the training data,
will not train on it, and will evaluate the loss and any model metrics
on this data at the end of each epoch.
validation_data (tuple): data on which to evaluate the loss and any model metrics
at the end of each epoch. The model will not be trained on this data.
This could be a tuple (val_s, val_a, val_s_next, val_r) or a tuple
(val_s, val_a, val_s_next, val_r, val_theta).
shuffle (boolean): whether to shuffle the training data before each epoch.
theta_metrics (dict): dictionary storing the pairs (name: callable object).
The callable object/function is used to evaluate the Q-function parameters
at each iteration. The signature of the callable is simple: f(theta)
e.g.: theta_metrics={'k': lambda theta: evaluate(theta)})
Returns:
A PBOHistory instance storing train information
"""
s, a, s_next, r, absorbing, theta = self._standardize_user_data(
s, a, s_next, r, absorbing, theta,
check_batch_dim=False
)
all_actions = standardize_input_data(
self.discrete_actions, ['all_actions'],
[(None, self.action_dim)] if self.action_dim is not None else None,
exception_prefix='discrete_actions')
n_updates = 0
history = {"theta": [], 'rho': []}
for k in theta_metrics.keys():
history.update({k: []})
ins = s + a + s_next + [r, absorbing]
self._make_train_function()
f = self.train_function
nb_train_sample = ins[0].shape[0]
index_array = np.arange(nb_train_sample)
# append evolution of theta for independent case
for _ in range(len(self.theta_list) - 1):
if self.incremental:
tmp = theta[-1] + self.bellman_model.predict(theta[-1])
else:
tmp = self.bellman_model.predict(theta[-1])
theta += [tmp]
term_condition = self.term_condition
stop = False
old_theta = theta
for epoch in range(nb_epoch):
if stop:
break
if shuffle == 'batch':
index_array = batch_shuffle(index_array, batch_size)
elif shuffle:
np.random.shuffle(index_array)
batches = make_batches(nb_train_sample, batch_size)
for batch_index, (batch_start, batch_end) in enumerate(batches):
history["theta"].append(theta[0])
if hasattr(self.bellman_model, '_model'):
history["rho"].append(
self.bellman_model._model.get_weights())
else:
history["rho"].append(self.bellman_model.get_weights())
for k, v in iteritems(theta_metrics):
history[k].append(v(theta))
batch_ids = index_array[batch_start:batch_end]
try:
if type(ins[-1]) is float:
# do not slice the training phase flag
ins_batch = slice_X(ins[:-1], batch_ids) + [ins[-1]]
else:
ins_batch = slice_X(ins, batch_ids)
except TypeError:
raise Exception('TypeError while preparing batch. '
'If using HDF5 input data, '
'pass shuffle="batch".')
inp = ins_batch + theta + all_actions
outs = f(*inp)
n_updates += 1
if self.update_theta_every > 0 and n_updates % self.update_theta_every == 0:
tmp = self.apply_bo(theta[0],
n_times=self.steps_per_theta_update)
theta = [tmp]
for _ in range(len(self.theta_list) - 1):
if self.incremental:
tmp = tmp + self.bellman_model.predict(tmp)
else:
tmp = self.bellman_model.predict(tmp)
theta += [tmp]
if term_condition is not None:
stop = term_condition(old_theta, theta)
if stop:
break
old_theta = theta
# finally apply the bellman operator K-times to get the final point
self.learned_theta_value = self.apply_bo(theta[0], n_times=100)
if self.verbose > 1:
print('learned theta: {}'.format(self.learned_theta_value))
self.history = history
return history
def _fit_loop(self,
f,
ins,
nb_train_samples,
out_labels=None,
batch_size=32,
nb_epoch=100,
verbose=1,
callbacks=None,
shuffle=True,
callback_metrics=None,
initial_epoch=0):
"""The core loop that fits the data."""
index_array = np.arange(nb_train_samples)
self.history = keras.callbacks.History()
callbacks = [keras.callbacks.BaseLogger()] + (callbacks or [])
callbacks += [self.history]
if verbose:
callbacks += [keras.callbacks.ProgbarLogger()]
callbacks = keras.callbacks.CallbackList(callbacks)
out_labels = out_labels or []
if hasattr(self, 'callback_model') and self.callback_model:
callback_model = self.callback_model
else:
callback_model = self
callbacks._set_model(callback_model)
callbacks._set_params({
'batch_size': batch_size,
'nb_epoch': nb_epoch,
'nb_sample': nb_train_samples,
'verbose': verbose,
'do_validation': False,
'metrics': callback_metrics or [],
})
callbacks.on_train_begin()
callback_model.stop_training = False
self.validation_data = None
for epoch in range(initial_epoch, nb_epoch):
callbacks.on_epoch_begin(epoch)
if shuffle == 'batch':
index_array = keras_training.batch_shuffle(
index_array, batch_size)
elif shuffle:
np.random.shuffle(index_array)
batches = keras_training.make_batches(nb_train_samples, batch_size)
epoch_logs = {}
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
try:
if isinstance(ins[-1], float):
ins_batch = get_batch(ins[:-1], batch_ids)
ins_batch += [ins[-1]]
else:
ins_batch = get_batch(ins, batch_ids)
except TypeError:
raise TypeError('TypeError while preparing batch. '
'If using HDF5 input data, '
'pass shuffle="batch".')
batch_logs = {}
batch_logs['batch'] = batch_index
batch_logs['size'] = len(batch_ids)
callbacks.on_batch_begin(batch_index, batch_logs)
outs = f(ins_batch)
if not isinstance(outs, list):
outs = [outs]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
callbacks.on_epoch_end(epoch, epoch_logs)
if callback_model.stop_training:
break
callbacks.on_train_end()
return self.history
# predict according to the input/output type
if model_name == 'Sequential':
if not isinstance(data, list):
data = [data]
elif model_name == 'Model':
if isinstance(data, dict):
data = [data[k] for k in model_k.input_names]
elif not isinstance(data, list):
data = [data]
else:
raise NotImplementedError(
'{}: This type of model is not supported'.format(model_name))
# Predict by batch to control GPU memory
len_data = len(data[0])
batches = make_batches(len_data, batch_size)
index_array = np.arange(len_data)
results_array = np.empty((len_data, ) + output_shape[1:])
for batch_start, batch_end in batches:
batch_ids = index_array[batch_start:batch_end]
data_b = [d[batch_ids] for d in data]
if learning_phase:
data_b.append(0.)
batch_prediction = pred_function(data_b)
if isinstance(batch_prediction, list): # pragma: no cover
batch_prediction = batch_prediction[0]
results_array[batch_ids] = batch_prediction
if async and json_serializer:
results_array = results_array.tolist()
return results_array
def _fit_loop(self, f, ins, discrete_actions, out_labels=[],
batch_size=32, nb_epoch=100, verbose=1, callbacks=[],
val_f=None, val_ins=None, shuffle=True, callback_metrics=[],
theta_metrics={}):
do_validation = False
if val_f and val_ins:
do_validation = True
if verbose:
print('Train on %d samples, validate on %d samples' %
(ins[0].shape[0], val_ins[0].shape[0]))
nb_train_sample = ins[0].shape[0]
print(nb_train_sample)
index_array = np.arange(nb_train_sample)
history = PBOHistory()
callbacks = [history] + callbacks
if verbose:
callbacks += [cbks.ProgbarLogger()]
callbacks = cbks.CallbackList(callbacks)
callback_model = self
callbacks._set_model(callback_model)
callbacks._set_params({
'batch_size': batch_size,
'nb_epoch': nb_epoch,
'nb_sample': nb_train_sample,
'verbose': verbose,
'do_validation': do_validation,
'metrics': callback_metrics + [el for el in theta_metrics.keys()],
'theta': self.q_model.trainable_weights[0].eval()
})
callbacks.on_train_begin()
callback_model.stop_training = False
for epoch in range(nb_epoch):
callbacks.on_epoch_begin(epoch)
if shuffle == 'batch':
index_array = batch_shuffle(index_array, batch_size)
elif shuffle:
np.random.shuffle(index_array)
batches = make_batches(nb_train_sample, batch_size)
epoch_logs = {}
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
try:
if type(ins[-1]) is float:
# do not slice the training phase flag
ins_batch = slice_X(ins[:-1], batch_ids) + [ins[-1]]
else:
ins_batch = slice_X(ins, batch_ids)
except TypeError:
raise Exception('TypeError while preparing batch. '
'If using HDF5 input data, '
'pass shuffle="batch".')
batch_logs = {}
batch_logs['batch'] = batch_index
batch_logs['size'] = len(batch_ids)
batch_logs['theta'] = self.q_model.trainable_weights[0].eval()
for k in theta_metrics.keys():
batch_logs[k] = theta_metrics[k](self.q_model.trainable_weights)
callbacks.on_batch_begin(batch_index, batch_logs)
inp = ins_batch + discrete_actions
outs = f(*inp)
if type(outs) != list:
outs = [outs]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
if batch_index == len(batches) - 1: # last batch
# validation
if do_validation:
# replace with self._evaluate
val_outs = self._test_loop(val_f, val_ins,
batch_size=batch_size,
verbose=0)
if type(val_outs) != list:
val_outs = [val_outs]
# same labels assumed
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_epoch_end(epoch, epoch_logs)
callbacks.on_train_end()
return history
# predict according to the input/output type
if model_name == 'Sequential':
if not isinstance(data, list):
data = [data]
elif model_name == 'Model':
if isinstance(data, dict):
data = [data[k] for k in model_k.input_names]
elif not isinstance(data, list):
data = [data]
else:
raise NotImplementedError(
'{}: This type of model is not supported'.format(model_name))
# Predict by batch to control GPU memory
len_data = len(data[0])
batches = make_batches(len_data, batch_size)
index_array = np.arange(len_data)
results_array = np.empty((len_data, ) + output_shape[1:])
for batch_start, batch_end in batches:
batch_ids = index_array[batch_start:batch_end]
data_b = [d[batch_ids] for d in data]
if learning_phase:
data_b.append(0.)
batch_prediction = pred_function(data_b)
if isinstance(batch_prediction, list): # pragma: no cover
batch_prediction = batch_prediction[0]
results_array[batch_ids] = batch_prediction
if async and json_serializer:
results_array = results_array.tolist()
return results_array
