def get_generator(self, batch_size):
board_tensors = self.board_tensors
policy_tensors = self.policy_tensors
value_tensors = self.value_tensors
size = board_tensors.shape[0] * 8
func_number = len(roting_fliping_functions)
batches = _make_batches(size, batch_size)
dimshuffle = True if K.image_data_format() == 'channels_last' else False
def generator():
indice = list(range(size))
while True:
np.random.shuffle(indice)
for batch in batches:
idxs = indice[batch[0]:batch[1]]
cache_board_tensors = []
cache_policy_tensors = []
cache_value_tensors = []
for idx in idxs:
sample_idx = idx // func_number
func = roting_fliping_functions[idx % func_number]
cache_board_tensors.append(func(board_tensors[sample_idx:sample_idx+1, ...]))
cache_policy_tensors.append(func(policy_tensors[sample_idx:sample_idx+1, ...]))
cache_value_tensors.append(value_tensors[sample_idx:sample_idx+1, ...])
cache_board_tensors = np.concatenate(cache_board_tensors, axis=0)
cache_policy_tensors = np.concatenate(cache_policy_tensors, axis=0).reshape((-1, SIZE**2))
cache_value_tensors = np.concatenate(cache_value_tensors, axis=0)
if dimshuffle:
cache_board_tensors = np.transpose(cache_board_tensors, (0, 2, 3, 1))
yield (cache_board_tensors, [cache_policy_tensors, cache_value_tensors])
return generator(), len(batches)
def data_generator(x, y, batch_size=50):
index_array = np.arange(len(x))
while 1:
batches = _make_batches(len(x_test), batch_size)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
x_batch = x[batch_ids]
y_batch = y[batch_ids]
yield (x_batch, y_batch)
def time_delay_generator(x, y, delays, batch_size, weights=None, shuffle=True):
'''A generator to make it easy to fit time-delay regression models,
i.e. a model where the value of y depends on past values of x
# Arguments
x: input data, as a Numpy array
y: targets, as a Numpy array or None for prediction generation
delays: number of time-steps to include in model
weights: Numpy array of weights for the samples
shuffle: Whether or not to shuffle the data (set True for training)
# Example
if X_train is (1000,200), Y_train is (1000,1)
train_gen = time_delay_generator(X_train, Y_train, delays=10, batch_size=100)
train_gen is a generator that gives:
x_batch as size (100,10,200) since each of the 100 samples includes the input
data at the current and nine previous time steps
y_batch as size (100,1)
w_batch as size (100,)
'''
if type(delays) is int:
delays = range(delays)
if type(x) is not list:
x = list([x])
index_array = np.arange(x[0].shape[0])
tlists = [[1, 0] + list(range(2, np.ndim(xx) + 1)) for xx in x]
batches = _make_batches(x[0].shape[0], batch_size)
while 1:
if shuffle:
np.random.shuffle(index_array)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
batch_ids_delay = [
np.minimum(np.maximum(0, batch_ids - d), x[0].shape[0] - 1)
for d in delays
]
x_batch = _standardize_input_data([
xx[batch_ids_delay, :].transpose(tt)
for xx, tt in zip(x, tlists)
], ['x_batch' + str(i) for i in range(1,
len(x) + 1)])
if y is None:
yield x_batch
else:
y_batch = _standardize_input_data(y[batch_ids, :], ['y_batch'])
if weights is not None:
w_batch = weights[batch_ids, :][:, 0]
else:
w_batch = np.ones(x_batch[0].shape[0])
w_batch[batch_ids < delays[-1]] = 0.
w_batch = _standardize_sample_weights(w_batch, ['w_batch'])
yield (x_batch, y_batch, w_batch)
def data_generator(x, y, batch_size=50):
index_array = np.arange(len(x))
while 1:
batches = _make_batches(len(x_test), batch_size)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
x_batch = x[batch_ids]
y_batch = y[batch_ids]
yield (x_batch, y_batch)
def train_neural_network(self, words, vocab_dict, batch_size):
num_word = len(vocab_dict)
logits, last_state, _, _, _ = self.def_model(num_word, batch_size)
targets = tf.reshape(self.output_targets, [-1])
loss = tf.contrib.legacy_seq2seq.sequence_loss_by_example([logits], [targets], \
[tf.ones_like(targets, dtype=tf.float32)], len(words))
cost = tf.reduce_mean(loss)
learning_rate = tf.Variable(0.0, trainable=False)
tvars = tf.trainable_variables()
grads, _ = tf.clip_by_global_norm(tf.gradients(cost, tvars), 5)
# optimizer = tf.train.GradientDescentOptimizer(learning_rate)
optimizer = tf.train.AdamOptimizer(learning_rate)
train_op = optimizer.apply_gradients(zip(grads, tvars))
Session_config = tf.ConfigProto(allow_soft_placement=True)
Session_config.gpu_options.allow_growth = True
batches = _make_batches(len(words), batch_size)
with tf.Session(config=Session_config) as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(tf.global_variables())
last_epoch = self.load_model(sess, saver, 'model/')
for epoch in range(last_epoch + 1, 100):
sess.run(tf.assign(learning_rate, 0.002 * (0.97**epoch)))
# sess.run(tf.assign(learning_rate, 0.01))
index_array = np.arange(len(words))
np.random.shuffle(index_array)
all_loss = 0.0
for batch_index, (batch_start,
batch_end) in enumerate(batches):
if batch_end - batch_start != batch_size:
# print('skip batch {} {}'.format(batch_start, batch_end))
continue
batch_ids = index_array[batch_start:batch_end]
xdata = words[batch_ids]
ydata = np.copy(xdata)
ydata[:, :-1] = xdata[:, 1:]
train_loss, _, _ = sess.run([cost, last_state, train_op],
feed_dict={
self.input_data: xdata,
self.output_targets: ydata
})
all_loss = all_loss + train_loss
if batch_index % 50 == 1:
print(epoch, batch_index, 0.002 * (0.97**epoch),
train_loss)
saver.save(sess, 'model/poetry.module', global_step=epoch)
print(epoch, ' Loss: ', all_loss * 1.0 / len(batches))
def next_batch(self):
self.batches = _make_batches(len(self.labels), self.batch_size)
index_array = np.arange(len(self.labels))
np.random.shuffle(index_array)
for batch_index, (batch_start, batch_end) in enumerate(self.batches):
if batch_end - batch_start != self.batch_size:
# print('skip batch {} {}'.format(batch_start, batch_end))
continue
batch_ids = index_array[batch_start:batch_end]
xdata = self.sentences[batch_ids]
y = self.labels[batch_ids]
yield xdata, y
def time_delay_generator_jitter(x, y, delays, batch_size, weights=None, shuffle=True, conv3d=False, jitter=True, jitter_axes=[3,4], max_jitter=1):
'''A generator to make it easy to fit time-delay regression models,
i.e. a model where the value of y depends on past values of x
# Arguments
x: input data, as a Numpy array
y: targets, as a Numpy array or None for prediction generation
delays: number of time-steps to include in model
weights: Numpy array of weights for the samples
shuffle: Whether or not to shuffle the data (set True for training)
# Example
if X_train is (1000,200), Y_train is (1000,1)
train_gen = time_delay_generator(X_train, Y_train, delays=10, batch_size=100)
train_gen is a generator that gives:
x_batch as size (100,10,200) since each of the 100 samples includes the input
data at the current and nine previous time steps
y_batch as size (100,1)
w_batch as size (100,)
'''
index_array = np.arange(x.shape[0])
if conv3d:
tlist = [1, 2, 0] + range(3, np.ndim(x) + 1)
else:
tlist = [1, 0] + range(2, np.ndim(x) + 1)
batches = _make_batches(x.shape[0], batch_size)
while 1:
if shuffle:
np.random.shuffle(index_array)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
batch_ids = [np.maximum(0, batch_ids - d) for d in range(delays)]
x_batch = x[batch_ids, :].transpose(tlist)
if jitter:
for j in jitter_axes:
x_batch = np.roll(x_batch, np.random.randint(-max_jitter,max_jitter+1), axis=j)
x_batch = _standardize_input_data(x_batch, ['x_batch'])
if y is None:
yield x_batch
else:
y_batch = _standardize_input_data(y[batch_ids[0], :], ['y_batch'])
if weights is not None:
w_batch = weights[batch_ids[0], :][:, 0]
else:
w_batch = np.ones(x_batch[0].shape[0])
w_batch[batch_ids[0] < delays] = 0.
w_batch = _standardize_sample_weights(w_batch, ['w_batch'])
yield (x_batch, y_batch, w_batch)
def next_batch(self):
assert len(self.data) == len(self.keywords)
self.batches = _make_batches(len(self.data), self.batch_size)
index_array = np.arange(len(self.data))
np.random.shuffle(index_array)
for batch_index, (batch_start, batch_end) in enumerate(self.batches):
if batch_end - batch_start != self.batch_size:
# print('skip batch {} {}'.format(batch_start, batch_end))
continue
batch_ids = index_array[batch_start:batch_end]
xdata = self.data[batch_ids]
xkeywords = self.keywords[batch_ids]
yield xdata, xkeywords
def train_gen(pairs_train, dist_train, batch_size):
'''
Generator used for training the siamese net with keras
pairs_train: training pairs
dist_train: training labels
returns: generator instance
'''
batches = _make_batches(len(pairs_train), batch_size)
while 1:
random_idx = np.random.permutation(len(pairs_train))
for batch_start, batch_end in batches:
p_ = random_idx[batch_start:batch_end]
x1, x2 = pairs_train[p_, 0], pairs_train[p_, 1]
y = dist_train[p_]
yield ([x1, x2], y)
def batch_generator(X, y, batch_size, samples_per_epoch):
while 1:
index_array = np.arange(X.shape[0])
np.random.shuffle(index_array)
batches = _make_batches(samples_per_epoch, batch_size)
# print("\n",index_array[0:2])
# print("=====================")
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
# print("\n", batch_start, batch_end, batch_index, len(batch_ids))
X_batch = _slice_X(X, batch_ids)
y_batch = _slice_X(y, batch_ids)
if (sps.issparse(X)):
X_batch = np.array(X_batch.todense())
else:
X_batch = np.array(X_batch)
y_batch = np.array(y_batch)
yield (X_batch, y_batch)
def time_delay_generator_AE(x, delays, batch_size, shuffle=True, conv3d=False):
'''A generator to make it easy to fit time-delay regression models,
i.e. a model where the value of y depends on past values of x
# Arguments
x: input data, as a Numpy array
y: targets, as a Numpy array or None for prediction generation
delays: number of time-steps to include in model
weights: Numpy array of weights for the samples
shuffle: Whether or not to shuffle the data (set True for training)
# Example
if X_train is (1000,200), Y_train is (1000,1)
train_gen = time_delay_generator(X_train, Y_train, delays=10, batch_size=100)
train_gen is a generator that gives:
x_batch as size (100,10,200) since each of the 100 samples includes the input
data at the current and nine previous time steps
y_batch as size (100,1)
w_batch as size (100,)
'''
index_array = np.arange(x.shape[0])
if conv3d:
tlist = [1, 2, 0] + range(3, np.ndim(x) + 1)
else:
tlist = [1, 0] + range(2, np.ndim(x) + 1)
batches = _make_batches(x.shape[0], batch_size)
while 1:
if shuffle:
np.random.shuffle(index_array)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
batch_ids = [np.maximum(0, batch_ids - d) for d in range(delays)]
x_batch = _standardize_input_data(x[batch_ids, :].transpose(tlist),
['x_batch'])
y_batch = _standardize_input_data(
np.copy(x_batch[0]).reshape((x_batch[0].shape[0], -1)),
['y_batch'])
yield (x_batch, y_batch)
def _fit_loop(self,
f,
ins,
out_labels=None,
batch_size=32,
epochs=100,
verbose=1,
callbacks=None,
val_f=None,
val_ins=None,
shuffle=True,
callback_metrics=None,
initial_epoch=0,
steps_per_epoch=None,
validation_steps=None):
"""Abstract fit function for f(ins).
Assume that f returns a list, labeled by out_labels.
# Arguments
f: Keras function returning a list of tensors
ins: List of tensors to be fed to `f`
out_labels: List of strings, display names of
the outputs of `f`
batch_size: Integer batch size or None if unknown.
epochs: Number of times to iterate over the data
verbose: Verbosity mode, 0, 1 or 2
callbacks: List of callbacks to be called during training
val_f: Keras function to call for validation
val_ins: List of tensors to be fed to `val_f`
shuffle: Whether to shuffle the data at the beginning of each epoch
callback_metrics: List of strings, the display names of the metrics
passed to the callbacks. They should be the
concatenation of list the display names of the outputs of
`f` and the list of display names of the outputs of `f_val`.
initial_epoch: Epoch at which to start training
(useful for resuming a previous training run)
steps_per_epoch: Total number of steps (batches of samples)
before declaring one epoch finished and starting the
next epoch. Ignored with the default value of `None`.
validation_steps: Number of steps to run validation for
(only if doing validation from data tensors).
Ignored with the default value of `None`.
# Returns
`History` object.
[A tweaked version.]
"""
do_validation = False
if val_f and val_ins:
do_validation = True
if verbose and ins and hasattr(ins[0], 'shape') and hasattr(
val_ins[0], 'shape'):
print('Train on %d samples, validate on %d samples' %
(ins[0].shape[0], val_ins[0].shape[0]))
if validation_steps:
do_validation = True
if steps_per_epoch is None:
raise ValueError('Can only use `validation_steps` '
'when doing step-wise '
'training, i.e. `steps_per_epoch` '
'must be set.')
num_train_samples = self._check_num_samples(ins, batch_size,
steps_per_epoch,
'steps_per_epoch')
if num_train_samples is not None:
index_array = np.arange(num_train_samples)
self.history = cbks.History()
callbacks = [cbks.BaseLogger()] + (callbacks or []) + [self.history]
if verbose:
if steps_per_epoch is not None:
count_mode = 'steps'
else:
count_mode = 'samples'
callbacks += [cbks.ProgbarLogger(count_mode)]
callbacks = cbks.CallbackList(callbacks)
out_labels = out_labels or []
# it's possible to callback a different model than self
# (used by Sequential models)
if hasattr(self, 'callback_model') and self.callback_model:
callback_model = self.callback_model
else:
callback_model = self
callbacks.set_model(callback_model)
callbacks.set_params({
'batch_size': batch_size,
'epochs': epochs,
'steps': steps_per_epoch,
'samples': num_train_samples,
'verbose': verbose,
'do_validation': do_validation,
'metrics': callback_metrics or [],
})
callbacks.on_train_begin()
callback_model.stop_training = False
# for cbk in callbacks:
# cbk.validation_data = val_ins
for epoch in range(initial_epoch, epochs):
callbacks.on_epoch_begin(epoch)
epoch_logs = {}
if steps_per_epoch is not None:
for step_index in range(steps_per_epoch):
batch_logs = {}
batch_logs['batch'] = step_index
batch_logs['size'] = 1
callbacks.on_batch_begin(step_index, batch_logs)
outs = f(ins)
if not isinstance(outs, list):
outs = [outs]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(step_index, batch_logs)
if callback_model.stop_training:
break
if do_validation:
val_outs = self._test_loop(val_f,
val_ins,
batch_size=batch_size,
steps=validation_steps,
verbose=0)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
else:
if shuffle == 'batch':
index_array = _batch_shuffle(index_array, batch_size)
elif shuffle:
np.random.shuffle(index_array)
batches = _make_batches(num_train_samples, batch_size)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
try:
if isinstance(ins[-1], float):
# do not slice the training phase flag
ins_batch = _slice_arrays(ins[:-1],
batch_ids) + [ins[-1]]
else:
ins_batch = _slice_arrays(ins, batch_ids)
except TypeError:
raise TypeError('TypeError while preparing batch. '
'If using HDF5 input data, '
'pass shuffle="batch".')
batch_logs = {}
batch_logs['batch'] = batch_index
batch_logs['size'] = len(batch_ids)
batch_logs['ids'] = batch_ids
callbacks.on_batch_begin(batch_index, batch_logs)
outs = f(ins_batch)
if not isinstance(outs, list):
outs = [outs]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
if callback_model.stop_training:
break
if batch_index == len(batches) - 1: # last batch.
if do_validation:
val_outs = self._test_loop(val_f,
val_ins,
batch_size=batch_size,
verbose=0)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# same labels assumed
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_epoch_end(epoch, epoch_logs)
if callback_model.stop_training:
break
callbacks.on_train_end()
return self.history
def _fit_loop(self,
f,
ins,
out_labels=None,
batch_size=32,
epochs=100,
verbose=1,
callbacks=None,
val_f=None,
val_ins=None,
shuffle=True,
callback_metrics=None,
initial_epoch=0,
steps_per_epoch=None):
"""Abstract fit function for `f(ins)`.
Assume that f returns a list, labeled by out_labels.
# Arguments
f: Keras function returning a list of tensors
ins: list of tensors to be fed to `f`
out_labels: list of strings, display names of
the outputs of `f`
batch_size: integer batch size
epochs: number of times to iterate over the data
verbose: verbosity mode, 0, 1 or 2
callbacks: list of callbacks to be called during training
val_f: Keras function to call for validation
val_ins: list of tensors to be fed to `val_f`
shuffle: whether to shuffle the data at the beginning of each epoch
callback_metrics: list of strings, the display names of the metrics
passed to the callbacks. They should be the
concatenation of list the display names of the outputs of
`f` and the list of display names of the outputs of `f_val`.
initial_epoch: epoch at which to start training
(useful for resuming a previous training run)
steps_per_epoch: Total number of steps (batches of samples)
before declaring one epoch finished and starting the
next epoch. The default `None` is equal to the number
of unique samples in your dataset divided by the batch
size, or 1 if that cannot be determined.
# Returns
`History` object.
"""
do_validation = False
if val_f and val_ins:
do_validation = True
if verbose and ins and hasattr(ins[0], 'shape'):
print('Train on %d samples, validate on %d samples' %
(ins[0].shape[0], val_ins[0].shape[0]))
if steps_per_epoch is not None:
num_train_samples = steps_per_epoch
else:
if ins and hasattr(ins[0], 'shape'):
num_train_samples = ins[0].shape[0]
else:
# May happen if we are running `fit` without Numpy input data,
# i.e. if all inputs to the models are data tensors
# instead of placeholders.
# In that case we will run `fit` over a single batch.
num_train_samples = batch_size
verbose = 2
index_array = np.arange(num_train_samples)
self.history = cbks.History()
callbacks = [cbks.BaseLogger()] + (callbacks or []) + [self.history]
if verbose:
# callbacks += [cbks.ProgbarLogger()]
callbacks += [ProgbarLogger_TFRecord()]
callbacks = cbks.CallbackList(callbacks)
out_labels = out_labels or []
# it's possible to callback a different model than self
# (used by Sequential models)
if hasattr(self, 'callback_model') and self.callback_model:
callback_model = self.callback_model
else:
callback_model = self
callbacks.set_model(callback_model)
callbacks.set_params({
'batch_size': batch_size,
'epochs': epochs,
'samples': num_train_samples,
'verbose': verbose,
'do_validation': do_validation,
'metrics': callback_metrics or [],
})
callbacks.on_train_begin()
callback_model.stop_training = False
for cbk in callbacks:
cbk.validation_data = val_ins
for epoch in range(initial_epoch, epochs):
callbacks.on_epoch_begin(epoch)
if shuffle == 'batch':
index_array = _batch_shuffle(index_array, batch_size)
elif shuffle:
np.random.shuffle(index_array)
batches = _make_batches(num_train_samples, batch_size)
epoch_logs = {}
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
try:
if isinstance(ins[-1], float):
# Do not slice the training phase flag.
ins_batch = \
_slice_arrays(ins[:-1], batch_ids) + [ins[-1]]
else:
ins_batch = _slice_arrays(ins, batch_ids)
except TypeError:
raise TypeError('TypeError while preparing batch. '
'If using HDF5 input data, '
'pass shuffle="batch".')
batch_logs = {}
batch_logs['batch'] = batch_index
batch_logs['size'] = len(batch_ids)
callbacks.on_batch_begin(batch_index, batch_logs)
outs = f(ins_batch)
if not isinstance(outs, list):
outs = [outs]
for l, o in zip(out_labels, outs):
batch_logs[l] = o
callbacks.on_batch_end(batch_index, batch_logs)
if callback_model.stop_training:
break
if batch_index == len(batches) - 1: # Last batch.
if do_validation:
val_outs = self._test_loop(val_f,
val_ins,
batch_size=batch_size,
verbose=0)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# Same labels assumed.
for l, o in zip(out_labels, val_outs):
epoch_logs['val_' + l] = o
callbacks.on_epoch_end(epoch, epoch_logs)
if callback_model.stop_training:
break
callbacks.on_train_end()
return self.history
def generator(self):
while True:
batches = _make_batches(size=self.total_images,
batch_size=self.batch_size)
for start, end in batches:
arr = []
labels = []
cur_batch = self.image_paths[start:end]
for image_path in cur_batch:
# print image_path
img = imread(
fname=os.path.join(self.data_path, image_path))
# if channels are not 3
ndim = len(img.shape)
if ndim == 2:
img = img[..., np.newaxis]
img = np.tile(A=img, reps=(1, 1, 3))
if ndim == 4:
img = img[..., :3]
# resizing image maintaining aspect ratio
img = resize_image(img=img, size=self.input_size)
if self.training:
# random cropping while training
img = random_crop_image(img=img, size=self.input_size)
img = augment(img=img,
horizontal_flip=True,
vertical_flip=True,
brightness=True,
contrast=True,
rotation=True,
translation=True,
blur=True,
noise=True)
else:
# center cropping
h, w, c = img.shape
center_h = h / 2
center_w = w / 2
center_new_img = self.input_size / 2
new_x1 = center_w - center_new_img
new_y1 = center_h - center_new_img
new_x2 = center_w + center_new_img
new_y2 = center_h + center_new_img
if self.input_size % 2 == 1:
new_x2 += 1
new_y2 += 1
img = img[new_y1:new_y2, new_x1:new_x2]
arr.append(img)
cls = image_path.split('/')[0]
id_for_cls = self.cls2id[cls]
labels.append(id_for_cls)
arr = np.array(arr)
arr.astype('float32')
# making mean of data 0 with standard deviation 1
arr /= 255.
arr -= 0.5
arr *= 2.
# one hot encoding
labels = to_categorical(y=labels,
num_classes=self.total_classes)
yield (arr, labels)
def predict(predict_var,
x_unlabeled,
inputs,
y_true,
batch_sizes,
x_labeled=None,
y_labeled=None):
'''
Evaluates predict_var, batchwise, over all points in x_unlabeled
and x_labeled.
predict_var: list of tensors to evaluate and return
x_unlabeled: unlabeled input data
inputs: dictionary containing input_types and
input_placeholders as key, value pairs, respectively
y_true: true labels tensorflow placeholder
batch_sizes: dictionary containing input_types and batch_sizes as
key, value pairs, respectively
x_labeled: labeled input data
y_labeled: labeled input labels
returns: a list of length n containing the result of all tensors
in return_var, where n = len(x_unlabeled) + len(x_labeled)
'''
x_unlabeled, x_labeled, y_labeled = check_inputs(x_unlabeled, x_labeled,
y_labeled, y_true)
# combined data
x = np.concatenate((x_unlabeled, x_labeled), 0)
# get shape of y_true
y_shape = y_true.get_shape()[1:K.ndim(y_true)].as_list()
# calculate batches for predict loop
unlabeled_batch_size = batch_sizes.get('Unlabeled', 0)
labeled_batch_size = batch_sizes.get('Labeled', 0)
if 'Labeled' in batch_sizes and 'Unlabeled' in batch_sizes:
assert unlabeled_batch_size == labeled_batch_size
batch_size = min(len(x), max(unlabeled_batch_size, labeled_batch_size))
batches = _make_batches(len(x), batch_size)
y_preds = []
# predict over all points
for i, (batch_start, batch_end) in enumerate(batches):
feed_dict = {K.learning_phase(): 0}
# feed corresponding input for each input_type
for input_type, input_placeholder in inputs.items():
if input_type == 'Unlabeled':
feed_dict[input_placeholder] = x[batch_start:batch_end]
elif input_type == 'Orthonorm':
batch_ids = np.random.choice(len(x),
size=min(len(x),
batch_sizes[input_type]),
replace=False)
feed_dict[input_placeholder] = x[batch_ids]
elif input_type == 'Labeled':
if len(x_labeled):
batch_ids = np.random.choice(len(x_labeled),
size=min(
batch_sizes[input_type],
len(x_labeled)),
replace=False)
feed_dict[input_placeholder] = x_labeled[batch_ids]
feed_dict[y_true] = y_labeled[batch_ids]
else:
# we have no labeled points, so feed an empty array
feed_dict[input_placeholder] = x[0:0]
feed_dict[y_true] = np.empty([0] + y_shape)
else:
raise Exception(
"Unrecognized feed name ['{}']".format(input_type))
# evaluate the batch
y_pred_batch = np.asarray(K.get_session().run(predict_var,
feed_dict=feed_dict))
y_preds.append(y_pred_batch)
if len(y_preds[0].shape):
return np.concatenate(y_preds)
else:
return np.sum(y_preds)
def keras_generator(self, delays=7, batch_size=400, cell=0, scale=5, flatten=True, center=None, crop_size=None, shuffle=True, color_chan=False, log_transform_events=True, correct_eye_pos=False, gaussian_filter=0):
from keras.engine.training import _standardize_input_data, _make_batches, _standardize_sample_weights
if type(cell) is int:
cell = [cell]
if type(delays) is int:
delays = range(delays)
(stim, events, frame_numbers, weights, shifts) = self.vectorize_data(delays)
evidx = np.where(events)[0]
print(str(len(frame_numbers)) + ' Samples')
print(str(len(evidx)) + ' Events')
if correct_eye_pos:
sh = stim.shape
shift_stim_shape = (len(shifts),
sh[1] + 2*np.maximum(self.min_max_shift[1][0], -self.min_max_shift[0][0]) + 3,
sh[2] + 2*np.maximum(self.min_max_shift[1][1], -self.min_max_shift[0][1]) + 3)
out_stim = np.zeros(shift_stim_shape, dtype='float32')
shifts = shifts + [shift_stim_shape[1]/2, shift_stim_shape[2]/2]
good_shift_locations = ~np.isnan(shifts[:, 0])
for dd in delays:
weights[np.minimum(np.where(np.isnan(shifts[:,0]))[0] + dd, len(weights)-1)] = 0
for i in range(len(shifts)):
if good_shift_locations[i]:
# print(-sh[1]/2 + np.int32(shifts[i, 0]))
# print(np.int32(shifts[i, 0]) + sh[1]/2)
out_stim[i, -sh[1]/2 + np.int32(shifts[i, 0]):np.int32(shifts[i, 0]) + sh[1]/2,
-sh[2]/2 + np.int32(shifts[i, 1]):np.int32(shifts[i, 1]) + sh[2]/2] = stim[frame_numbers[i]]
stim = out_stim
frame_numbers_i = np.arange(len(frame_numbers))
else:
frame_numbers_i = frame_numbers
if color_chan:
stim = stim[:, None, :, :]
if crop_size is not None and center is not None:
crop_range = np.arange(-crop_size/2, crop_size/2)
stim = stim[:, (center[0]-crop_size/2):(center[0]+crop_size/2), (center[1]-crop_size/2):(center[1]+crop_size/2)]
if flatten:
stim = stim.reshape(stim.shape[0], -1)
events = np.asarray(events)
events = events[cell].T * scale
if log_transform_events:
events = np.log(1 + events)
if gaussian_filter > 0:
events = gaussian_filter1d(events, gaussian_filter)
index_array = np.arange(events.shape[0])
tlist = [1, 0] + list(range(2, np.ndim(stim) + 1))
batches = _make_batches(events.shape[0], batch_size)
while 1:
if shuffle:
np.random.shuffle(index_array)
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
frame_numbers_b = frame_numbers[batch_ids]
batch_ids_stim = [frame_numbers_i[np.maximum(0, batch_ids - d)] for d in delays]
x_batch = _standardize_input_data(stim[batch_ids_stim, :].transpose(tlist), ['x_batch'])
y_batch = _standardize_input_data(events[batch_ids, :], ['y_batch'])
w_batch = weights[batch_ids]
w_batch[frame_numbers_b < delays[-1]] = 0.
w_batch = _standardize_sample_weights(w_batch, ['w_batch'])
yield (x_batch, y_batch, w_batch)
def _fit_loop(self,
f: callable,
ins: List[numpy.array],
out_labels: List[str] = None,
batch_size: int = 32,
epochs: int = 100,
verbose: int = 1,
callbacks: List[Callback] = None,
val_f: callable = None,
val_ins: List[numpy.array] = None,
shuffle: bool = True,
callback_metrics: List[str] = None,
initial_epoch: int = 0):
"""
Abstract fit function which preprocesses and batches
data before training a model. We override this keras backend
function to support multi-gpu training via splitting a large
batch size across multiple gpus. This function is broadly the
same as the Keras backend version aside from this - changed elements
have corresponding comments attached.
Note that this should not be called directly - it is used by calling
model.fit().
Assume that step_function returns a list, labeled by out_labels.
Parameters
----------
f: A callable ``Step`` or a Keras ``Function``, required.
A DeepQA Step or Keras Function returning a list of tensors.
ins: List[numpy.array], required.
The list of tensors to be fed to ``step_function``.
out_labels: List[str], optional (default = None).
The display names of the outputs of ``step_function``.
batch_size: int, optional (default = 32).
The integer batch size.
epochs: int, optional (default = 100).
Number of times to iterate over the data.
verbose: int, optional, (default = 1)
Verbosity mode, 0, 1 or 2.
callbacks: List[Callback], optional (default = None).
A list of Keras callbacks to be called during training.
val_f: A callable ``Step`` or a Keras ``Function``, optional (default = None).
The Keras function to call for validation.
val_ins: List[numpy.array], optional (default)
A list of tensors to be fed to ``val_f``.
shuffle: bool, optional (default = True).
whether to shuffle the data at the beginning of each epoch
callback_metrics: List[str], optional, (default = None).
A list of strings, the display names of the validation metrics.
passed to the callbacks. They should be the concatenation of list the display
names of the outputs of ``f`` and the list of display names of the outputs of ``f_val``.
initial_epoch: int, optional (default = 0).
The epoch at which to start training (useful for resuming a previous training run).
Returns
-------
A Keras ``History`` object.
"""
do_validation = False
if val_f and val_ins:
do_validation = True
if verbose:
print('Train on %d samples, validate on %d samples' %
(ins[0].shape[0], val_ins[0].shape[0]))
if ins and hasattr(ins[0], 'shape'):
num_train_samples = ins[0].shape[0]
else:
# May happen if we are running `fit` without Numpy input data,
# i.e. if all inputs to the models are data tensors
# instead of placeholders.
# In that case we will run `fit` over a single batch.
num_train_samples = batch_size
verbose = 2
index_array = numpy.arange(num_train_samples)
out_labels = out_labels or []
callbacks, callback_model = self._prepare_callbacks(
callbacks, val_ins, epochs, batch_size, num_train_samples,
callback_metrics, do_validation, verbose)
for epoch in range(initial_epoch, epochs):
callbacks.on_epoch_begin(epoch)
if shuffle == 'batch':
index_array = _batch_shuffle(index_array, batch_size)
elif shuffle:
numpy.random.shuffle(index_array)
batches = _make_batches(num_train_samples, batch_size)
epoch_logs = {}
for batch_index, (batch_start, batch_end) in enumerate(batches):
batch_ids = index_array[batch_start:batch_end]
try:
if isinstance(ins[-1], float):
# Do not slice the training phase flag.
ins_batch = _slice_arrays(ins[:-1],
batch_ids) + [ins[-1]]
else:
ins_batch = _slice_arrays(ins, batch_ids)
except TypeError:
raise TypeError('TypeError while preparing batch. '
'If using HDF5 input data, '
'pass shuffle="batch".')
# Here is the main difference between a single gpu model and one split
# across multiple gpus. In our multiple gpu model, all of the inputs
# are replicated num_gpus times, so we need to split our large batch
# into the corresponding sets of smaller batches for each model.
if self.num_gpus > 1:
# The Keras learning phase is a global variable used across model towers.
# If it is present, we remove it before splitting up the inputs
# and add it back on afterwards.
if isinstance(ins_batch[-1], float):
model_inputs = self._multi_gpu_batch(ins_batch[:-1])
model_inputs.append(ins_batch[-1])
else:
model_inputs = self._multi_gpu_batch(ins_batch)
ins_batch = model_inputs
batch_logs = {}
batch_logs['batch'] = batch_index
batch_logs['size'] = len(batch_ids)
callbacks.on_batch_begin(batch_index, batch_logs)
outs = f(ins_batch)
if not isinstance(outs, list):
outs = [outs]
for label, output in zip(out_labels, outs):
batch_logs[label] = output
callbacks.on_batch_end(batch_index, batch_logs)
if batch_index == len(batches) - 1: # Last batch.
if do_validation:
# If we are using multiple gpus, our batch size will be
# scaled up accordingly. However, validation will run
# on a single gpu, so we divide by the number of gpus
# to avoid OOM errors.
if self.num_gpus > 1:
val_batch_size = int(batch_size / self.num_gpus) # pylint: disable=no-member
else:
val_batch_size = batch_size
val_outs = self._test_loop(val_f,
val_ins,
batch_size=val_batch_size,
verbose=0)
if not isinstance(val_outs, list):
val_outs = [val_outs]
# Same labels assumed.
for label, output in zip(out_labels, val_outs):
epoch_logs['val_' + label] = output
callbacks.on_epoch_end(epoch, epoch_logs)
if callback_model.stop_training: # pylint: disable=no-member
break
callbacks.on_train_end()
return self.history
