def get_request_iterator(self):
indices = list(self.indices)
self.rng.shuffle(indices)
if self.sorted_indices:
return imap(sorted, partition_all(self.batch_size, indices))
else:
return imap(list, partition_all(self.batch_size, indices))
def get_request_iterator(self):
indices = list(self.indices)
self.rng.shuffle(indices)
fpv = self.frames_per_video
if self.r_subsample:
subsample = np.random.randint(1, self.f_subsample)
else:
subsample = self.f_subsample
frames_array = np.empty([len(indices), fpv])
for j in xrange(len(indices)):
i = indices[j]
if i == 0:
c_subsample = self.correct_subsample(0, self.video_indexes[i],
fpv, subsample)
t = self.get_start_frame(0, self.video_indexes[i], fpv,
c_subsample)
else:
c_subsample = self.correct_subsample(self.video_indexes[i - 1],
self.video_indexes[i],
fpv, subsample)
t = self.get_start_frame(self.video_indexes[i - 1],
self.video_indexes[i], fpv,
c_subsample)
for k in range(fpv):
frames_array[j][k] = t + c_subsample * k
frames_array = frames_array.flatten()
if self.sorted_indices:
return imap(sorted,
partition_all(self.batch_size * fpv, frames_array))
else:
return imap(list, partition_all(self.batch_size * fpv,
frames_array))
def get_request_iterator(self) :
indices = list(self.indices)
self.rng.shuffle(indices)
fpv = self.frames_per_video
if self.r_subsample:
subsample = np.random.randint(1, self.f_subsample)
else:
subsample = self.f_subsample
frames_array = np.empty([len(indices),fpv])
#each element of indices is the jth video we want
for j in xrange(len(indices)):
i = indices[j]
if i==0 :
c_subsample = self.correct_subsample(0, self.video_indexes[i],
fpv, subsample)
t = self.get_start_frame(0, self.video_indexes[i],
fpv, c_subsample)
else :
c_subsample = self.correct_subsample(self.video_indexes[i-1],
self.video_indexes[i],
fpv, subsample)
t = self.get_start_frame(self.video_indexes[i-1],
self.video_indexes[i],
fpv, c_subsample)
for k in range(fpv):
frames_array[j][k] = t + c_subsample * k
frames_array = frames_array.flatten()
if self.sorted_indices:
return imap(sorted, partition_all(self.batch_size*fpv, frames_array))
else:
return imap(list, partition_all(self.batch_size*fpv, frames_array))
def get_request_iterator(self):
indices = list(self.indices)
count = len(indices)
if count < self.batch_size:
count = self.batch_size
indices = self.rng.choice(indices, count)
return imap(list, partition_all(self.batch_size, indices))
def batch_generator(self, full_sequences, rnd):
indices = list(self.fragment_indices(full_sequences))
if self.randomize_batch_order:
rnd.shuffle(indices)
batches_parted = [
batch for batch in partition_all(self.batch_size, indices)
]
start_index = len(batches_parted) // self.num_gpus * self.train_rank
batches_gpu = batches_parted[start_index:]
batches = cycle(batches_gpu)
for batch in batches:
if len(batch) < self.batch_size:
continue
yield np.array(
[
self.one_hot(
full_sequences[e[0]][e[1]:e[1] + self.fragment_length])
for e in batch
],
dtype='uint8'), np.array([
self.one_hot(
full_sequences[e[0]][e[1] + 1:e[1] +
self.fragment_length + 1])
for e in batch
],
dtype='uint8')
def get_request_iterator(self):
indices = list(self.indices)
count = len(indices)
if count < self.batch_size:
count = self.batch_size
indices = self.rng.choice(indices, count)
return imap(list, partition_all(self.batch_size, indices))
def get_request_iterator(self):
iterator_list = []
start = 0
for size, examples in self.size_dict.items():
iterator_list.append(
partition_all(self.batch_size, xrange(start,
start + examples)))
start += examples
return chain(*iterator_list)
def test_minimum_dimensions_batch_stream(self):
stream = MinimumImageDimensions(self.batch_stream, (4, 5),
which_sources=('source1', ))
it = stream.get_epoch_iterator()
for batch, shapes in zip(it, partition_all(self.batch_size,
self.shapes)):
assert (example.shape[0] >= 4 and example.shape[1] >= 5
for example in batch[0])
assert (example.shape[1] == shp[0] and example.shape[0] == shp[1]
for example, shp in zip(batch[1], shapes))
def test_minimum_dimensions_batch_stream(self):
stream = MinimumImageDimensions(self.batch_stream, (4, 5),
which_sources=('source1',))
it = stream.get_epoch_iterator()
for batch, shapes in zip(it, partition_all(self.batch_size,
self.shapes)):
assert (example.shape[0] >= 4 and example.shape[1] >= 5
for example in batch[0])
assert (example.shape[1] == shp[0] and
example.shape[0] == shp[1]
for example, shp in zip(batch[1], shapes))
def batch_generator(full_sequences, fragment_length, batch_size, fragment_stride, nb_output_bins, randomize_batch_order, _rnd):
indices = list(fragment_indices(full_sequences, fragment_length, batch_size, fragment_stride, nb_output_bins))
if randomize_batch_order:
_rnd.shuffle(indices)
batches = cycle(partition_all(batch_size, indices))
for batch in batches:
if len(batch) < batch_size:
continue
yield np.array(
[one_hot(full_sequences[e[0]][e[1]:e[1] + fragment_length]) for e in batch], dtype='uint8'), np.array(
[one_hot(full_sequences[e[0]][e[1] + 1:e[1] + fragment_length + 1]) for e in batch], dtype='uint8')
def get_request_iterator(self):
chunks = len(self.indices) / self.batch_size
assert len(self.indices) % chunks == 0
data = np.array(self.indices)
data = data.reshape(chunks, self.batch_size)
np.random.shuffle(data)
data = data.flatten()
self.indices = np.ndarray.tolist(data)
return imap(list, partition_all(self.batch_size, self.indices))
def get_request_iterator(self):
chunks = len(self.indices) / self.batch_size
assert len(self.indices)%chunks == 0
data = np.array(self.indices)
data = data.reshape(chunks, self.batch_size)
np.random.shuffle(data)
data = data.flatten()
self.indices = np.ndarray.tolist(data)
return imap(list, partition_all(self.batch_size, self.indices))
def batch_generator(full_sequences, fragment_length, batch_size,
fragment_stride, nb_output_bins, learn_all_outputs):
indices = list(
fragment_indices(full_sequences, fragment_length, batch_size,
fragment_stride, nb_output_bins))
# TODO: shuffle
batches = cycle(partition_all(batch_size, indices))
for batch in batches:
if len(batch) < batch_size:
continue
yield np.array(
[one_hot(full_sequences[e[0][0]][e[0][1]:e[0][2]]) for e in batch],
dtype='uint8'), np.array([
one_hot(full_sequences[e[1][0]][e[1][1]:e[1][2]])
for e in batch
],
dtype='uint8')
def batch_generator(full_sequences, fragment_length, batch_size,
fragment_stride, nb_output_bins, randomize_batch_order,
_rnd):
indices = list(
fragment_indices(full_sequences, fragment_length, batch_size,
fragment_stride, nb_output_bins))
global g_multi_gpu
if g_multi_gpu:
import horovod.keras as hvd
gpu_count = hvd.size()
current_gpu = hvd.rank()
else:
gpu_count = 1
current_gpu = 0
if randomize_batch_order:
_rnd.shuffle(indices)
batches_parted = [batch for batch in partition_all(batch_size, indices)]
start_index = len(batches_parted) // gpu_count * current_gpu
batches_gpu = batches_parted[start_index:]
batches = cycle(batches_gpu)
for batch in batches:
if len(batch) < batch_size:
continue
yield np.array([
one_hot(full_sequences[e[0]][e[1]:e[1] + fragment_length])
for e in batch
],
dtype='uint8'), np.array([
one_hot(full_sequences[e[0]][e[1] + 1:e[1] +
fragment_length + 1])
for e in batch
],
dtype='uint8')
def get_request_iterator(self):
indices = list(self.indices)
# shuffle indices
indicesShuffled = []
permutation = numpy.random.permutation(len(indices))
return imap(list, partition_all(self.batch_size, permutation))
def get_request_iterator(self):
'''
Careful this is indeed infinite
'''
return imap(list, partition_all(self.batch_size, cycle(self.indices)))
def get_request_iterator(self):
indices = list(self.indices)
self.rng.shuffle(indices)
return imap(list, partition_all(self.batch_size, indices))
def get_request_iterator(self):
return imap(list, partition_all(self.batch_size, self.indices))
pass
if len(items) == 0:
raise StopIteration
return items[0]
class SequentialScheme_slice(BatchScheme):
"""Sequential batches iterator.
Iterate over all the examples in a dataset of fixed size sequentially
in batches of a given size.
Notes
-----
The batch size isn't enforced, so the last batch could be smaller.
"""
def get_request_iterator(self):
return partition_slice(self.batch_size, self.indices)
if __name__ == '__main__':
import numpy as np
indices = np.arange(100)
batch_size = 10
it_1 = partition_slice(batch_size, indices)
it_2 = partition_all(batch_size, indices)
print indices[it_1.next()]
print indices[list(it_2.next())]
print indices[it_1.next()]
print indices[list(it_2.next())]
