def get_request_iterator(self): indices = list(self.indices) self.rng.shuffle(indices) if self.sorted_indices: return imap(sorted, partition_all(self.batch_size, indices)) else: return imap(list, partition_all(self.batch_size, indices))
def get_request_iterator(self): indices = list(self.indices) self.rng.shuffle(indices) fpv = self.frames_per_video if self.r_subsample: subsample = np.random.randint(1, self.f_subsample) else: subsample = self.f_subsample frames_array = np.empty([len(indices), fpv]) for j in xrange(len(indices)): i = indices[j] if i == 0: c_subsample = self.correct_subsample(0, self.video_indexes[i], fpv, subsample) t = self.get_start_frame(0, self.video_indexes[i], fpv, c_subsample) else: c_subsample = self.correct_subsample(self.video_indexes[i - 1], self.video_indexes[i], fpv, subsample) t = self.get_start_frame(self.video_indexes[i - 1], self.video_indexes[i], fpv, c_subsample) for k in range(fpv): frames_array[j][k] = t + c_subsample * k frames_array = frames_array.flatten() if self.sorted_indices: return imap(sorted, partition_all(self.batch_size * fpv, frames_array)) else: return imap(list, partition_all(self.batch_size * fpv, frames_array))
def get_request_iterator(self) : indices = list(self.indices) self.rng.shuffle(indices) fpv = self.frames_per_video if self.r_subsample: subsample = np.random.randint(1, self.f_subsample) else: subsample = self.f_subsample frames_array = np.empty([len(indices),fpv]) #each element of indices is the jth video we want for j in xrange(len(indices)): i = indices[j] if i==0 : c_subsample = self.correct_subsample(0, self.video_indexes[i], fpv, subsample) t = self.get_start_frame(0, self.video_indexes[i], fpv, c_subsample) else : c_subsample = self.correct_subsample(self.video_indexes[i-1], self.video_indexes[i], fpv, subsample) t = self.get_start_frame(self.video_indexes[i-1], self.video_indexes[i], fpv, c_subsample) for k in range(fpv): frames_array[j][k] = t + c_subsample * k frames_array = frames_array.flatten() if self.sorted_indices: return imap(sorted, partition_all(self.batch_size*fpv, frames_array)) else: return imap(list, partition_all(self.batch_size*fpv, frames_array))
def get_request_iterator(self): indices = list(self.indices) count = len(indices) if count < self.batch_size: count = self.batch_size indices = self.rng.choice(indices, count) return imap(list, partition_all(self.batch_size, indices))
def batch_generator(self, full_sequences, rnd): indices = list(self.fragment_indices(full_sequences)) if self.randomize_batch_order: rnd.shuffle(indices) batches_parted = [ batch for batch in partition_all(self.batch_size, indices) ] start_index = len(batches_parted) // self.num_gpus * self.train_rank batches_gpu = batches_parted[start_index:] batches = cycle(batches_gpu) for batch in batches: if len(batch) < self.batch_size: continue yield np.array( [ self.one_hot( full_sequences[e[0]][e[1]:e[1] + self.fragment_length]) for e in batch ], dtype='uint8'), np.array([ self.one_hot( full_sequences[e[0]][e[1] + 1:e[1] + self.fragment_length + 1]) for e in batch ], dtype='uint8')
def get_request_iterator(self): iterator_list = [] start = 0 for size, examples in self.size_dict.items(): iterator_list.append( partition_all(self.batch_size, xrange(start, start + examples))) start += examples return chain(*iterator_list)
def test_minimum_dimensions_batch_stream(self): stream = MinimumImageDimensions(self.batch_stream, (4, 5), which_sources=('source1', )) it = stream.get_epoch_iterator() for batch, shapes in zip(it, partition_all(self.batch_size, self.shapes)): assert (example.shape[0] >= 4 and example.shape[1] >= 5 for example in batch[0]) assert (example.shape[1] == shp[0] and example.shape[0] == shp[1] for example, shp in zip(batch[1], shapes))
def test_minimum_dimensions_batch_stream(self): stream = MinimumImageDimensions(self.batch_stream, (4, 5), which_sources=('source1',)) it = stream.get_epoch_iterator() for batch, shapes in zip(it, partition_all(self.batch_size, self.shapes)): assert (example.shape[0] >= 4 and example.shape[1] >= 5 for example in batch[0]) assert (example.shape[1] == shp[0] and example.shape[0] == shp[1] for example, shp in zip(batch[1], shapes))
def batch_generator(full_sequences, fragment_length, batch_size, fragment_stride, nb_output_bins, randomize_batch_order, _rnd): indices = list(fragment_indices(full_sequences, fragment_length, batch_size, fragment_stride, nb_output_bins)) if randomize_batch_order: _rnd.shuffle(indices) batches = cycle(partition_all(batch_size, indices)) for batch in batches: if len(batch) < batch_size: continue yield np.array( [one_hot(full_sequences[e[0]][e[1]:e[1] + fragment_length]) for e in batch], dtype='uint8'), np.array( [one_hot(full_sequences[e[0]][e[1] + 1:e[1] + fragment_length + 1]) for e in batch], dtype='uint8')
def get_request_iterator(self): chunks = len(self.indices) / self.batch_size assert len(self.indices) % chunks == 0 data = np.array(self.indices) data = data.reshape(chunks, self.batch_size) np.random.shuffle(data) data = data.flatten() self.indices = np.ndarray.tolist(data) return imap(list, partition_all(self.batch_size, self.indices))
def get_request_iterator(self): chunks = len(self.indices) / self.batch_size assert len(self.indices)%chunks == 0 data = np.array(self.indices) data = data.reshape(chunks, self.batch_size) np.random.shuffle(data) data = data.flatten() self.indices = np.ndarray.tolist(data) return imap(list, partition_all(self.batch_size, self.indices))
def batch_generator(full_sequences, fragment_length, batch_size, fragment_stride, nb_output_bins, learn_all_outputs): indices = list( fragment_indices(full_sequences, fragment_length, batch_size, fragment_stride, nb_output_bins)) # TODO: shuffle batches = cycle(partition_all(batch_size, indices)) for batch in batches: if len(batch) < batch_size: continue yield np.array( [one_hot(full_sequences[e[0][0]][e[0][1]:e[0][2]]) for e in batch], dtype='uint8'), np.array([ one_hot(full_sequences[e[1][0]][e[1][1]:e[1][2]]) for e in batch ], dtype='uint8')
def batch_generator(full_sequences, fragment_length, batch_size, fragment_stride, nb_output_bins, randomize_batch_order, _rnd): indices = list( fragment_indices(full_sequences, fragment_length, batch_size, fragment_stride, nb_output_bins)) global g_multi_gpu if g_multi_gpu: import horovod.keras as hvd gpu_count = hvd.size() current_gpu = hvd.rank() else: gpu_count = 1 current_gpu = 0 if randomize_batch_order: _rnd.shuffle(indices) batches_parted = [batch for batch in partition_all(batch_size, indices)] start_index = len(batches_parted) // gpu_count * current_gpu batches_gpu = batches_parted[start_index:] batches = cycle(batches_gpu) for batch in batches: if len(batch) < batch_size: continue yield np.array([ one_hot(full_sequences[e[0]][e[1]:e[1] + fragment_length]) for e in batch ], dtype='uint8'), np.array([ one_hot(full_sequences[e[0]][e[1] + 1:e[1] + fragment_length + 1]) for e in batch ], dtype='uint8')
def get_request_iterator(self): indices = list(self.indices) # shuffle indices indicesShuffled = [] permutation = numpy.random.permutation(len(indices)) return imap(list, partition_all(self.batch_size, permutation))
def get_request_iterator(self): ''' Careful this is indeed infinite ''' return imap(list, partition_all(self.batch_size, cycle(self.indices)))
def get_request_iterator(self): indices = list(self.indices) self.rng.shuffle(indices) return imap(list, partition_all(self.batch_size, indices))
def get_request_iterator(self): return imap(list, partition_all(self.batch_size, self.indices))
pass if len(items) == 0: raise StopIteration return items[0] class SequentialScheme_slice(BatchScheme): """Sequential batches iterator. Iterate over all the examples in a dataset of fixed size sequentially in batches of a given size. Notes ----- The batch size isn't enforced, so the last batch could be smaller. """ def get_request_iterator(self): return partition_slice(self.batch_size, self.indices) if __name__ == '__main__': import numpy as np indices = np.arange(100) batch_size = 10 it_1 = partition_slice(batch_size, indices) it_2 = partition_all(batch_size, indices) print indices[it_1.next()] print indices[list(it_2.next())] print indices[it_1.next()] print indices[list(it_2.next())]