def get_request_iterator(self) : indices = list(self.indices) self.rng.shuffle(indices) fpv = self.frames_per_video if self.r_subsample: subsample = np.random.randint(1, self.f_subsample) else: subsample = self.f_subsample frames_array = np.empty([len(indices),fpv]) #each element of indices is the jth video we want for j in xrange(len(indices)): i = indices[j] if i==0 : c_subsample = self.correct_subsample(0, self.video_indexes[i], fpv, subsample) t = self.get_start_frame(0, self.video_indexes[i], fpv, c_subsample) else : c_subsample = self.correct_subsample(self.video_indexes[i-1], self.video_indexes[i], fpv, subsample) t = self.get_start_frame(self.video_indexes[i-1], self.video_indexes[i], fpv, c_subsample) for k in range(fpv): frames_array[j][k] = t + c_subsample * k frames_array = frames_array.flatten() if self.sorted_indices: return imap(sorted, partition_all(self.batch_size*fpv, frames_array)) else: return imap(list, partition_all(self.batch_size*fpv, frames_array))
def get_request_iterator(self): indices = list(self.indices) self.rng.shuffle(indices) fpv = self.frames_per_video if self.r_subsample: subsample = np.random.randint(1, self.f_subsample) else: subsample = self.f_subsample frames_array = np.empty([len(indices), fpv]) for j in xrange(len(indices)): i = indices[j] if i == 0: c_subsample = self.correct_subsample(0, self.video_indexes[i], fpv, subsample) t = self.get_start_frame(0, self.video_indexes[i], fpv, c_subsample) else: c_subsample = self.correct_subsample(self.video_indexes[i - 1], self.video_indexes[i], fpv, subsample) t = self.get_start_frame(self.video_indexes[i - 1], self.video_indexes[i], fpv, c_subsample) for k in range(fpv): frames_array[j][k] = t + c_subsample * k frames_array = frames_array.flatten() if self.sorted_indices: return imap(sorted, partition_all(self.batch_size * fpv, frames_array)) else: return imap(list, partition_all(self.batch_size * fpv, frames_array))
def get_request_iterator(self): indices = list(self.indices) self.rng.shuffle(indices) if self.sorted_indices: return imap(sorted, partition_all(self.batch_size, indices)) else: return imap(list, partition_all(self.batch_size, indices))
def get_request_iterator(self): indices = list(range(self.num_examples)) self.rng.shuffle(indices) return imap( list, imap(islice, repeat(_iter(indices), self.num_batches), repeat(self.batch_size, self.num_batches)))
def get_request_iterator(self): indices = list(self.indices)[::self.batch_size] self.rng.shuffle(indices) if self.use_slice: return imap(slice, _iter(indices), imap(lambda x: x + self.batch_size, _iter(indices))) else: return imap(range, _iter(indices), imap(lambda x: x + self.batch_size if x != self.indices[-1] - (self.indices[-1] % self.batch_size) else self.indices[-1], _iter(indices)))
def get_request_iterator(self): indices = list(self.indices)[::self.batch_size] self.rng.shuffle(indices) if self.use_slice: return imap(slice, _iter(indices), imap(lambda x: x + self.batch_size, _iter(indices))) else: return imap( range, _iter(indices), imap( lambda x: x + self.batch_size if x != self.indices[-1] - (self.indices[-1] % self.batch_size) else self.indices[-1], _iter(indices)))
def get_request_iterator(self): indices = list(self.indices) count = len(indices) if count < self.batch_size: count = self.batch_size indices = self.rng.choice(indices, count) return imap(list, partition_all(self.batch_size, indices))
def get_data(self, state=None, request=None): if isinstance(request, list): batch_index = request[0] / self.batch_size data = imap(self.filter_sources, [self.filenames[idx] for idx in request]) else: raise ValueError("request should be a list instance") return (data, batch_index)
def get_epoch_iterator(self, **kwargs): batches = chain.from_iterable(izip(*[data_stream.get_epoch_iterator() for data_stream in self.data_streams])) part = partition(len(self.sources), chain.from_iterable(batches)) as_dict = kwargs.get("as_dict", False) if as_dict: return imap(dict, starmap(zip, izip(repeat(self.sources), part))) else: return part
def get_epoch_iterator(self, **kwargs): batches = chain.from_iterable( izip(*[data_stream.get_epoch_iterator() for data_stream in self.data_streams])) part = partition(len(self.sources), chain.from_iterable(batches)) as_dict = kwargs.get('as_dict', False) if as_dict: return imap(dict, starmap(zip, izip(repeat(self.sources), part))) else: return part
def get_request_iterator(self): chunks = len(self.indices) / self.batch_size assert len(self.indices)%chunks == 0 data = np.array(self.indices) data = data.reshape(chunks, self.batch_size) np.random.shuffle(data) data = data.flatten() self.indices = np.ndarray.tolist(data) return imap(list, partition_all(self.batch_size, self.indices))
def get_request_iterator(self): chunks = len(self.indices) / self.batch_size assert len(self.indices) % chunks == 0 data = np.array(self.indices) data = data.reshape(chunks, self.batch_size) np.random.shuffle(data) data = data.flatten() self.indices = np.ndarray.tolist(data) return imap(list, partition_all(self.batch_size, self.indices))
def get_request_iterator(self): request_iterator = self.iteration_scheme.get_request_iterator() return chain.from_iterable(imap(partial(repeat, times=self.times), request_iterator))
def get_request_iterator(self): ''' Careful this is indeed infinite ''' return imap(list, partition_all(self.batch_size, cycle(self.indices)))
def get_epoch_iterator(self, **kwargs): labeled = cycle(self.ds_labeled.get_epoch_iterator, **kwargs) unlabeled = self.ds_unlabeled.get_epoch_iterator(**kwargs) return imap(self.mergedicts, labeled, unlabeled)
def get_request_iterator(self): tmp = list(My_partition_all(self.batch_size_list, self.indices)) self.rng.shuffle(tmp) return imap(list, tmp)
def get_request_iterator(self): return imap(list, My_partition_all(self.batch_size_list, self.indices))
def get_epoch_iterator(self, **kwargs): unlabeled = self.ds_unlabeled.get_epoch_iterator(**kwargs) labeled = self.ds_labeled.get_epoch_iterator(**kwargs) assert type(labeled) == type(unlabeled) return imap(self.mergedicts, cycle(labeled), unlabeled)
def open(self): return chain.from_iterable(izip(*[chain.from_iterable( imap(open, repeat(f))) for f in self.files]))
def get_request_iterator(self): return imap(list, partition_all(self.batch_size, self.indices))
def get_request_iterator(self): return imap(list, imap( islice, repeat(xrange(self.num_examples), self.num_batches), repeat(self.batch_size, self.num_batches)))
def get_request_iterator(self): indices = list(range(self.num_examples)) self.rng.shuffle(indices) return imap(list, imap( islice, repeat(iter_(indices), self.num_batches), repeat(self.batch_size, self.num_batches)))
def get_request_iterator(self): return imap( list, imap(islice, repeat(_iter(xrange(self.num_examples)), self.num_batches), repeat(self.batch_size, self.num_batches)))
def get_request_iterator(self): indices = list(self.indices) # shuffle indices indicesShuffled = [] permutation = numpy.random.permutation(len(indices)) return imap(list, partition_all(self.batch_size, permutation))
def get_request_iterator(self): indices = list(self.indices) self.rng.shuffle(indices) return imap(list, partition_all(self.batch_size, indices))