def get_request_iterator(self) :
indices = list(self.indices)
self.rng.shuffle(indices)
fpv = self.frames_per_video
if self.r_subsample:
subsample = np.random.randint(1, self.f_subsample)
else:
subsample = self.f_subsample
frames_array = np.empty([len(indices),fpv])
#each element of indices is the jth video we want
for j in xrange(len(indices)):
i = indices[j]
if i==0 :
c_subsample = self.correct_subsample(0, self.video_indexes[i],
fpv, subsample)
t = self.get_start_frame(0, self.video_indexes[i],
fpv, c_subsample)
else :
c_subsample = self.correct_subsample(self.video_indexes[i-1],
self.video_indexes[i],
fpv, subsample)
t = self.get_start_frame(self.video_indexes[i-1],
self.video_indexes[i],
fpv, c_subsample)
for k in range(fpv):
frames_array[j][k] = t + c_subsample * k
frames_array = frames_array.flatten()
if self.sorted_indices:
return imap(sorted, partition_all(self.batch_size*fpv, frames_array))
else:
return imap(list, partition_all(self.batch_size*fpv, frames_array))
def get_request_iterator(self):
indices = list(self.indices)
self.rng.shuffle(indices)
fpv = self.frames_per_video
if self.r_subsample:
subsample = np.random.randint(1, self.f_subsample)
else:
subsample = self.f_subsample
frames_array = np.empty([len(indices), fpv])
for j in xrange(len(indices)):
i = indices[j]
if i == 0:
c_subsample = self.correct_subsample(0, self.video_indexes[i],
fpv, subsample)
t = self.get_start_frame(0, self.video_indexes[i], fpv,
c_subsample)
else:
c_subsample = self.correct_subsample(self.video_indexes[i - 1],
self.video_indexes[i],
fpv, subsample)
t = self.get_start_frame(self.video_indexes[i - 1],
self.video_indexes[i], fpv,
c_subsample)
for k in range(fpv):
frames_array[j][k] = t + c_subsample * k
frames_array = frames_array.flatten()
if self.sorted_indices:
return imap(sorted,
partition_all(self.batch_size * fpv, frames_array))
else:
return imap(list, partition_all(self.batch_size * fpv,
frames_array))
def get_request_iterator(self):
indices = list(self.indices)
self.rng.shuffle(indices)
if self.sorted_indices:
return imap(sorted, partition_all(self.batch_size, indices))
else:
return imap(list, partition_all(self.batch_size, indices))
def get_request_iterator(self):
indices = list(range(self.num_examples))
self.rng.shuffle(indices)
return imap(
list,
imap(islice, repeat(_iter(indices), self.num_batches),
repeat(self.batch_size, self.num_batches)))
def get_request_iterator(self):
indices = list(self.indices)[::self.batch_size]
self.rng.shuffle(indices)
if self.use_slice:
return imap(slice, _iter(indices),
imap(lambda x: x + self.batch_size, _iter(indices)))
else:
return imap(range, _iter(indices),
imap(lambda x: x + self.batch_size
if x != self.indices[-1]
- (self.indices[-1] % self.batch_size)
else self.indices[-1], _iter(indices)))
def get_request_iterator(self):
indices = list(self.indices)[::self.batch_size]
self.rng.shuffle(indices)
if self.use_slice:
return imap(slice, _iter(indices),
imap(lambda x: x + self.batch_size, _iter(indices)))
else:
return imap(
range, _iter(indices),
imap(
lambda x: x + self.batch_size if x != self.indices[-1] -
(self.indices[-1] % self.batch_size) else self.indices[-1],
_iter(indices)))
def get_request_iterator(self):
indices = list(self.indices)
count = len(indices)
if count < self.batch_size:
count = self.batch_size
indices = self.rng.choice(indices, count)
return imap(list, partition_all(self.batch_size, indices))
def get_request_iterator(self):
indices = list(self.indices)
count = len(indices)
if count < self.batch_size:
count = self.batch_size
indices = self.rng.choice(indices, count)
return imap(list, partition_all(self.batch_size, indices))
def get_data(self, state=None, request=None):
if isinstance(request, list):
batch_index = request[0] / self.batch_size
data = imap(self.filter_sources,
[self.filenames[idx] for idx in request])
else:
raise ValueError("request should be a list instance")
return (data, batch_index)
def get_epoch_iterator(self, **kwargs):
batches = chain.from_iterable(izip(*[data_stream.get_epoch_iterator() for data_stream in self.data_streams]))
part = partition(len(self.sources), chain.from_iterable(batches))
as_dict = kwargs.get("as_dict", False)
if as_dict:
return imap(dict, starmap(zip, izip(repeat(self.sources), part)))
else:
return part
def get_epoch_iterator(self, **kwargs):
batches = chain.from_iterable(
izip(*[data_stream.get_epoch_iterator()
for data_stream in self.data_streams]))
part = partition(len(self.sources), chain.from_iterable(batches))
as_dict = kwargs.get('as_dict', False)
if as_dict:
return imap(dict, starmap(zip, izip(repeat(self.sources), part)))
else:
return part
def get_request_iterator(self):
chunks = len(self.indices) / self.batch_size
assert len(self.indices)%chunks == 0
data = np.array(self.indices)
data = data.reshape(chunks, self.batch_size)
np.random.shuffle(data)
data = data.flatten()
self.indices = np.ndarray.tolist(data)
return imap(list, partition_all(self.batch_size, self.indices))
def get_request_iterator(self):
chunks = len(self.indices) / self.batch_size
assert len(self.indices) % chunks == 0
data = np.array(self.indices)
data = data.reshape(chunks, self.batch_size)
np.random.shuffle(data)
data = data.flatten()
self.indices = np.ndarray.tolist(data)
return imap(list, partition_all(self.batch_size, self.indices))
def get_request_iterator(self):
request_iterator = self.iteration_scheme.get_request_iterator()
return chain.from_iterable(imap(partial(repeat, times=self.times), request_iterator))
def get_request_iterator(self):
'''
Careful this is indeed infinite
'''
return imap(list, partition_all(self.batch_size, cycle(self.indices)))
def get_epoch_iterator(self, **kwargs):
labeled = cycle(self.ds_labeled.get_epoch_iterator, **kwargs)
unlabeled = self.ds_unlabeled.get_epoch_iterator(**kwargs)
return imap(self.mergedicts, labeled, unlabeled)
def get_request_iterator(self):
tmp = list(My_partition_all(self.batch_size_list, self.indices))
self.rng.shuffle(tmp)
return imap(list, tmp)
def get_request_iterator(self):
return imap(list, My_partition_all(self.batch_size_list, self.indices))
def get_epoch_iterator(self, **kwargs):
unlabeled = self.ds_unlabeled.get_epoch_iterator(**kwargs)
labeled = self.ds_labeled.get_epoch_iterator(**kwargs)
assert type(labeled) == type(unlabeled)
return imap(self.mergedicts, cycle(labeled), unlabeled)
def open(self):
return chain.from_iterable(izip(*[chain.from_iterable(
imap(open, repeat(f))) for f in self.files]))
def get_request_iterator(self):
return imap(list, partition_all(self.batch_size, self.indices))
def get_request_iterator(self):
return imap(list, imap(
islice, repeat(xrange(self.num_examples), self.num_batches),
repeat(self.batch_size, self.num_batches)))
def get_request_iterator(self):
indices = list(range(self.num_examples))
self.rng.shuffle(indices)
return imap(list, imap(
islice, repeat(iter_(indices), self.num_batches),
repeat(self.batch_size, self.num_batches)))
def get_request_iterator(self):
return imap(
list,
imap(islice,
repeat(_iter(xrange(self.num_examples)), self.num_batches),
repeat(self.batch_size, self.num_batches)))
def get_request_iterator(self):
indices = list(self.indices)
# shuffle indices
indicesShuffled = []
permutation = numpy.random.permutation(len(indices))
return imap(list, partition_all(self.batch_size, permutation))
def get_request_iterator(self):
indices = list(self.indices)
self.rng.shuffle(indices)
return imap(list, partition_all(self.batch_size, indices))
def get_epoch_iterator(self, **kwargs):
unlabeled = self.ds_unlabeled.get_epoch_iterator(**kwargs)
labeled = self.ds_labeled.get_epoch_iterator(**kwargs)
assert type(labeled) == type(unlabeled)
return imap(self.mergedicts, cycle(labeled), unlabeled)
def get_epoch_iterator(self, **kwargs):
labeled = cycle(self.ds_labeled.get_epoch_iterator, **kwargs)
unlabeled = self.ds_unlabeled.get_epoch_iterator(**kwargs)
return imap(self.mergedicts, labeled, unlabeled)
