def get_epoch_iterator(self, **kwargs):
batches = chain.from_iterable(izip(*[data_stream.get_epoch_iterator() for data_stream in self.data_streams]))
part = partition(len(self.sources), chain.from_iterable(batches))
as_dict = kwargs.get("as_dict", False)
if as_dict:
return imap(dict, starmap(zip, izip(repeat(self.sources), part)))
else:
return part
def get_epoch_iterator(self, **kwargs):
batches = chain.from_iterable(
izip(*[data_stream.get_epoch_iterator()
for data_stream in self.data_streams]))
part = partition(len(self.sources), chain.from_iterable(batches))
as_dict = kwargs.get('as_dict', False)
if as_dict:
return imap(dict, starmap(zip, izip(repeat(self.sources), part)))
else:
return part
def get_request_iterator(self):
request_iterator = self.iteration_scheme.get_request_iterator()
return chain.from_iterable(imap(partial(repeat, times=self.times), request_iterator))
def open(self):
return chain.from_iterable(izip(*[chain.from_iterable(
imap(open, repeat(f))) for f in self.files]))
def get_epoch_iterator(self, **kwargs):
batches = chain.from_iterable(
izip(*[data_stream.get_epoch_iterator()
for data_stream in self.data_streams]))
return partition(len(self.sources), chain.from_iterable(batches))
