def get_epoch_iterator(self, **kwargs):
batches = chain.from_iterable(izip(*[data_stream.get_epoch_iterator() for data_stream in self.data_streams]))
part = partition(len(self.sources), chain.from_iterable(batches))
as_dict = kwargs.get("as_dict", False)
if as_dict:
return imap(dict, starmap(zip, izip(repeat(self.sources), part)))
else:
return part
def get_data(self, request=None):
if request is not None:
raise ValueError
data = next(self.child_epoch_iterator)
return [d for d, s in izip(data, self.data_stream.sources)
if s in self.sources]
def get_data_from_batch(self, request=None):
if not self.data:
data = next(self.child_epoch_iterator)
self.data = izip(*data)
try:
return next(self.data)
except StopIteration:
self.data = None
return self.get_data()
def get_data(self, request=None):
if request is not None:
raise ValueError
if not self.data:
data = next(self.child_epoch_iterator)
self.data = izip(*data)
try:
return next(self.data)
except StopIteration:
self.data = None
return self.get_data()
def transform_any(self, data):
return [d for d, s in izip(data, self.data_stream.sources)
if s in self.sources]
def open(self):
iterators = [iter_(channel) for channel in self.iterables]
return izip(*iterators)
def open(self):
return chain.from_iterable(izip(*[chain.from_iterable(
imap(open, repeat(f))) for f in self.files]))
def open(self):
iterators = [iter_(channel) for channel in self.iterables]
return izip(*iterators)
def _get_src_sentences_with_references(stream):
iterator = _get_iterator(stream)
references = []
n = next(iterator)
references.extend(izip(*n[2:]))
return list(n[:2]), references
def get_epoch_iterator(self, **kwargs):
batches = chain.from_iterable(
izip(*[data_stream.get_epoch_iterator()
for data_stream in self.data_streams]))
return partition(len(self.sources), chain.from_iterable(batches))
def transform_any(self, data):
return [
d for d, s in izip(data, self.data_stream.sources)
if s in self.sources
]
