class CorpusIteratorFuncHeadFraction():
def __init__(self,
language,
partition="train",
fraction=1.0,
storeMorph=False,
splitLemmas=False):
self.basis = CorpusIterator(language,
partition=partition,
storeMorph=storeMorph,
splitLemmas=splitLemmas,
shuffleDataSeed=4)
self.basis.data = self.basis.data[:int(fraction *
len(self.basis.data))]
self.permute()
self.fraction = fraction
def permute(self):
self.basis.permute()
def length(self):
return self.basis.length()
def iterator(self, rejectShortSentences=False):
iterator = self.basis.iterator(
rejectShortSentences=rejectShortSentences)
counter = 0
print("Actual length", self.length())
for sentence in iterator:
reverse_content_head(sentence)
yield sentence
def getSentence(self, index):
return reverse_content_head(self.basis.getSentence(index))
class CorpusIteratorFuncHead():
def __init__(self,
language,
partition="train",
storeMorph=False,
splitLemmas=False):
self.basis = CorpusIterator(language,
partition=partition,
storeMorph=storeMorph,
splitLemmas=splitLemmas)
def permute(self):
self.basis.permute()
def length(self):
return self.basis.length()
def iterator(self, rejectShortSentences=False):
iterator = self.basis.iterator(
rejectShortSentences=rejectShortSentences)
for sentence in iterator:
reverse_content_head(sentence)
yield sentence
def getSentence(self, index):
return reverse_content_head(self.basis.getSentence(index))