class CorpusIteratorFuncHead_V():
def __init__(self,
language,
partition="train",
storeMorph=False,
splitLemmas=False,
shuffleDataSeed=None):
self.basis = CorpusIterator_V(language,
partition=partition,
storeMorph=storeMorph,
splitLemmas=splitLemmas,
shuffleDataSeed=shuffleDataSeed)
def permute(self):
self.basis.permute()
def length(self):
return self.basis.length()
def iterator(self, rejectShortSentences=False):
iterator = self.basis.iterator(
rejectShortSentences=rejectShortSentences)
for sentence in iterator:
reverse_content_head(sentence)
yield sentence
def getSentence(self, index):
return reverse_content_head(self.basis.getSentence(index))
class CorpusIteratorFuncHeadFraction_V():
def __init__(self, language, partition="train", fraction=1.0, storeMorph=False, splitLemmas=False):
self.basis = CorpusIterator_V(language, partition=partition, storeMorph=storeMorph, splitLemmas=splitLemmas, shuffleDataSeed=4)
self.basis.data = self.basis.data[:int(fraction*len(self.basis.data))]
self.permute()
self.fraction = fraction
def permute(self):
self.basis.permute()
def length(self):
return self.basis.length()
def iterator(self, rejectShortSentences = False):
iterator = self.basis.iterator(rejectShortSentences=rejectShortSentences)
counter = 0
print("Actual length", self.length())
for sentence in iterator:
# if counter > self.fraction * self.length():
# break
# counter += 1
reverse_content_head(sentence)
yield sentence
def getSentence(self, index):
return reverse_content_head(self.basis.getSentence(index))
