def test_returns_word_when_multiple_words_records_scanWindow_3(self):
        tree = Tree(3)
        tree.build(['alpha', 'bids', 'blue', 'carves', 'charlie', 'dances', 'delta', 'eats'])

        tree.mergeCandidates([(0, 2)], 0)
        self.assertEqual([(0, 2)], tree.candidates)
        self.assertEqual([['alpha', 'bids', 'blue'], ['bids', 'blue', 'carves']], tree.get_words(2))

        tree.mergeCandidates([(0, 3)], 0)
        self.assertEqual([(0, 3)], tree.candidates)
        self.assertEqual([['alpha', 'bids', 'blue'], ['bids', 'blue', 'carves'], ['blue', 'carves', 'charlie']], tree.get_words(2))

        tree.mergeCandidates([(0, 4)], 0)
        self.assertEqual([(0, 4)], tree.candidates)
        self.assertEqual([['alpha', 'bids', 'blue'], ['bids', 'blue', 'carves'], ['blue', 'carves', 'charlie'], ['carves', 'charlie', 'dances']], tree.get_words(2))

        tree.mergeCandidates([(1, 4)], 0)
        self.assertEqual([(1, 4)], tree.candidates)
        self.assertEqual([['bids', 'blue', 'carves'], ['blue', 'carves', 'charlie'], ['carves', 'charlie', 'dances']], tree.get_words(2))

        tree.mergeCandidates([(2, 4)], 0)
        self.assertEqual([(2, 4)], tree.candidates)
        self.assertEqual([['blue', 'carves', 'charlie'], ['carves', 'charlie', 'dances']], tree.get_words(2))

        tree.mergeCandidates([(3, 4)], 0)
        self.assertEqual([(3, 4)], tree.candidates)
        self.assertEqual([['carves', 'charlie', 'dances']], tree.get_words(2))
    def test_ignores_punctuation_at_the_end_of_a_word(self):
        tree = Tree(1)
        tree.build(['alpha.', 'bids,', 'blue,', 'carves', 'charlie', 'dances', 'delta', 'eats'])

        tree.mergeCandidates([(0, 3)], 0)
        self.assertEqual([(0, 3)], tree.candidates)
        self.assertEqual([['alpha'], ['bids'], ['blue']], tree.get_words(0))
    def test_returns_word_when_single_word_records(self):
        tree = Tree(1)
        tree.build(['alpha', 'blue', 'charlie', 'delta'])

        tree.mergeCandidates([(0, 1)], 0)
        self.assertEqual([(0, 1)], tree.candidates)
        self.assertEqual([['alpha']], tree.get_words(0))
    def test_returns_words_from_avaialble_candidates(self):
        tree = Tree(1)
        tree.build(['alpha', 'bids', 'blue', 'carves', 'charlie', 'dances', 'delta', 'eats'])

        tree.mergeCandidates([(0, 2)], 0)
        self.assertEqual([(0, 2)], tree.candidates)
        self.assertEqual([['alpha'], ['bids']], tree.get_words(0))

        tree.mergeCandidates([(0, 3)], 0)
        self.assertEqual([(0, 3)], tree.candidates)
        self.assertEqual([['alpha'], ['bids'], ['blue']], tree.get_words(0))

        tree.mergeCandidates([(0, 4)], 0)
        self.assertEqual([(0, 4)], tree.candidates)
        self.assertEqual([['alpha'], ['bids'], ['blue'], ['carves']], tree.get_words(0))

        tree.mergeCandidates([(1, 4)], 0)
        self.assertEqual([(1, 4)], tree.candidates)
        self.assertEqual([['bids'], ['blue'], ['carves']], tree.get_words(0))

        tree.mergeCandidates([(2, 4)], 0)
        self.assertEqual([(2, 4)], tree.candidates)
        self.assertEqual([['blue'], ['carves']], tree.get_words(0))

        tree.mergeCandidates([(3, 4)], 0)
        self.assertEqual([(3, 4)], tree.candidates)
        self.assertEqual([['carves']], tree.get_words(0))
    with smart_open.open(entailData, 'r', encoding='utf-8') as entail:
        for line in entail:
            words.extend(line.strip().split())

    tree.build(words)
    print(tree.records)

    print("Saving Tree to \"Tree\"")
    with open("Tree", 'wb') as outfile:
        pickle.dump(tree, outfile)

    print("Saving Tree done")

    tree.update_words(0, ["today", "Are", "Earth", "hit", "a"])
    tree.update_words(1, ["meteor,", "Earth,"])
    print(tree.get_words(2))

    print(tree.is_contain("meteor, "))
    print(tree.is_contain("a "))
    print(tree.is_contain("a, "))

    print(tree.calc_frequency("meteor, "))
    print(tree.calc_frequency("a "))
    print(tree.calc_frequency("a, "))    
    print("yes yes yes yes", tree.calc_frequency("yes yes yes yes"))

if __name__ == "__main__":
    entailData = "data/entailData.txt"  # entailData to make tree
    gloveModel = "data/glove_model.txt"        # glove model which is converted with glove2word2vector.py
    commonWordFile = "data/wiki-100k.txt"    # wiki common word file.
class TextGenerator(object):
    def __init__(self,
                 window=4,
                 verify=2,
                 ofTranslates=2,
                 ofMatches=1,
                 ofOriginalsNearNextWord=0,
                 CommasPerSentence=2,
                 WordsPerCommaOrPeriod=(4, 7),
                 commonWordCoefficient=(1, 100, 0.0011),
                 contextualCoefficient=(1, 0.000001, 0.0002)):
        self.Tree = None
        self.w2vModel = None
        self.window = window
        self.verify = verify
        self.ofMatches = ofMatches
        self.ofTranslates = ofTranslates
        self.ofOriginalsNearNextWord = ofOriginalsNearNextWord
        self.CommasPerSentence = CommasPerSentence
        self.WordsPerCommaOrPeriod = WordsPerCommaOrPeriod
        self.commonWordCoefficient = commonWordCoefficient
        self.contextualCoefficient = contextualCoefficient
        self.TreeSaveName = "Tree"
        self.treeScanwindow = max(window + 1, verify)
        self.translatesBucket = {
        }  # translatesbucket to contain all translates of words up to now.
        self.pickfromTranslatesBucket = 1  # how many words we should pick from a bucket for softmax.
        self.addToTranslatesBucket = 30  # how many gloves of a word we should insert to translatesBucket.
        self.nofFrequencyCheckCandidates = 8  # how many last candidates we should check for frequency.
        self.lenFrequencyCheckWindow = 2  # length of frequency check window.
        self.fakeCommaChoiceBucket = []
        self.seed_word_list = []
        self.wordTransScore = {}

    def load_Model(self, model_file):
        print("Loading w2v model from \"{}\"...".format(model_file))
        self.w2vModel = gensim.models.KeyedVectors.load_word2vec_format(
            model_file, binary=False)  # GloVe Model
        print("Loading w2v model done")

    def save_Tree(self):
        print("Saving Tree to \"{}\"".format(self.TreeSaveName))
        with open(self.TreeSaveName, 'wb') as outfile:
            pickle.dump(self.Tree, outfile)
        print("Saving Tree done")

    def load_Tree(self):
        print("Loading Tree from \"{}\"...".format(self.TreeSaveName))
        with open(self.TreeSaveName, 'rb') as infile:
            self.Tree = pickle.load(infile)
        print("Loading Tree done")

    def init_Tree(self, entailData: str):
        """
        Function to construct tree from entailData
        """
        self.Tree = Tree(self.treeScanwindow)
        words = []
        with smart_open.smart_open(entailData, 'r') as entail:
            for line in entail:
                words.extend(line.strip().split())

        self.Tree.build(words)
        self.save_Tree()
        # if DEBUG:
        # print(self.Tree.records)

    def load_CommonWord(self, wordFile: str):
        self.wordTransScore = {}
        cnt = 1
        with smart_open.smart_open(wordFile, encoding='utf8') as f:
            for line in f:
                splited_line = line.strip().split()
                word = splited_line[0].lower()
                if len(splited_line) == 1 and word[0] != '#':
                    if not word in self.wordTransScore:
                        self.wordTransScore[word] = cnt
                        cnt = cnt + 1
        """
        we are using following function for exponential curve according to word list.
        y = k * exp(-a * x) + b
        this curve will pass 2 points (1, M1) & (L, M2)
        a will be the incline prameter.
        """

        M1 = self.commonWordCoefficient[1]
        M2 = self.commonWordCoefficient[0]
        a = self.commonWordCoefficient[2]
        L = len(self.wordTransScore)

        k = (M1 - M2) / (math.exp(-a) - math.exp(-a * L))
        b = (M2 * math.exp(-a) - M1 * math.exp(-a * L)) / (math.exp(-a) -
                                                           math.exp(-a * L))

        for word in self.wordTransScore:
            self.wordTransScore[word] = M1 + M2 - (
                k * math.exp(-a * (self.wordTransScore[word])) + b)
            # if DEBUG:
            # print(word, self.wordTransScore[word])

    def calc_Transscore(self, word: str):
        if word in self.wordTransScore:
            return self.wordTransScore[word]
        else:
            return self.commonWordCoefficient[1]

    def makeGlove(self, word, ofTranslates):
        if word[-1] == "," or word[-1] == ".":
            suffix = word[-1]
            word = word[:-1]
        else:
            suffix = ''
        word_list = [word + suffix]
        try:
            for tup in self.w2vModel.similar_by_word(word, ofTranslates):
                word_list.append(tup[0] + suffix)
        except:
            pass
        return word_list

    def insertToTranslatesBucket(self, insert_word, seed_word):
        insert_score = self.calc_Transscore(seed_word)
        if insert_word in self.translatesBucket:
            self.translatesBucket[insert_word] += insert_score
        else:
            self.translatesBucket[insert_word] = insert_score

    def pickNextword(self, hypolist):  #
        predictedHypoList = []
        IndexesofPredictedHypo = {}

        for idx, hp in enumerate(hypolist):
            word, cnt = hp[0][-1], hp[1]
            if not (word in predictedHypoList):
                predictedHypoList.append(word)
                IndexesofPredictedHypo[word] = [idx]
            else:
                IndexesofPredictedHypo[word].append(idx)

        def dist(hp, orig):
            L = len(hp) - 1
            ret = 0
            for i in range(0, L):
                if hp[-2 - i] == orig[-1 - i]:
                    ret += 1
            return ret

        if DEBUG:
            print("Scores of each word")
            print("{:<20}: {:<10}: {:<10}, {:<10}, {:<10}, {:<10}".format(
                "word", "score", "TransScore", "LenScore", "Contextual",
                "Originals"))

        relatedTranslatesBucket = []

        M1 = self.contextualCoefficient[1]
        M2 = self.contextualCoefficient[0]
        a = self.contextualCoefficient[2]
        L = self.window + 1

        k = (M2 - M1) / (math.exp(L * a) - math.exp(a))
        b = (M1 * math.exp(L * a) - M2 * math.exp(a)) / (math.exp(L * a) -
                                                         math.exp(a))

        for word in predictedHypoList:
            maxLen = 0
            contextualFrequency = 0
            originals = 0
            for idx in IndexesofPredictedHypo[word]:
                curLen = len(hypolist[idx][0])
                maxLen = max(maxLen, curLen)
                exponentialCoeff = (k * math.exp(a * curLen) +
                                    b) * hypolist[idx][1]
                contextualFrequency += exponentialCoeff
                originals += exponentialCoeff * dist(hypolist[idx][0],
                                                     self.seed_word_list)

            if word in self.translatesBucket:
                transScore = self.translatesBucket[word]
            else:
                transScore = 0

            score = transScore + maxLen + contextualFrequency + originals
            if DEBUG:
                print(
                    "{:<20}: {:10.4f}: {:10.4f}, {:10.4f}, {:10.4f}, {:10.4f}".
                    format(word, score, transScore, maxLen,
                           contextualFrequency, originals))

            relatedTranslatesBucket.append((score, word))

        ntopCandidates = min(self.pickfromTranslatesBucket,
                             len(predictedHypoList))
        topCandidates = heapq.nlargest(ntopCandidates, relatedTranslatesBucket)

        predictedHypoList = []
        softMaxVals = []

        for (score, word) in topCandidates:
            softMaxVals.append(score)
            predictedHypoList.append(word)

        return random.choices(predictedHypoList, weights=softMaxVals)[0]

    def predict(self, input_seed: list) -> str:
        """
        Function to predict next word of a sentence.
        """

        self.seed_word_list = []
        for word in input_seed:
            self.seed_word_list.append(word.lower())

        verified_hypo_list = []
        total_verified = 0
        windowLen = min(len(input_seed), self.window)

        while windowLen > 0:
            ofOriginalsNearNextWord = min(windowLen,
                                          self.ofOriginalsNearNextWord)
            verifyLen = min(self.verify, windowLen + 1)

            ## get window list from seed word list
            windowlist = self.seed_word_list[-windowLen:]

            ## make listoflists
            listoflists = []

            idx = -1
            for i, word in enumerate(windowlist):
                if word[-1] == ',' or word[-1] == '.':
                    idx = i

            for i, word in enumerate(windowlist):
                if i == idx:
                    listoflists.append(self.makeGlove(word, self.ofTranslates))
                else:
                    listoflists.append(
                        self.makeGlove(self.stem_Word(word),
                                       self.ofTranslates))

            if DEBUG:
                print("Lists of %d word window =" % windowLen, listoflists)

            ## do vanilla search
            idx = 0
            for word_list in listoflists:
                self.Tree.update_words(idx, word_list)
                idx = idx + 1

            ## get all hypothesis of next word
            hypo_list = self.Tree.get_words(idx)

            if DEBUG:
                print("hypo_list = ", hypo_list)

            def is_ofOriginalsNearNextWord(a, b, L):
                """
                check if last L words of a[:-1] are same as b[-L:]
                """
                for i in range(L):
                    if a[-i - 2] != b[-i - 1]:
                        return False
                return True

            ## get ofOriginalNearNextWord list
            ofOs_hypo_list = []
            for hp in hypo_list:
                if is_ofOriginalsNearNextWord(hp, windowlist,
                                              ofOriginalsNearNextWord) == True:
                    ofOs_hypo_list.append(hp)
            if DEBUG:
                print("ofOs_hypo_list = ", ofOs_hypo_list)

            def add_to_verified_HypoList(hp):
                kLen = 0  # sum of length that contain this hp in verified hypo list.
                jdx = -1  # index fo current hp in verified hypo list.
                lenVerified = len(verified_hypo_list)
                for idx in range(0, lenVerified):
                    prev = verified_hypo_list[idx][0]
                    L = len(hp)
                    isContained = True
                    for i in range(L):
                        if hp[-i - 1] != prev[-i - 1]:
                            isContained = False
                            break
                    if isContained == True:
                        if len(prev) == L:
                            jdx = idx
                        else:
                            kLen += verified_hypo_list[idx][1]

                ret = 1

                if jdx == -1:
                    if kLen == 0:
                        verified_hypo_list.append((hp, 1))
                    else:
                        verified_hypo_list.append((hp, 1 - kLen))
                        ret -= kLen

                else:
                    verified_hypo_list[jdx] = (verified_hypo_list[jdx][0],
                                               verified_hypo_list[jdx][1] + 1)

                return ret

            ## do verification
            prefix = ' '.join(windowlist[-verifyLen + 1:])
            for hp in ofOs_hypo_list:
                search_str = prefix + " " + hp[-1] + " "
                search_str_comma = prefix + " " + hp[-1] + ", "
                search_str_stop = prefix + " " + hp[-1] + ". "
                if self.Tree.is_contain(
                        search_str) == True or self.Tree.is_contain(
                            search_str_comma) == True or self.Tree.is_contain(
                                search_str_stop) == True:  # verified
                    total_verified += add_to_verified_HypoList(hp)

            if DEBUG:
                print("verified_hypo_list = ", verified_hypo_list)

            if total_verified >= self.ofMatches:
                break
            else:
                windowLen = windowLen - 1

        if total_verified == 0:
            print("Matches =", [])
            return None

        def dist(hp, orig):
            L = len(hp) - 1
            LO = len(orig)
            nSame = 0
            pattern = 0
            for i in range(0, L):
                if hp[-2 - i] == orig[-1 - i]:
                    nSame = nSame + 1
                    pattern = pattern + (1 << (LO - i))

            return (nSame, pattern)

        windowlist = self.seed_word_list[-self.window:]

        L = len(verified_hypo_list)

        ofMatches = min(total_verified, self.ofMatches)

        for i in range(0, L):
            for j in range(i + 1, L):
                if dist(verified_hypo_list[i][0], windowlist) < dist(
                        verified_hypo_list[j][0], windowlist):
                    verified_hypo_list[i], verified_hypo_list[
                        j] = verified_hypo_list[j], verified_hypo_list[i]

        curMatches = 0
        for i in range(0, L):
            curMatches += verified_hypo_list[i][1]
            if curMatches >= ofMatches:
                verified_hypo_list[i] = (verified_hypo_list[i][0],
                                         verified_hypo_list[i][1] -
                                         (curMatches - ofMatches))
                verified_hypo_list = verified_hypo_list[:i + 1]
                break

        matcheStr = []
        for hp in verified_hypo_list:
            matcheStr.append((' '.join(hp[0]), hp[1]))

        print("Matches =", matcheStr)
        return self.pickNextword(verified_hypo_list)

    def calcCommaLen(self, words):
        L = len(words)
        for i in range(L):
            if words[-i - 1][-1] == '.' or words[-i - 1][-1] == ',':
                return i
        return L

    def stem_Word(self, word):
        res = word.lower()
        if res[-1] == '.' or res[-1] == ',':
            res = res[:-1]
        return res

    def make_FirstWord(self, word):
        res = word[0].upper() + word[1:]
        return res

    def isLastCandidateMostFrequent(self, words):
        """
        check if the last candidates is most frequent in entailData if we add comma/stop.
        """
        if DEBUG:
            print("Checking if last word is most frequent for hypo -> ",
                  ' '.join(words))

        L = len(words)
        last_frequency = 0
        most_frequency = 0
        for i in range(self.nofFrequencyCheckCandidates):
            if L - self.lenFrequencyCheckWindow - i < 0:
                break
            candy = []
            for j in range(self.lenFrequencyCheckWindow):
                candy.append(
                    self.stem_Word(words[L - self.lenFrequencyCheckWindow - i +
                                         j]))

            comma_cnt = self.Tree.calc_frequency(' '.join(candy) + ', ')
            stop_cnt = self.Tree.calc_frequency(' '.join(candy) + '. ')
            total_cnt = self.Tree.calc_frequency(' '.join(candy) +
                                                 ' ') + comma_cnt + stop_cnt
            # next_word, next_cnt = self.Tree.calc_most_frequent_next_word(' '.join(candy)+' ')

            # if next_cnt == 0:
            if total_cnt == 0:
                freq = 0
            else:
                # freq = (comma_cnt + stop_cnt)/next_cnt
                freq = (comma_cnt + stop_cnt) / total_cnt

            if DEBUG:
                # print("\"{}\": {},\t\"{}\": {},\tMost freq next word: ({}, {}),\tFreq: {}".format(' '.join(candy)+",", comma_cnt, ' '.join(candy)+".", stop_cnt, next_word, next_cnt, freq))
                print("\"{}\": {},\t\"{}\": {},\tTotal : {},\tFreq: {}".format(
                    ' '.join(candy) + ",", comma_cnt, ' '.join(candy) + ".",
                    stop_cnt, total_cnt, freq))

            if i == 0:
                last_frequency = freq
            else:
                most_frequency = max(most_frequency, freq)

        if last_frequency > most_frequency:
            return True
        else:
            return False

    def addNextWord(self, words, next_word):
        if words[-1][-1] == '.':
            words.append(self.make_FirstWord(next_word))
        else:
            words.append(next_word)

        for glove_next_word in self.makeGlove(next_word,
                                              self.addToTranslatesBucket):
            self.insertToTranslatesBucket(glove_next_word, next_word)

        if self.isLastCandidateMostFrequent(
                words) == False:  # we couldn't add any comma/stop
            if DEBUG:
                print("we can't add comma due to last word frequency rule.")
            return words

        commaLen = self.calcCommaLen(words)

        bshouldAddComma = False
        if commaLen >= self.WordsPerCommaOrPeriod[0]:
            if commaLen >= self.WordsPerCommaOrPeriod[1]:
                bshouldAddComma = True
            else:
                if random.choice([True, False]) == True:
                    bshouldAddComma = True

        L = len(words)

        if bshouldAddComma == True:  # we should add real comma/stop
            numCommas = 0  # calc number of commas in current sentence.
            for i in range(L):
                if words[-i - 1][-1] == ',':
                    numCommas = numCommas + 1
                if words[-i - 1][-1] == '.':
                    break
            """
            we have to add addCharacter(./,) after last word now.
            """
            if DEBUG:
                print("add real comma/stop")

            if numCommas >= self.CommasPerSentence:  # we have to add stop
                words[-1] = words[-1] + '.'
            else:  # we have to add comma
                words[-1] = words[-1] + ','
        else:
            if DEBUG:
                print("we can't add comma due to WordsPerCommaOrPeriod rule.")

        return words

    def morph_seed(self, seed: str, length):
        self.translatesBucket = {}
        words = seed.strip().split()

        for word in words:
            for glove_word in self.makeGlove(self.stem_Word(word),
                                             self.addToTranslatesBucket):
                self.insertToTranslatesBucket(glove_word, self.stem_Word(word))

        for i in range(length):
            print("Seed=" + " ".join(words))
            word = self.predict(words)
            if word == None:
                print("can't predict next word and end morphing.")
                return
            #if keyboard.is_pressed('q'):
            #    return
            words = self.addNextWord(words, word)
        print("Seed=" + " ".join(words))