def create(self,folder_path,index_path):
     "create inverted index from all text files in path"
     # clean old inverted index and docList in memory
     if(not self.root == None):
         self.inverted_index.clean(self.root)
         self.inverted_index = Inverted_Index()
         self.root = None
         self.docList[:] =[]
     # check whether folder does exist?
     if(not os.path.isdir(folder_path)):
         return False
     cwd = os.getcwd()
     os.chdir(folder_path)
     docCount=0
     for file in glob.glob("*"):
         fp = open(file)
         self.docList.append((docCount,file))
         #print(self.docList[docCount])
         # find max term count
         max_tc = 0
         for i,line in enumerate(fp):
             for word in line.split():
                 if(self.check_stopword(word)):continue
                 if(self.root==None):
                     self.root = self.inverted_index.createNode(word,self.docList[docCount])
                     self.inverted_index.setRoot(self.root)
                 else:
                     self.inverted_index.insert(self.root,word,self.docList[docCount])
         fp.close()
         docCount=docCount+1
     self.inverted_index.setDocCount(docCount)
     os.chdir(cwd)
     with open(index_path,'wb') as index_file:
         pickle.dump(self.inverted_index,index_file)
     return True
class Index_System:
    "an inverted index structure with hash table as dictionary and simple list as docID list"
    root, inverted_index, docList = None, None, []
    stop_words=["a","an","and","are","as","at","be","by","for","from","has","he","in","is","it","its","of","on","that","the","to","was","were","will","with"]

    def __init__(self): #constructor
        self.inverted_index = Inverted_Index()
        self.docList = []

    # TODO:
    def normalize(word):pass

    def check_stopword(self,word):
        for stop_word in self.stop_words:
            if(stop_word ==word):
                return True
        return False

    def create(self,folder_path,index_path):
        "create inverted index from all text files in path"
        # clean old inverted index and docList in memory
        if(not self.root == None):
            self.inverted_index.clean(self.root)
            self.inverted_index = Inverted_Index()
            self.root = None
            self.docList[:] =[]
        # check whether folder does exist?
        if(not os.path.isdir(folder_path)):
            return False
        cwd = os.getcwd()
        os.chdir(folder_path)
        docCount=0
        for file in glob.glob("*"):
            fp = open(file)
            self.docList.append((docCount,file))
            #print(self.docList[docCount])
            # find max term count
            max_tc = 0
            for i,line in enumerate(fp):
                for word in line.split():
                    if(self.check_stopword(word)):continue
                    if(self.root==None):
                        self.root = self.inverted_index.createNode(word,self.docList[docCount])
                        self.inverted_index.setRoot(self.root)
                    else:
                        self.inverted_index.insert(self.root,word,self.docList[docCount])
            fp.close()
            docCount=docCount+1
        self.inverted_index.setDocCount(docCount)
        os.chdir(cwd)
        with open(index_path,'wb') as index_file:
            pickle.dump(self.inverted_index,index_file)
        return True

    def show(self):
        "show the inverted index data"
        print("All inverted index data is as below:")
        print("--------------- Inverted Index ----------------")
        self.inverted_index.printTree(self.root)
        print("-----------------------------------------------")
        print()
 def __init__(self): #constructor
     self.inverted_index = Inverted_Index()
class Search_System:
    "a search system"
    inverted_index = None

    def __init__(self): #constructor
        self.inverted_index = Inverted_Index()

    def rank_answer(list, score_list):
        mix_list = zip(list,score_list)
        sorted_list = sorted(mix_list,key=lambda value:value[1],reverse = True)
        answer_list = [i for i,j in sorted_list]
        return answer_list

    def search3(self,query):
        "search for many query with boolean expression"

        root = self.inverted_index.root
        query_list = query.split()
        expression = shunting_yard.infixToRPN(query_list)
        expression.reverse()
        stack = []
        plist_num=0
        plist_hashlist={}
        #[2012/08/22 added] to initial variables to find score
        slist_num=0
        slist_hashlist={}

        # just find that term if doing only single word query.
        if len(expression) ==0:
            return []
        elif len(expression) <= 1:
            word = expression.pop()
            answer_last= self.inverted_index.lookup(root,word)
            # get node to find tf-idf
            node_answer_last = self.inverted_index.lookupNode(root,word)
            if(node_answer_last==None):
                return []
            tf_idf_list = node_answer_last.get_tf_idf_list(self.inverted_index.docCount,self.inverted_index.highest_tc)
            ranked_answer = Search_System.rank_answer(answer_last,tf_idf_list)
            return ranked_answer
        # in case of > 1 word query
        while len(expression)!=0:
            token = expression.pop()
            if shunting_yard.isOperator(token):
                word1 = stack.pop()
                if( not word1 in plist_hashlist ):
                    answer_list1 = self.inverted_index.lookup(root,word1)
                    # [2012/08/22 added] to get score of word
                    node1 = self.inverted_index.lookupNode(root,word1)
                    if(node1!=None):
                        score_list1 = node1.get_tf_idf_list(self.inverted_index.docCount,self.inverted_index.highest_tc)
                    else: score_list1 = []
                else:
                    answer_list1 = plist_hashlist[word1]
                    # [2012/08/22 added] to get score of word
                    score_list1 = slist_hashlist[word1]

                word2 = stack.pop()
                if( not word2 in plist_hashlist ):
                    answer_list2 = self.inverted_index.lookup(root,word2)
                    # [2012/08/22 added] to get score of word
                    node2 = self.inverted_index.lookupNode(root,word2)
                    if(node2!=None):
                        score_list2 = node2.get_tf_idf_list(self.inverted_index.docCount,self.inverted_index.highest_tc)
                    else: score_list2 = []
                else:
                    answer_list2 = plist_hashlist[word2]
                    # [2012/08/22 added] to get score of word
                    score_list2 = slist_hashlist[word2]

                if(token=="&&"):
                    answer_last = self.intersect(answer_list1,answer_list2)
                    # [2012/08/22 added] to get score of word
                    score_list_last = self.intersect_score(answer_list1,answer_list2,score_list1,score_list2)
                elif(token=="||"):
                    answer_last = self.union(answer_list1,answer_list2)
                    # [2012/08/22 added] to get score of word
                    score_list_last = self.union_score(answer_list1,answer_list2,score_list1,score_list2)
                else:
                    print("Wrong Operator: program will exit")
                    exit()
                stack.append("plist"+str(plist_num))
                plist_hashlist["plist"+str(plist_num)] = answer_last
                plist_num+=1
                #[2012/08/22 added] to store new score list in hash table
                slist_hashlist["plist"+str(slist_num)] = score_list_last
                slist_num+=1

            else:
                stack.append(token)

        # get all the last answers
        #[2012/08/22 added] to store new score list in hash table
        last_word = stack.pop()
        answer = plist_hashlist[last_word];
        #[2012/08/22 added] to get answer score list in hash table
        s_answer = slist_hashlist[last_word]
        print(answer)
        print(s_answer)
        ranked_answer = Search_System.rank_answer(answer,s_answer)
        return ranked_answer

    def union(self, sorted_plist1, sorted_plist2):
        "union of two postings lists plist1 and plist2, in other word, it is operand OR"
        answer = []
        p1,p2=0,0
        while(p1<len(sorted_plist1) and p2<len(sorted_plist2)):
            if(sorted_plist1[p1]==sorted_plist2[p2]):
                answer.append(sorted_plist1[p1])
                p1+=1;p2+=1
            elif(sorted_plist1[p1][0]<sorted_plist2[p2][0]):
                answer.append(sorted_plist1[p1])
                p1+=1
            else:
                answer.append(sorted_plist2[p2])
                p2+=1
        while(p1<len(sorted_plist1)):
            answer.append(sorted_plist1[p1])
            p1+=1
        while(p2<len(sorted_plist2)):
            answer.append(sorted_plist2[p2])
            p2+=1
        return answer

    # [2012/08/22 added] find union of 2 score lists
    def union_score(self, sorted_plist1, sorted_plist2,slist1,slist2):
        "union of two postings lists plist1 and plist2, in other word, it is operand OR"
        answer = []
        s_answer=[]
        p1,p2=0,0
        while(p1<len(sorted_plist1) and p2<len(sorted_plist2)):
            if(sorted_plist1[p1]==sorted_plist2[p2]):
                answer.append(sorted_plist1[p1])
                s_answer.append(slist1[p1]+slist2[p2])
                p1+=1;p2+=1
            elif(sorted_plist1[p1][0]<sorted_plist2[p2][0]):
                answer.append(sorted_plist1[p1])
                s_answer.append(slist1[p1])
                p1+=1
            else:
                answer.append(sorted_plist2[p2])
                s_answer.append(slist2[p2])
                p2+=1
        while(p1<len(sorted_plist1)):
            answer.append(sorted_plist1[p1])
            s_answer.append(slist1[p1])
            p1+=1
        while(p2<len(sorted_plist2)):
            answer.append(sorted_plist2[p2])
            s_answer.append(slist2[p2])
            p2+=1
        return s_answer

    def intersect(self,sorted_plist1,sorted_plist2):
        "intersection of two postings lists plist1 and plist2, in other word, it is operand AND"
        answer = []
        p1,p2=0,0
        while(p1<len(sorted_plist1) and p2<len(sorted_plist2)):
            if(sorted_plist1[p1]==sorted_plist2[p2]):
                answer.append(sorted_plist1[p1])
                p1+=1; p2+=1
            elif(sorted_plist1[p1][0]<sorted_plist2[p2][0]):
                p1+=1
            else: p2+=1
        return answer

    # [2012/08/22 added] find intersection of 2 score lists
    def intersect_score(self,sorted_plist1,sorted_plist2,slist1,slist2):
        "intersection of two postings lists plist1 and plist2, in other word, it is operand AND"
        answer = []
        s_answer= []
        p1,p2=0,0
        while(p1<len(sorted_plist1) and p2<len(sorted_plist2)):
            if(sorted_plist1[p1]==sorted_plist2[p2]):
                answer.append(sorted_plist1[p1])
                s_answer.append(slist1[p1]+slist2[p2])
                p1+=1; p2+=1
            elif(sorted_plist1[p1][0]<sorted_plist2[p2][0]):
                p1+=1
            else: p2+=1
        return s_answer

    def load_index(self,path):
        "load inverted index to search system"
        self.inverted_index.clean(self.inverted_index.root)
        if(not os.path.isfile(path)):
            print("no such file - "+path)
            return False
        with open(path,'rb') as index_file:
            self.inverted_index = pickle.load(index_file)
        #self.inverted_index.printTree(self.inverted_index.root)
        return True