#! /usr/bin/python

#

# Stephen Poletto (spoletto)

# CSCI1580 - Web Search

# Spring 2011 - Brown University

#

# Reads in a list of queries

# and returns matching documents

# using the provided index file.

from bool_parser import bool_expr_ast

from documentVector import *

from positionalIndex import *

from porterStemmer import *

from kgramIndex import *

import marshal

import string

import bsddb

import math

import sys

import re

import os

K_TOP_DOCUMENTS = 10

class QuerySideIndex:

return marshal.loads(self.file.read(int(stop)-int(start)))

if (len(sys.argv) != 10):

print ""

print "usage: queryIndex <index> <stopwords> <titleindex> <kgramIndex> <outgoingLinks> <pageRank> <features> <vecrep> <training>"

print ""

sys.exit()

for i in range(1, 10):

if not os.path.exists(sys.argv[i]):

print ""

print "File " + sys.argv[i] + " does not exist."

print ""

sys.exit()

# Load the index:

indexFile = open(sys.argv[1], 'r')

term_to_file_position_start = int(indexFile.readlines()[-1])

term_to_file_position_end = indexFile.tell()

indexFile.seek(term_to_file_position_start, 0)

term_to_file_position = marshal.loads(indexFile.read(term_to_file_position_start - term_to_file_position_end))

index = QuerySideIndex(indexFile, term_to_file_position)

# Load the k-gram index

kgramFile = open(sys.argv[4])

k_gram = KGramIndex(marshal.loads(kgramFile.read()))

kgramFile.close()

# Load the page rank:

pageRankFile = open(sys.argv[6])

docIDToPageRank = {}

currDocID = 0

for line in pageRankFile:

line = line.rstrip('\n')

docIDToPageRank[currDocID] = float(line)

currDocID += 1

pageRankFile.close()

# Load the outgoing links

outgoingLinksFile = open(sys.argv[5])

docIDToOutgoingLinks = {}

currDocID = 0

for line in outgoingLinksFile:

line = line.rstrip('\n')

docIDToOutgoingLinks[currDocID] = line

currDocID += 1

outgoingLinksFile.close()

# Read total number of documents in collection.

titleIndexFile = open(sys.argv[3])

documents_in_collection = 0

docIDToTitle = {}

for line in titleIndexFile:

line = line.rstrip("\n")

docID = int(line.split(" ")[0])

title = " ".join(line.split(' ')[1:])

docIDToTitle[docID] = title.lower()

documents_in_collection += 1

titleIndexFile.close()

# Read in the stop words

stopWordsFile = open(sys.argv[2], "r")

stopWords = []

for stop in stopWordsFile.readlines():

word = stop.rstrip("\n")

stopWords.append(word)

stopWords = set(stopWords)

# Open the classifier files

featuresFile = open(sys.argv[7], "r")

vecrepFile = open(sys.argv[8], "r")

trainingFile = open(sys.argv[9], "r")

features = {}

featureID = 0

for feature in featuresFile.readlines():

feature = feature.rstrip("\n")

features[feature] = featureID

featureID += 1

featureCount = featureID

featuresFile.close()

# For each document in the file generated by

# vecrep, construct a DocumentVector.

# The DocumentVector will store both a normalized

# vector and a regular count vector for feature

# occurrences.

docIDToVector = {}

for vecrep in vecrepFile.readlines():

vecrep = vecrep.rstrip("\n")

vector = DocumentVector(vecrep[:-1], featureCount)

docIDToVector[vector.docID] = vector

vecrepFile.close()

# Build a dictionary mapping each class to

# a set of documents that have been classified as

# belonging to that class.

classIDToPreclassifiedDocs = {}

for classifiedDoc in trainingFile.readlines():

docTuple = classifiedDoc.split(' ')

docID = int(docTuple[0])

classID = int(docTuple[1])

classIDToPreclassifiedDocs.setdefault(classID, []).append(docID)

trainingFile.close()

# Compute the total number of documents in the collection.

totalNumberOfDocs = 0

for classID in classIDToPreclassifiedDocs:

docsInClass = classIDToPreclassifiedDocs[classID]

totalNumberOfDocs += len(docsInClass)

# Build the Porter Stemmer

stemmer = PorterStemmer()

# Compute the idf for each feature for tf-idf calculation

def compute_feature_id_to_idf():

return featureIDToIDF

# The centroid of each class is computed as the vector average

# or center of mass of its members. We iterate over all docs

# belonging to the class, and sum up their normalized vectors.

# We then divide this normalized vector sum by the count of

# documents occurring in the class.

def train_rocchio(logOfFeatureIDToIDF):

return classIDToCentroid

logOfFeatureIDToIDF = numpy.log10(compute_feature_id_to_idf())

classIDToCentroid = train_rocchio(logOfFeatureIDToIDF)

def class_for_doc_ID(docID):

return bestClass[0]

def class_for_query(term_to_idf):

return bestClass[0]

def score_matching_docs(setOfDocIDs, term_to_idf, term_to_postings_list):

return scores

def ranked_results(setOfDocIDs, term_to_idf, num_top_docs):

return [scores[i][1] for i in range(0, upper_bound)]

def wildcard_ranked_results(matchingDocIDs, queryTermToDocIDToWildcardWeight, term_to_idf):

return [scores[i][1] for i in range(0, upper_bound)]

def wildcard_weight(docIDs, terms):

return docIDToWildcardWeight, bestIDF

def print_docIDs(topDocuments):

sys.stdout.write('\n')

# Print the results to stdout

def print_ranked_results(setOfDocIDs, term_to_idf):

print_docIDs(ranked_results(setOfDocIDs, term_to_idf, K_TOP_DOCUMENTS))

# For processing one word queries.

# Simply look up the word in the

# dictionary, and print the postings

# list of document IDs.

def processOWQ_private(query):

return set(docIDs)

def processOWQ(query, term_to_idf):

print_ranked_results(processOWQ_private(query), term_to_idf)

# For processing phrase queries.

# Each term in the keyword list

# must occur within a single

# document, so first we take

# the intersection of documents

# from the postings lists associated

# with each term. Then, once we have

# our candidate documents, we check

# the positional indices to make sure

# the phrase appears as specified

# in the query.

def docs_containing_all_terms_in_list(keywordList):

return candidateDocs, wordToDocList

def docs_containing_all_terms_in_order(keywordList, candidateDocs, wordToDocList):

return successfulDocuments

def processPQ_private(keywordList, term_to_idf):

return docs_containing_all_terms_in_order(keywordList, candidateDocs, wordToDocList)

def processPQ(keywordList, term_to_idf):

print_ranked_results(successfulDocuments, term_to_idf)

# For processing free text queries.

# Simply look up each word in the

# dictionary, and print the union

# of documents from the associated

# postings lists.

def processFTQ(keywordList, term_to_idf):

print_docIDs(ranked_results_list)

# For processing boolean queries.

# The AST generated by the TA

# support code will be passed in.

# The first element of the AST is

# always the operation. The subsequent

# items may be stings (single word

# base case) or tuples, which require

# recursive evaluation.

def processBQ_private(boolean_ast):

return results

def processBQ(boolean_ast, term_to_idf):

results = processBQ_private(boolean_ast)

print_ranked_results(results, term_to_idf)

def terms_matching_wildcard_query(possible_terms, query):

return matches

# For processing single word wildcard queries.

def processWQ(query, term_to_idf):

print_docIDs(wildcard_ranked_results(matchingDocIDs, queryTermToDocIDToWildcardWeight, term_to_idf))

wildcardPhraseResults = set([])

def removeOperatorsFromString(string, operators):

for op in operators:

string = string.replace(op, "")

return string

def stemmedQuery(query):

return outputStream

def calculate_idf_for_term(term):

return 0

def processQuery(query):

return

# Main run loop.

# Read input from stdin and process

# each incoming query.

while True:

try:

query = raw_input()

processQuery(query.rstrip('\n'))

except EOFError:

break