global termsList,termData,termMapList

offset = 0

for term,data in termData.items():

Sstr=term+" "+str(data[0])+" "+str(data[1])+getStr(data[2])

termsList.append(Sstr)

lenSstr = len(Sstr)

termMapList.append(term+" "+str(offset)+" "+str(offset+lenSstr)+"\n")

for root, subFolders, files in os.walk("./indexes/tmp"):

files.sort()

linesToAdd = 0

linesinFile = len(files)

fullSize = pow(2,int(log(linesinFile,2))+1)-1

if (fullSize == linesinFile ):

linesToAdd = 0

elif (fullSize < linesinFile):

fullSize2 = pow(2,int(log(linesinFile,2))+1)-1

linesToAdd = fullSize2 - linesinFile

else:

linesToAdd = fullSize - linesinFile

print(" Line to add : "+str(linesToAdd)+" mapSize"+str(linesinFile))

while linesToAdd > 0:

Str = "_"+str(linesToAdd) #+" "+makeFixedLengthStr(0,6)+" "+makeFixedLengthStr(0,6)+"\n"

# print("Adding Line")

linesToAdd = linesToAdd-1

files.append(Str)

files.sort()

termFile = "./indexes/termsx.txt"

termMapFile = "./indexes/termsMapy.txt"

termsListFile = "./indexes/termsList.txt"

with open(termFile, "wb") as f:

f.write(b"*")

with open(termFile, "r+b") as f:

# memory-map the file, size 0 means whole file

map = mmap.mmap(f.fileno(), 0)

fr = open(termsListFile,"w")

#with open(termMapFile, "wb") as f2:

# f2.write(b"*")

with open(termMapFile, "wb") as f2:

# memory-map the file, size 0 means whole file

# map2 = mmap.mmap(f2.fileno(), 0)

byteLen = 0

for filex in files:

if (filex[0:1] != "_"):

fr.write(filex+" ")

fx = open("./indexes/tmp/"+filex, "r+b")

# memory-map the file, size 0 means whole file

map1 = mmap.mmap(fx.fileno(), 0)

map1.seek(0)

map.resize(map.size()+map1.size())

map.write(map1[0:])

#map.flush()

Str = makeFixedLengthSpace(filex,20)+" "+makeFixedLengthStr(byteLen,6)+" "+makeFixedLengthStr(byteLen+map1.size(),6)+"\n"

f2.write(Str.encode("utf-8"))

byteLen = byteLen+map1.size()

else:

Str = makeFixedLengthSpace(filex,20)+" "+makeFixedLengthStr(0,6)+" "+makeFixedLengthStr(0,6)+"\n"

f2.write(Str.encode("utf-8"))

fr.close()

#map.close()

#map1.close()

#map2.flush()

dWordsCnt = Counter(lTerms)

for term,cnt in dWordsCnt.items():

term = "./indexes/tmp/"+term

lTermProp = []

tf = cnt

sdx = " "+str(docID)+" "+str(tf)

if os.path.isfile(term):

with open(term, "a") as f:

# memory-map the file, size 0 means whole file

#map = mmap.mmap(f.fileno(), 0)

#ctfx = int(map[0:6])

#dfx = int(map[7:13])

#sctfx=makeFixedLengthStr(ctfx+ctf,6)

#sdfx =makeFixedLengthStr(dfx+df,6)

#sx = sctfx+" "+sdfx+" "

#map.seek(0)

#map.write(sx.encode("utf-8"))

#map.flush()

#fileLen = map.size()

#map.seek(fileLen-1)

#map.resize(fileLen+len(sdx))

#map.write(sdx.encode("utf-8"))

f.write(sdx)

f.close()

#map.flush()

#map.close

else:

with open(term, "w") as f:

f.write( term+" "+sdx)

f.close()

#with open(term, "r+b") as f:

# memory-map the file, size 0 means whole file

# map = mmap.mmap(f.fileno(), 0)

#sctfx=makeFixedLengthStr(ctf,6)

#sdfx =makeFixedLengthStr(df,6)

# sx = str(docID)+" "+str(tf)

# map.seek(0)

# map.resize(map.size()+len(sx))

# sx=sx.encode("utf-8")

# map.write(sx)

global T,W,B,A,N,I

lAllLists = []

if (T in dDocProps):

lAllLists.append(dDocProps[T])

if (W in dDocProps):

lAllLists.append(dDocProps[W])

#if (B in dDocProps):

# lAllLists.append(dDocProps[B])

if (A in dDocProps):

lAllLists.append(dDocProps[A])

#if (N in dDocProps):

# lAllLists.append(dDocProps[N])

lAllLines = []

for lList in lAllLists:

lAllLines.extend(lList)

lAllWords = []

for sLine in lAllLines:

lWords = sLine.split()

lAllWords.extend(lWords)

lAllWords = helperFunctions.remStopWords(lAllWords)

p = PorterStemmer()

lAllWordsStemmed = []

for word in lAllWords:

word = p.stem(word,0,len(word)-1)

lAllWordsStemmed.append(word)

#print("All words :", lAllWordsStemmed,"\n")

lUniqueWords = list(set(lAllWordsStemmed))

lenAllWords = len(lAllWordsStemmed)

lenAllWords

sRet = makeFixedLengthStr(len(lAllWordsStemmed),6)+" "+makeFixedLengthStr(len(lUniqueWords),6) #+":"+dDocProps[B][0]

sLen = str(length)

zeros=""

#print(n-len(sLen))

while (n-len(sLen)) > 0:

zeros=zeros+"0"

#print(zeros)

n=n-1

zeros = zeros+sLen

#print(zeros)

sLen = str(length)

zeros=""

#print(n-len(sLen))

while (n-len(sLen)) > 0:

zeros=zeros+" "

#print(zeros)

n=n-1

zeros = sLen+zeros

#print(zeros)

global T,W,B,A,N,I,dFileData,termData,termsList,termMapList

docFile = "./indexes/d3.txt"

docWordsFile = "./indexes/docWords.txt"

termsFile = "./indexes/terms.txt"

termsMapFile = "./indexes/termsMap.txt"

docList = []

lDocWords = []

lTerms = []

with open(docFile, "wb") as f:

f.write(b"Hello Python!\n")

with open(docFile, "r+b") as f:

# memory-map the file, size 0 means whole file

map = mmap.mmap(f.fileno(), 0)

map.resize(5000*14)

for docID,dProps in dFileData.items():

docList.append(makeFixedLengthStr(int(docID),6)+" "+getDocStuff(dProps)[0]+"\n")

map.seek(int(docID)*14)

str1 = getDocStuff(dProps)[0]+"\n"

map.write(str1.encode("utf-8"))

sAllWords = " ".join(getDocStuff(dProps)[1])

lDocWords.append(docID+":"+sAllWords+"\n")

fillTerms(docID,getDocStuff(dProps)[1])

map.close()

mergeTermFiles()

makeTermsList()

#FILE = open(docFile,"w")

FILE1 = open(docWordsFile,"w")

FILE2 = open(termsFile,"w")

FILE3 = open(termsMapFile,"w")

# Write all the lines at once:

#FILE.writelines(docList)

FILE1.writelines(lDocWords)

FILE2.writelines(termsList)

global T,W,B,A,N,I,dFileData

with open(constants.filetoIndex,encoding=constants.encoding) as inLinksFile:

dFileContents = {} # {title:S,contents:S,date:S,author:S,entryDate:S}

dFile={} # {docID:dFileContents}

previousProp=""

for line in inLinksFile:

#print(line[0:2])

if(len(line)>2):

line = line.replace("\n","")

if(line[0:2] == ".I"):

#print("Line has docID")

dFileData.update(dFile)

dFile={}

dFileContents={}

dFile[line.split()[1]]=dFileContents

#print(dFile)

previousProp=I

elif(line[0:2] == ".T"):

#print("Line has title ")

previousProp=T

elif(line[0:2] == ".W"):

#print("Line has contents")

previousProp=W

elif(line[0:2] == ".B"):

#print("Line has date of pub")

previousProp=B

elif(line[0:2] == ".A"):

#print("Line has Author")

previousProp=A

elif(line[0:2] == ".N"):

#print("Line has ENtry Date")

previousProp=N

elif(line[0:2] == ".X"):

#print("Line has reference")

previousProp=X

else :

#print()

tempX = []

if( previousProp != X):

if(previousProp in dFileContents):

tempX = dFileContents[previousProp]

#print("tempX : ",tempX)

line = re.sub('[^a-zA-Z0-9]', ' ', line)

tempX.append(line.lower())

dFileContents[previousProp]=tempX

dFileData.update(dFile)

#print(dFileData)