Example #1
0
	def getScore(self, keyWords, avoids):
		lt = ListTool()
		lWords = lt.getNonUniques(keyWords, self.link)
		tWords = lt.getNonUniques(keyWords, self.title)
		hWords = lt.getNonUniques(keyWords, self.header)
		sWords = lt.getNonUniques(keyWords, self.spec)
		nWords = lt.getNonUniques(keyWords, self.norm)

		score = len(lWords) * self.lMulti
		score += len(tWords) * self.tMulti
		score += len(hWords) * self.hMulti
		score += len(sWords) * self.sMulti
		score += len(nWords) * self.nMulti

		lWords = lt.getNonUniques(avoids, self.link)
		tWords = lt.getNonUniques(avoids, self.title)
		hWords = lt.getNonUniques(avoids, self.header)
		sWords = lt.getNonUniques(avoids, self.spec)
		nWords = lt.getNonUniques(avoids, self.norm)

		score -= len(lWords) * self.lMulti
		score -= len(tWords) * self.tMulti
		score -= len(hWords) * self.hMulti
		score -= len(sWords) * self.sMulti
		score -= len(nWords) * self.nMulti
		self.score = score
class OwnThread(Thread):
	connections = []
	listTool = None
	responseUrls = []

	def __init__(self, connections):
		Thread.__init__(self)
		self.connections = connections
		self.listTool = ListTool()

	def run(self):
		for c in self.connections:
			c.startConnection()

		self.messengerLinks()
		self.messengerConfirm()

		for c in self.connections:
			c.waitForChild()

	def messengerLinks(self):
		numConns = len(self.connections)
		messages = 0
		while (messages < numConns):
			for c in self.connections:
				if not c.hadMessage:
					c.hasMessage = c.pollMessage(1)

			for ind, c in enumerate(self.connections):
				if c.hasMessage:
					msg = c.getMessage()
					messages += 1
					self.responseUrls = self.listTool.addOnlyUniqueFromList(msg, self.responseUrls)
					c.sendMessage(self.listTool.getNonUniques(msg, self.responseUrls))

	def messengerConfirm(self):
		numConns = len(self.connections)
		messages = 0
		for c in self.connections:
			c.hadMessage = False

		while (messages < numConns):
			for c in self.connections:
				if not c.hadMessage:
					c.hasMessage = c.pollMessage(1)

			for ind, c in enumerate(self.connections):
				if c.hasMessage:
					msg = c.getMessage()
					messages += 1
					c.endResult = msg
Example #3
0
	def startSubProcesses(self):
		CM = ConnectionManager()
		lt = ListTool()
		sitesList = []
		sitesList = lt.addOnlyUniqueFromList(self.__sites.goodSites, self.__sites.badSites)

		CM.initializeConnection(	self.__associations.keywordsList, 	self.__associations.avoidsList, 
									sitesList, 	self.__conf.siteToSearchList,	 self.__conf.pagesToSearch, 
									self.__conf.searchParamsList)
		CM.startThread()
		CM.join()
		CM.parseResults()
		
		self.resultList = CM.getResults()		
class Associations():
	keywordsList = None		# 2D format [val,occ]
	avoidsList = None		# 2D format [val,occ]
	listTool = None

	def __init__(self):
		self.keywordsList = []
		self.avoidsList = []
		self.listTool = ListTool()

	def setParams(self, keywords, avoids, fKeywords, fAvoids):
		self.keywordsList = self.listTool.addOnlyUniqueFromList(keywords, fKeywords)
		self.avoidsList = self.listTool.addOnlyUniqueFromList(avoids, fAvoids)

	def addKeyword(self, word, occurance, isForced):
		self.keywordsList.append(Keywords(word, occurance, isForced))

	def addAvoids(self, word, occurance, isForced):
		self.avoidsList.append(Avoids(word, occurance, isForced))
Example #5
0
	def parseResults(self):
		lt = ListTool()
		patt = "^[a-zA-Z0-9.,+-:;!-\"']*$"
		self.link = self.link.split(' ')
		self.title = self.title.split(' ')
		for ind, item in enumerate(self.link):
			if re.match(patt, item) == None or len(item) > self.maxLen:
				self.link.pop(ind)
		self.link = lt.removeDuplicates(self.link)

		for ind, item in enumerate(self.title):
			if re.match(patt, item) == None or len(item) > self.maxLen:
				self.title.pop(ind)
		self.title = lt.removeDuplicates(self.title)

		for ind, item in enumerate(self.header):
			if re.match(patt, item) == None or len(item) > self.maxLen:
				self.header.pop(ind)
		self.header = lt.removeDuplicates(self.header)

		for ind, item in enumerate(self.spec):
			if re.match(patt, item) == None or len(item) > self.maxLen:
				self.spec.pop(ind)
		self.spec = lt.removeDuplicates(self.spec)

		for ind, item in enumerate(self.norm):
			if re.match(patt, item) == None or len(item) > self.maxLen:
				self.norm.pop(ind)
		self.norm = lt.removeDuplicates(self.norm)
Example #6
0
	def createMasterInspectionXML(self, delChildXMLs = False):
		lt = ListTool()
		os = OSTool()
		sort = Sorter()
		insp = Inspector()

		xmls = os.getFilesInDir('results/')
		xmls = lt.popByWord(xmls, self.masterInspectionPath)

		XMLInspections = insp.getInspections(xmls)				

		if len(XMLInspections) == 0:
			print('No files read.')
			exit()

		XMLInspections = sort.sortInspectionList(XMLInspections)

		xWriter = XMLWriter()
		xWriter.writeMIXML(XMLInspections, self.masterInspectionPath)

		if delChildXMLs:
			for xml in xmls:
				os.deleteFile(xml)
Example #7
0
    def sortInspectionList(self, li):
        if li == None or len(li) == 0:
            li = []
            return li

        lt = ListTool()
        newList = []
        done = False
        while not done:
            currHS = -1000000
            currIN = -1
            done = True
            for ind, obj in enumerate(li):
                if int(obj.score) > currHS:
                    currHS = int(obj.score)
                    currIN = ind
                    done = False

            if not done:
                newList.append(li[currIN])
                li = lt.pop(li, currIN)

        return newList
	def getScore(self):
		lt = ListTool()
		for data in self.xmlInsp:
			pl = PageLoader(data.fil)
			if not pl.isReadable():
				print('Abort. XMLInspections data corrupted. File not readable:', data.fil)
				return False
			pl.read()
			lWords = lt.getNonUniques(self.keyWords, pl.linkWords)
			tWords = lt.getNonUniques(self.keyWords, pl.titleWords)
			hWords = lt.getNonUniques(self.keyWords, pl.headerWords)
			sWords = lt.getNonUniques(self.keyWords, pl.specialWords)
			nWords = lt.getNonUniques(self.keyWords, pl.normalWords)
			score = len(lWords) * self.lMulti
			score += len(tWords) * self.tMulti
			score += len(hWords) * self.hMulti
			score += len(sWords) * self.sMulti
			score += len(nWords) * self.nMulti
			data.score = score
		return self.xmlInsp
	def __init__(self, connections):
		Thread.__init__(self)
		self.connections = connections
		self.listTool = ListTool()
	def __init__(self):
		self.keywordsList = []
		self.avoidsList = []
		self.listTool = ListTool()