Esempio n. 1
0
def start_engine():
	database = Database()
	size = database.getSize()
	while size <= maxentries:
		size = database.getSize()
		print 'Rows in database: ' + str(size)
		url = database.getUnvisited()
		database.update(url)
		if url is None:
			break
		try:
			finder = LinkFinder(url)
			if finder.isOkay():
				linklist = finder.getLinks()
				print 'Links found: ' + str(len(linklist))
				for link in linklist:
					if len(link) > 255:
						linklist.remove(link)
				try:
					database.addNew(linklist)
				except UnicodeEncodeError:
					print 'Link encoding error'
			else:
				database.markBAD(url)
				print 'Marked link as bad, reason: ' + finder.reason()
			finder.close()
		except socket.error:
			print 'timeout error'
			database.markBAD(url)
		except LookupError:
			print 'lookup error'
		if die[0]:
			break
	database.close()