Python WebPageDownloader.WebPageDownloader Examples

Example #1

File: proxyloader.py Project: yinonbaron/aminer-spider

	def __init__(self, manager_instance, load_from_web=False):
		self.settings = Settings.getInstance()
		self.proxyResource = ProxyResource()
		self.manager = manager_instance
		self.html_getter = WebPageDownloader();
		self.filename = os.path.join(self.settings.resourcedir, "proxies.txt")
		self.filename_static = os.path.join(self.settings.resourcedir, "proxies_static.txt")

		self.autosave_interval = (5, 12 * 5) # (seconds * check times)
		self.autosave_checkcount = 0

		# load first
		if os.path.exists(self.filename) and not load_from_web:
			self.loadFromFile()

		if len(self.manager.proxies) < 10:
			print "Load too less proxies from file."
			self.loadProxyFromWeb()
			self.saveToFile()

Example #2

File: proxyloader.py Project: yinonbaron/aminer-spider

	def __init__(self):
		self.settings = Settings.getInstance()
		self.html_getter = WebPageDownloader();