예제 #1
0
	def run(self):
		spider_start_time = str(datetime.now()).split('.')[0]
		print spider_start_time, 'time to spider start!'
		proxy_manager = ProxyManager()
		page = get_html(BASE_URL)
		page = unicode(page,'GBK').encode('UTF-8')
		page_count = self.get_page_count(page)
		page_count_time = str(datetime.now()).split('.')[0]
		print page_count_time, 'get page count:', page_count
		default_ip = get_default_ip()
		if page_count != 0:
			last_proxy = None
			for i in xrange(1, page_count):
				page = get_html(URL_HEADER + str(i) + URL_END, last_proxy)
				proxy_list = filte(page)
				for proxy in proxy_list:
					if proxy.anonymous_type == '高匿':
						check_result = check_anonymous(proxy, default_ip)
						spider_time = str(datetime.now()).split('.')[0]
						if check_result[0]:
							proxy.delay_time = check_result[1]
							proxy.created_time = str(datetime.now()).split('.')[0]
							proxy.is_in_china = 2
							proxy_manager.add_proxy(proxy, spider_time)
							last_proxy = proxy
						else:
							pass
예제 #2
0
 def run(self):
     spider_start_time = str(datetime.now()).split('.')[0]
     print spider_start_time, 'time to spider start!'
     proxy_manager = ProxyManager()
     page = get_html(BASE_URL)
     page = unicode(page, 'GBK').encode('UTF-8')
     page_count = self.get_page_count(page)
     page_count_time = str(datetime.now()).split('.')[0]
     print page_count_time, 'get page count:', page_count
     default_ip = get_default_ip()
     if page_count != 0:
         last_proxy = None
         for i in xrange(1, page_count):
             page = get_html(URL_HEADER + str(i) + URL_END, last_proxy)
             proxy_list = filte(page)
             for proxy in proxy_list:
                 if proxy.anonymous_type == '高匿':
                     check_result = check_anonymous(proxy, default_ip)
                     spider_time = str(datetime.now()).split('.')[0]
                     if check_result[0]:
                         proxy.delay_time = check_result[1]
                         proxy.created_time = str(
                             datetime.now()).split('.')[0]
                         proxy.is_in_china = 2
                         proxy_manager.add_proxy(proxy, spider_time)
                         last_proxy = proxy
                     else:
                         pass
예제 #3
0
	def run(self):
		spider_start_time = str(datetime.now()).split('.')[0]
		print spider_start_time, 'time to spider start!'
		proxy_manager = ProxyManager()
		last_proxy = None
		for url in self.urls:
			page = get_html(url)
			page_count = self.get_page_count(page)
			page_count_time = str(datetime.now()).split('.')[0]
			print page_count_time, 'get page count:', page_count
			default_ip = get_default_ip()
			for i in xrange(1, page_count):
				page = get_html(url + str(i))
				proxy_list = filte(page)
				for proxy in proxy_list:
					if proxy.anonymous_type == '高匿':
						check_result = check_anonymous(proxy, default_ip)
						spider_time = str(datetime.now()).split('.')[0]
						if check_result[0]:
							proxy.delay_time = check_result[1]
							proxy.created_time = str(datetime.now()).split('.')[0]
							proxy.is_in_china = 0
							if url.endswith(CHINA_ANONYMOUS) or url.endswith(CHINA_NORMAL):
								proxy.is_in_china = 1
							proxy_manager.add_proxy(proxy, spider_time)
							last_proxy = proxy
						else:
							pass
예제 #4
0
 def run(self):
     spider_start_time = str(datetime.now()).split('.')[0]
     print spider_start_time, 'time to spider start!'
     proxy_manager = ProxyManager()
     last_proxy = None
     for url in self.urls:
         page = get_html(url)
         page_count = self.get_page_count(page)
         page_count_time = str(datetime.now()).split('.')[0]
         print page_count_time, 'get page count:', page_count
         default_ip = get_default_ip()
         for i in xrange(1, page_count):
             page = get_html(url + str(i))
             proxy_list = filte(page)
             for proxy in proxy_list:
                 if proxy.anonymous_type == '高匿':
                     check_result = check_anonymous(proxy, default_ip)
                     spider_time = str(datetime.now()).split('.')[0]
                     if check_result[0]:
                         proxy.delay_time = check_result[1]
                         proxy.created_time = str(
                             datetime.now()).split('.')[0]
                         proxy.is_in_china = 0
                         if url.endswith(CHINA_ANONYMOUS) or url.endswith(
                                 CHINA_NORMAL):
                             proxy.is_in_china = 1
                         proxy_manager.add_proxy(proxy, spider_time)
                         last_proxy = proxy
                     else:
                         pass
예제 #5
0
 def check_all_proxy_anonymous(self):
     default_ip = usefull_check.get_default_ip()
     check_start_time = str(datetime.now()).split('.')[0]
     print check_start_time, 'start check all proxy anonymous!'
     proxys = self.db.query(Proxy)
     for proxy in proxys:
         check_result = usefull_check.check_anonymous(proxy, default_ip)
         if not check_result[0]:
             self.del_proxy(proxy.ip, proxy.port)
             print proxy.ip, ':', proxy.port, ' is not anonymous and delete it!'
         else:
             print proxy.ip, ':', proxy.port, ' is anonymous!'
예제 #6
0
	def check_all_proxy_anonymous(self):
		default_ip = usefull_check.get_default_ip()
		check_start_time = str(datetime.now()).split('.')[0]
		print check_start_time, 'start check all proxy anonymous!'
		proxys = self.db.query(Proxy)
		for proxy in proxys:
			check_result = usefull_check.check_anonymous(proxy, default_ip)
			if not check_result[0]:
				self.del_proxy(proxy.ip, proxy.port)
				print proxy.ip, ':', proxy.port, ' is not anonymous and delete it!'
			else:
				print proxy.ip, ':', proxy.port, ' is anonymous!'