-
Notifications
You must be signed in to change notification settings - Fork 0
/
main.py
41 lines (30 loc) · 776 Bytes
/
main.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
#-*- coding:utf8 -*-
#coding=utf-8
import JD
import sys
import threading
import Proxy
reload(sys)
sys.setdefaultencoding('utf-8')
class CrawlJD(threading.Thread,JD.JD_crawl):
def __init__(self,sets=set(),num = 1):
threading.Thread.__init__(self)
JD.JD_crawl.__init__(self)
self.crawl_set = sets
self.num_of_thread = num
def run(self):
for i in self.crawl_set:
self.crawl_from_catalog(i[0],i[1])
proxy = Proxy.crawl_Proxy()
list = proxy.crawl_proxy(10)
for i in list:
print i[0] + " " + str(i[1])
'''
jd = CrawlJD()
threads = jd.create_class("JD",6)
sets = jd.seprate_set(jd.crawl_sort_page(),6)
combined =zip(threads, sets)
for thread,set in combined:
thread = CrawlJD(set)
thread.start()
'''