def muti_control():
threads = 5#线程的数目
datetime_info = get_data_from_datetime(threads)#按照线程数目从数据库里读出需要爬的数据
pool = ThreadPool(threads)
cookies = read_cookies_from_local()
#转成列表形式，并加入cookie信息
datetime_info = [list(t) for t in datetime_info]
for i in datetime_info:
i.append(cookies)
result = pool.map(spider, datetime_info)
renew_cookies = 0#是否更新cookies的标志位，默认是0，即不更新
for r in result:
print('id is {}'.format(r[0]))
#对返回的数据，如果status是1的话就写入数据库，否则不动，并且把cookie更新标志位置为1
if r[1] == 1:
print('content ok')
write_db(r[0], r[2])
elif r[1] == 2:
print('content no ok')
renew_cookies = 1
if renew_cookies:
get_cookies('keen123', 'keen123')
def control():
cookies = read_cookies_from_local()
t = get_data_from_datetime()
datetime = t[1]
start = t[-2]
totalCount = t[2]
print(t)
content, status = spider(datetime, start, totalCount, cookies)
if status == 1:
print('go to db')
write_db(t[0], content)
elif status == 2:
print('cookies too old')
get_cookies('keen123', 'keen123')
def setUp(self):
self.base_url1 = "http://qa.eipsev.com/user/GetCurrentUserInfo" #个人中心
self.base_url2 = "http://qa.eipsev.com/UserMessage/GetUserMessage" #首页消息
self.cookie = get_cookie.get_cookies()
def setUp(self):
self.base_url = "http://qa.eipsev.com//ApplicationForPay/SubmitApplicationForPay"
self.cookie = get_cookie.get_cookies()