def push():
db = model.Express_by_MS()
item = db.get_push()
# print(item)
if item:
tiantu = model.Express_by_Tiantu()
tiantu.push_update(item)
print(datetime.datetime.now(), '推送单号信息')
else:
print(datetime.datetime.now(), "暂无需要推送的数据")
def get_target(q):
"""
:param q: 获取的记录数
:return: 随机获取 q 条快递记录，[('#快递单号','#快递公司','#最近一次查询状态'),('#快递单号','#快递公司','#最近一次查询状态')]
"""
db = model.Express_by_MS()
item = db.get_unfinished_random(q)
targets = []
for i in item:
targets.append((i[1], i[2], i[7]))
return targets
def main(i):
proxies = get_proxies()
targets = get_target(i) # 获取i个单号
if not targets:
log.critical('已无信息需要爬取,重新推送未签收数据')
db = model.Express_by_MS()
db.repeat()
time.sleep(5)
global switch
switch = False
switch = True
for i in targets:
if i[2]:
if i[2] == 'ok':
# 上一次查询失败调用付费接口
api = random.choice(api_free)
else:
api = random.choice(api_pay)
else:
# 第一次查询用免费接口
api = random.choice(api_free)
L.acquire() # 加锁
item = parsing(api, i, proxies)
db = model.Express_by_MS()
db.save_result(item)
log.info('通过【' + api.__name__ + '】抓取单号完成=>' + str(i[0]))
L.release() # 放锁
def update_config():
db = model.Express_by_MS()
result = db.loading_config()
config.query_interval = result['query_interval']
config.query_first_interval = result['query_first_interval']
config.thread_nbmber = result['thread_nbmber']
config.parse_nbmber = result['parse_nbmber']
config.query_err_interval = result['query_err_interval']
print('****本轮配置信息****')
print(
'已揽收更新间隔[{query_interval}]-未揽收更新间隔[{query_first_interval}]-查询失败更新间隔[{query_err_interval}]-开启线程[{thread_nbmber}]-每线程查询数[{parse_nbmber}]'
.format(query_interval=result['query_interval'],
query_first_interval=result['query_first_interval'],
query_err_interval=result['query_err_interval'],
thread_nbmber=result['thread_nbmber'],
parse_nbmber=result['parse_nbmber']))
def update(q):
db = model.Express_by_MS()
db.get_temp_express(q)
print(datetime.datetime.now(), "更新快递查询池")
def update_err_ord():
db = model.Express_by_MS()
db.repeat_rootless()
print(datetime.datetime.now(), '更新未有揽收状态的运单信息')
def update_rootless():
db = model.Express_by_MS()
db.repeat_err()
print(datetime.datetime.now(), '更新查询失败的运单信息')
db.save_result(item)
log.info('通过【' + api.__name__ + '】抓取单号完成=>' + str(i[0]))
L.release() # 放锁
if __name__ == '__main__':
update_temp.update(get_temp)
while switch:
db = model.Express_by_MS()
db.init_get_unfinished_random()
my_thread = []
thread_count = config.thread_nbmber # 调用 thread_count 个线程
parse = config.parse_nbmber # 一个线程解析 parse 个url
for i in range(thread_count):
t = threading.Thread(target=main, args=(parse,))
my_thread.append(t)
for i in range(thread_count):
my_thread[i].start()
time.sleep(1)