def luck(id: int, luckname: str):
"""
开始抽奖
"""
rd = RedisQueue(luckname)
myredis = MyRedis()
winner = luckname + "_winner"
if myredis.hexists(winner, id):
return {"ret": 0, "msg": "您已经抽过了，不能再抽了"}
award = rd.get_nowait()
if award:
myredis.hset(winner, id, award)
myredis.hincrby(luckname, award, -1)
result = {
"ret": 0,
'data': {
"flag": 1,
"msg": "恭喜你中奖了",
"award": award
}
}
else:
result = {
"ret": 0,
'data': {
"flag": 0,
"msg": "奖抽完了",
}
}
return result
def creat(item: Item):
"""
通过该接口可以创建一个抽奖规则
"""
myredis = MyRedis()
rq = RedisQueue(item.luckname)
print("ok")
if rq.qsize():
return {"ret": 500, "msg": "该抽奖已经存在，请删除后重试"}
result = {"ret": 0, "item": item}
awardlist = item.award
lucklist = []
luckdict = {}
for ward in awardlist:
luckdict[ward.name] = ward.count
for i in range(ward.count):
lucklist.append(ward.name)
othercount = item.total - len(lucklist)
if othercount:
luckdict[item.other] = othercount
others = [item.other] * othercount
lucklist = lucklist + others
random.shuffle(lucklist)
print(lucklist)
for luck in lucklist:
rq.put(luck)
myredis.hmset(item.luckname, luckdict)
result = {'ret': 0, 'msg': "succses"}
return result
def __init__(self, bluetuuid="94f39d29-7d6d-437d-973b-fba39e49d4ee"):
self.usedtosendEPCQueue = RedisQueue(name='UsedEPCSendQueue',
host='127.0.0.1',
psw='123456',
db=1)
self.devID = b''
self.uuid = bluetuuid
self.thread_read_flg = False
self.thread_read = None
self.runallflg = False
self.allthread = None
def __init__(self):
self.usedtosendEPCQueue = RedisQueue(name='UsedEPCSendQueue',host='127.0.0.1',psw='123456',db=1)
self.influxdbclass=PInfluxdbClass(host='127.0.0.1', port=8086, username='******', password='******', database = 'RFIDreportdb')
self.tagstoRedis=DataObjectJSRedisClass(host='127.0.0.1',psw='123456',port=6379,indexkey='EPC')
self.devID = b''
self.thread_read_flg=False
self.thread_read=None
self.runallflg=False
self.allthread=None
self.clientID=""
self.sendEmqClient=EMQSendClass(host='192.168.1.178',port=1883,username='******',pwd='shuiJing1',client_id='')
def _run_script_background(self, command, username, region):
p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
time.sleep(0.1) # forget why i sleep here, if not sure, don't modify this
rq = RedisQueue('script_serviceDeploy_%s_%s' % (region, username))
rq.delete_queue() # todo: delete key only
stdout, stderr = p.communicate()
ret_code = p.poll()
if not ret_code:
script_result = stdout
logger.info('run command: %s success, output:\n%s' % (command, stdout))
rq.put('*finished*')
self.add_script_exec_log(username, self.script_name, self.script_path, True, script_result)
else:
script_result = stdout + stderr
logger.error('run command: %s failed, output:\n%s' % (command, script_result))
errout = [err for err in stderr.splitlines() if err]
errout = '\n'.join(errout)
rq.put('*failed*%s' % errout)
self.add_script_exec_log(username, self.script_name, self.script_path, False, script_result)
def __init__(self,
bluetuuid="94f39d29-7d6d-437d-973b-fba39e49d4ee",
name="FireFly-3288C",
port=6):
self.usedtosendEPCQueue = RedisQueue(name='UsedEPCSendQueue',
host='127.0.0.1',
psw='123456',
db=1)
self.devID = b''
self.uuid = bluetuuid
self.bluetoothname = name
self.bluetoothport = port
self.thread_read_flg = False
self.thread_read = None
self.runallflg = False
self.allthread = None
self.allerr = False
def RFIDGetTagStart():
global issysset
global e
global pool
global redisconnpool
global GetTagProsseFlg
global epcsendthreadalive
sl = 1
if not SysIniJson(): # if not SysIni():
print('SysIni Erro')
return
sleeptimes = 0
if e.RunAllStart():
pass
else:
print('RFIDGetTagStart ERR Return')
return
pool = redis.ConnectionPool(host='127.0.0.1',
password='******',
port=6379,
db=1)
# q = RedisQueue(name='RfidControl',host='127.0.0.1',psw='123456')
try:
redisconnpool = redis.Redis(connection_pool=pool)
if redisconnpool.ping():
pass
except:
e.CloseSerialCOM()
return
q = RedisQueue(name='RfidControl', pool=pool)
if not q.istrue:
e.CloseSerialCOM()
print('CloseSerialCOM 0')
return
# while True:
# msg = q.get_nowait()
# print(msg)
# if msg != None:
# if msg==b'Rfidstop':
# issysset=199
# q.put_extend('RfidControlAnswer','RfidstopOK')
# break
# elif msg==b'Rfidstart':
# issysset=0;
# q.put_extend('RfidControlAnswer','RfidstartOK')
# break
# time.sleep(2)
# while True:
# SysCommand1(q)
# time.sleep(2)
epcsendthreadalive = 1
thread_sendepc = threading.Thread(target=EPCSendToServerThread,
name='EPCSendToServerThread')
thread_sendepc.setDaemon(True)
thread_sendepc.start()
while issysset < 200:
# msg = redis_sub.parse_response()
msg = q.get_nowait()
if msg != None:
print(msg)
if msg == b'Rfidstop':
issysset = 199
q.put_extend('RfidControlAnswer', 'RfidstopOK')
elif msg == b'Rfidstart':
issysset = 0
q.put_extend('RfidControlAnswer', 'RfidstartOK')
# SysCommand1(q)
# SysCommand(q=q,flg=1)
if issysset != 199:
# EPCProsseRedis()
EPCProsseRedis1()
if issysset == 0:
sendcmd = RfidCMD.IsAppoBootLoad()
print('IsAppoBootLoad:{0},{1}'.format(sendcmd[0], sendcmd[1]))
send = e.WriteSerialCOM(sendcmd[1])
time.sleep(sl) # time.sleep(2)
continue
elif issysset == 1:
sendcmd = RfidCMD.BtoApp()
print('BtoApp:{0},{1}'.format(sendcmd[0], sendcmd[1]))
send = e.WriteSerialCOM(sendcmd[1])
time.sleep(sl) # time.sleep(2)
continue
elif issysset == 2:
sendcmd = RfidCMD.GetDNum()
# sendcmd=RfidCMD.ApptoB()
print('GetDNum:{0},{1}'.format(sendcmd[0], sendcmd[1]))
send = e.WriteSerialCOM(sendcmd[1])
time.sleep(sl) # time.sleep(2)
continue
elif issysset == 6:
GetTagProsseFlg += 1
if GetTagProsseFlg < int(sysdict['ts']):
time.sleep(sl)
continue
sendcmd = RfidCMD.GetTagMultiple()
GetTagProsseFlg = 0
print('GetTagMultiple:{0},{1}'.format(sendcmd[0], sendcmd[1]))
send = e.WriteSerialCOM(sendcmd[1])
sleeptimes = 0
issysset = 106
time.sleep(sl) # time.sleep(2)
continue
elif issysset == 7:
sendcmd = RfidCMD.GetTagMultipleCommand()
print('GetTagMultiple:{0},{1}'.format(sendcmd[0], sendcmd[1]))
send = e.WriteSerialCOM(sendcmd[1])
sleeptimes = 0
issysset = 107
time.sleep(sl) # time.sleep(2)
continue
elif issysset == 106:
sleeptimes += 1
if sleeptimes > 10:
issysset = 6
loggererro.debug('106Sleep')
time.sleep(sl) # time.sleep(2)
continue
elif issysset == 107:
sleeptimes += 1
if sleeptimes > 10:
issysset = 7
loggererro.debug('107Sleep')
time.sleep(sl) # time.sleep(2)
continue
elif issysset == 8:
sendcmd = RfidCMD.SubSetGetTagData()
print('GetTagMultiple:{0},{1}'.format(sendcmd[0], sendcmd[1]))
send = e.WriteSerialCOM(sendcmd[1])
time.sleep(sl) # time.sleep(2)
continue
elif issysset == 9:
# if usedAnt==1:
# sendcmd=RfidCMD.SetAntEnable(ant1=usedAnt,ant2=0,ant3=0,ant4=0)
# elif usedAnt==2:
# sendcmd=RfidCMD.SetAntEnable(ant1=0,ant2=usedAnt,ant3=0,ant4=0)
# elif usedAnt==3:
# sendcmd=RfidCMD.SetAntEnable(ant1=0,ant2=0,ant3=usedAnt,ant4=0)
# elif usedAnt==4:
# sendcmd=RfidCMD.SetAntEnable(ant1=0,ant2=0,ant3=0,ant4=usedAnt)
# else:
# sendcmd=RfidCMD.SetAntEnable(ant1=1,ant2=0,ant3=0,ant4=0)
sendcmd = RfidCMD.SetAntEnable(ant1=int(sysdict['a1']),
ant2=int(sysdict['a2']),
ant3=int(sysdict['a3']),
ant4=int(sysdict['a4']))
print('SetAntEnable:{0},{1}'.format(sendcmd[0], sendcmd[1]))
send = e.WriteSerialCOM(sendcmd[1])
time.sleep(sl) # time.sleep(2)
continue
elif issysset == 10:
# if usedAnt==1:
# sendcmd=RfidCMD.SetAntPower(ant1=usedAnt,ant2=0,ant3=0,ant4=0,rpower=usedPower)
# elif usedAnt==2:
# sendcmd=RfidCMD.SetAntPower(ant1=0,ant2=usedAnt,ant3=0,ant4=0,rpower=usedPower)
# elif usedAnt==3:
# sendcmd=RfidCMD.SetAntPower(ant1=0,ant2=0,ant3=usedAnt,ant4=0,rpower=usedPower)
# elif usedAnt==4:
# sendcmd=RfidCMD.SetAntPower(ant1=0,ant2=0,ant3=0,ant4=usedAnt,rpower=usedPower)
# else:
# sendcmd=RfidCMD.SetAntPower()
sendcmd = RfidCMD.SetAntPower(ant1=int(sysdict['a1']),
ant2=int(sysdict['a2']),
ant3=int(sysdict['a3']),
ant4=int(sysdict['a4']),
rpower=int(sysdict['aw']))
print('SetAntPower:{0},{1}'.format(sendcmd[0], sendcmd[1]))
send = e.WriteSerialCOM(sendcmd[1])
time.sleep(sl) # time.sleep(2)
continue
else:
time.sleep(sl) # time.sleep(2)
import RFIDSYSSqliteClass
import socket
import json
import threading
# import psycopg2
import datetime
#import Com4GModularClass
from redisqueue import RedisQueue
from bluetooth import *
import threading
usedtosendEPCQueue = RedisQueue(name='UsedEPCSendQueue',
host='127.0.0.1',
psw='123456',
db=1)
devID = b''
def GetRfidDevID():
global devID
if devID == b'':
try:
conn = redis.Redis(host='127.0.0.1',
password='******',
port=6379,
db=1)
if conn.ping():
print('conn ok')
class Spider():
base_url = 'http://weixin.sogou.com/weixin'
keyword = 'NBA'
headers = {
'Accept': 'text / html, application / xhtml + xml, application / xml;q = 0.9, image / webp, image / apng, * / *;q = 0.8',
'Accept - Encoding': 'gzip, deflate',
'Accept-Language': 'zh-CN,zh;q=0.9',
'Cache-Control': 'max-age=0',
'Connection': 'keep-alive',
'Host': 'mp.weixin.qq.com',
'Referer': 'http://weixin.sogou.com/weixin?oq=&query=NBA&_sug_type_=1&sut=0&lkt=0%2C0%2C0&s_from=input&ri=5&_sug_=n&type=2&sst0=1530794048607&page=1&ie=utf8&p=40040108&dp=1&w=01015002&dr=1',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
}
session = Session()
queue = RedisQueue()
mysql = MySQL()
def get_proxy(self):
try:
response = requests.get(PROXY_POOL_URL)
if response.status_code == 200:
print('get proxy', response.text)
return response.text
return None
except requests.ConnectionError:
return None
def parse_index(self, response):
'''
解析索引页
:return:
'''
doc = pq(response.text)
items = doc('.news-list > li > div.txt-box > h3 > a').items()
for item in items:
url = item.attr('href')
weixin_request = WeixinRequest(url=url, callback=self.parse_detail)
yield weixin_request
next = doc('#sogou_next').attr('href')
if next:
url = self.base_url + str(next)
weixin_request = WeixinRequest(url=url, callback=self.parse_index, need_proxy=False)
yield weixin_request
def parse_detail(self, response):
'''
解析详情页
:param response:
:return:
'''
doc = pq(response.text)
data = {
'title': doc('#activity-name').text(),
'content': doc('.rich_media_content').text(),
'date': doc('#publish_time').text(),
'nickname': doc('#js_profile_qrcode > div > strong').text(),
'wechat': doc('#js_profile_qrcode > div > p:nth-child(3) > span').text()
}
yield data
def schedule(self):
while not self.queue.empty():
weixin_request = self.queue.pop()
callback = weixin_request.callback
print('schedule', weixin_request.url)
response = self.request(weixin_request)
time.sleep(1)
if response and response.status_code in VALID_STATUS:
results = list(callback(response))
if results:
for result in results:
print('New result', result)
if isinstance(result, WeixinRequest):
self.queue.add(result)
if isinstance(result, dict):
self.mysql.insert('articles', result)
else:
self.error(weixin_request)
else:
self.error(weixin_request)
def error(self, weixin_request):
weixin_request.fail_time = weixin_request.fail_time + 1
print('Request failed', weixin_request.fail_time, 'times', weixin_request.url)
if weixin_request.fail_time < MAX_FAILED_TIME:
self.queue.add(weixin_request)
def request(self, weixin_request):
try:
if weixin_request.need_proxy:
proxy = self.get_proxy()
if proxy:
proxies = {
'http': 'http://' + proxy,
'https': 'http://' + proxy
}
return self.session.send(weixin_request.prepare(), timeout=weixin_request.timeout, proxies=proxies)
return self.session.send(weixin_request.prepare(), timeout=weixin_request.timeout)
except (requests.ConnectionError, requests.ReadTimeout) as e:
print(e.args)
return False
def start(self):
self.session.headers.update(self.headers)
start_url = self.base_url + '?' + urlencode({'query': self.keyword, 'type': 2})
weixin_request = WeixinRequest(url=start_url, callback=self.parse_index, need_proxy=True)
self.queue.add(weixin_request)
def run(self):
self.start()
self.schedule()
from redis import Redis
from redisqueue import RedisQueue
from easysnmp import Session, EasySNMPError
from worker import Pool
# loading .env file
load_dotenv()
# Init redis connection
memo = Redis(host=os.getenv('REDIS_IP'),
port=os.getenv('REDIS_PORT'),
db=os.getenv('REDIS_DB'))
# init redis queue
queue = RedisQueue(connection=memo, name=os.getenv('QUEUE'))
# init logging config
logging.basicConfig(filename=os.getenv('LOGGING'),
format="%(asctime)s - %(levelname)s - %(message)s",
level=getattr(logging,
os.getenv('LOG_LEVEL').upper()))
class ServiceExit(Exception):
"""
Custom exception which is used to trigger the clean exit
of all running threads and the main program.
"""
pass
def handle(self):
print(self.client_address)
sqstr = '{0}:{1}'.format(self.client_address[0],
self.client_address[1])
print(sqstr)
q = RedisQueue(name=sqstr)
#q.redisidrun=q.isrun()
#print ('running')
while True:
try:
data = self.request.recv(1024)
if (data == b''):
try:
conn = psycopg2.connect(database="materiel",
user="******",
password="******",
host="58.214.232.165",
port="5432")
cur = conn.cursor()
print('lin postgresql OK')
except:
print('InsertToPostgresql Connect Fail')
break
#ProssData(q)
while True:
if q.isrun() == 0:
break
qdata = q.get_nowait()
if qdata == None:
try:
conn.commit()
conn.close()
print('InsertToPostgresql close')
break
except:
print('InsertToPostgresql commit Fail')
break
#break
print(qdata)
if b'DEVID' in qdata:
devid, value = qdata.split(b':')
print('did:{0}'.format(devid))
lisoo = value.split(b';')
print('did:{0}-epc:{1}'.format(lisoo[0], lisoo[1]))
sqlselectstr = (
"INSERT INTO tb_epc_record (device_id,epc,report_time) VALUES ('{0}','{1}',NOW()::timestamp)"
).format(lisoo[0].decode(encoding='utf-8'),
lisoo[1].decode(encoding='utf-8'))
print(sqlselectstr)
cur.execute(sqlselectstr)
break #continue
#print (data)
q.put(data)
#if data == b'exit':
# print ('exit')
# self.server.shutdown()
# self.request.close()
# break
except Exception as e:
self.server.shutdown()
self.request.close()
break
def setUp(self):
self.rq = RedisQueue("testqueue", "testworker01", db=10)
self.r = Redis(db=10)
self.r.delete("testqueue")
self.r.delete(self.rq.workeritem)
class Spider():
base_url = 'http://weixin.sogou.com/weixin'
keyword = 'NBA'
headers = {
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
'Accept-Encoding':
'gzip, deflate',
'Accept-Language':
'zh-CN,zh;q=0.8,en;q=0.6,ja;q=0.4,zh-TW;q=0.2,mt;q=0.2',
'Cache-Control':
'max-age=0',
'Connection':
'keep-alive',
'Cookie':
'CXID=300E866F4542ED22AE87B0EAAC93DE49; SUID=DE19F93A4B238B0A5B45763600018391; ABTEST=7|1535183484|v1; IPLOC=CN4401; weixinIndexVisited=1; SUV=002D1DEC1B2F3AEE5B810A7E9992A029; sct=1; JSESSIONID=aaalrUYv_EZgp9x02tBvw; PHPSESSID=1l812rviclinat3jkguqd8h4k5; SUIR=3DE9FCC8D3D1A7E910541485D4DA05AF; SNUID=C10A1F2B30355B0E7FDD6C8430137DAF; seccodeRight=success; successCount=1|Sat, 25 Aug 2018 08:36:18 GMT',
'Host':
'weixin.sogou.com',
'Upgrade-Insecure-Requests':
'1',
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'
}
session = Session()
queue = RedisQueue()
mysql = MySQL()
def get_proxy(self):
"""
从代理池获取代理
:return:
"""
try:
response = requests.get(PROXY_POOL_URL)
if response.status_code == 200:
print('Get Proxy', response.text)
return response.text
return None
except requests.ConnectionError:
return None
def start(self):
"""
初始化工作
"""
# 全局更新Headers
self.session.headers.update(self.headers)
start_url = self.base_url + '?' + urlencode({
'query': self.keyword,
'type': 2
})
weixin_request = WeixinRequest(url=start_url,
callback=self.parse_index,
need_proxy=True)
# print(start_url) # http://weixin.sogou.com/weixin?query=NBA&type=2
# 调度第一个请求( 网页 )
self.queue.add(weixin_request)
# callback 函数
def parse_index(self, response):
"""
解析索引页
:param response: 响应
:return: 新的响应
"""
doc = pq(response.text)
# 拿到item中的title
items = doc('.news-box .news-list li .txt-box h3 a').items()
for item in items:
# title 是a 标签
url = item.attr('href')
weixin_request = WeixinRequest(url=url, callback=self.parse_detail)
yield weixin_request
next = doc('#sogou_next').attr('href')
if next:
url = self.base_url + str(next)
weixin_request = WeixinRequest(url=url,
callback=self.parse_index,
need_proxy=True)
yield weixin_request
# 请求的回调
def parse_detail(self, response):
"""
解析详情页
:param response: 响应
:return: 微信公众号文章
"""
doc = pq(response.text)
data = {
'title':
doc('.rich_media_title').text(),
'content':
doc('.rich_media_content').text(),
'date':
doc('#post-date').text(),
'nickname':
doc('#js_profile_qrcode > div > strong').text(),
'wechat':
doc('#js_profile_qrcode > div > p:nth-child(3) > span').text()
}
yield data
# 发起网络请求
def request(self, weixin_request):
"""
执行请求
:param weixin_request: 请求
:return: 响应
"""
# try:
if weixin_request.need_proxy:
proxy = self.get_proxy()
if proxy:
proxies = {
'http': 'http://' + proxy,
'https': 'https://' + proxy
}
return self.session.send(weixin_request.prepare(),
timeout=weixin_request.timeout,
allow_redirects=False,
proxies=proxies)
return self.session.send(weixin_request.prepare(),
timeout=weixin_request.timeout,
allow_redirects=False)
# except (ConnectionError, ReadTimeout) as e:
# print(e.args)
# return False
def error(self, weixin_request):
"""
错误处理
:param weixin_request: 请求
:return:
"""
weixin_request.fail_time = weixin_request.fail_time + 1
print('Request Failed', weixin_request.fail_time, 'Times',
weixin_request.url)
if weixin_request.fail_time < MAX_FAILED_TIME:
self.queue.add(weixin_request)
# 调度请求
def schedule(self):
"""
调度请求
:return:
"""
while not self.queue.empty():
weixin_request = self.queue.pop()
# callback有两种情况
callback = weixin_request.callback
print('Schedule', weixin_request.url)
# 发起网络请求（每次都会修改代理）
response = self.request(weixin_request)
print('response ===: ', response)
if response and response.status_code in VALID_STATUSES:
# https://mp.weixin.qq.com/s?src=11×tamp=1535185884&ver=1081&signature=es5CzC0VNUqdPKGjOndCHZF9*9lta4Z4SCyS9V0IzD9aHF9ruvINszh-1xqfZ4YmVqDIl9KxQacmv3KmJyXz*Aby5hVGKgaWbPt6buCTNxZeuokt7OhireVn*BJy*89Y&new=1
# 1.results 是给生成器函数：里面有很多的 weixin_request , 获取每一个item 的detail
# 2.results 是一个字典
results = list(callback(response))
if results:
for result in results:
print('New Result', type(result))
if isinstance(result, WeixinRequest):
# 把请求添加多请求队列
self.queue.add(result)
if isinstance(result, dict):
self.mysql.insert('articles', result)
else:
self.error(weixin_request)
else:
self.error(weixin_request)
# 只执行一次
break
def run(self):
"""
入口
:return:
"""
self.start()
self.schedule()
class Spider():
base_url = 'https://weixin.sogou.com/weixin'
keyword = 'Python'
headers = {
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3',
'Accept-Encoding':
'gzip, deflate, br',
'Accept-Language':
'zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7',
'Cache-Control':
'max-age=0',
'Connection':
'keep-alive',
'Cookie':
'SUV=00EE712F77F8F1285C9A141F36334105; ABTEST=0|1562225137|v1; IPLOC=CN4403; SUID=1C013C3A4A42910A000000005D1DA9F1; SUID=1C013C3A5218910A000000005D1DA9F2; weixinIndexVisited=1; sct=1; SNUID=43615C5A5F65ECB5BB7AE454600977ED; ppinf=5|1562234904|1563444504|dHJ1c3Q6MToxfGNsaWVudGlkOjQ6MjAxN3x1bmlxbmFtZTo3OnN0aWNrZXJ8Y3J0OjEwOjE1NjIyMzQ5MDR8cmVmbmljazo3OnN0aWNrZXJ8dXNlcmlkOjQ0Om85dDJsdU9oeDFXdktJNm1GLV9JSk1PZ241NHdAd2VpeGluLnNvaHUuY29tfA; pprdig=j3pXVh43ek0qmlyN738VY_0Lno7YTfzPO1-52g0fRZ3SW93xnLY_uCVNTjYNKkADx0ySyXYgvrWojM6Lb8vFNzELWfGt6IoZ-G7QiYNH83yeiXZ2j4y1fexHB--636SyKIU_My2i1UdrvvNOQnSWUC-iVWJd04vyVl0ZQCpbHpo; sgid=12-41039275-AV0d0BgzrgksahCjtjianUTM; ppmdig=1562306345000000b94d15df908fdedf755f385f9678951a; JSESSIONID=aaasgD9OOLOWpcz5edkRw',
'DNT':
'1',
'Host':
'weixin.sougou.com',
'Upgrade-Insecure-Requests':
'1',
'User-agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36',
}
session = Session()
queue = RedisQueue()
mysql = MySQL()
def get_proxy(self):
"""
从代理池中获取代理
:return:
"""
try:
response = requests.get(PROXY_POOL_URL)
if response.status_code == 200:
print('Get Proxy', response.text)
return response.text
return None
except requests.ConnectionError:
return None
def start(self):
"""
初始化工作
:return:
"""
# 全局更新Headers
self.session.headers.update(self.headers)
start_url = self.base_url + '?' + urlencode({
'query': self.keyword,
'type': 2
})
weixin_request = WeixinRequest(url=start_url,
callback=self.parse_index,
need_proxy=True)
# 调度第一个请求
self.queue.add(weixin_request)
def parse_index(self, response):
"""
解析索引页
:param response: 响应
:return: 新的响应
"""
doc = pq(response)
items = doc('.new-box .news-list li .txt-box h3 a').items()
for item in items:
url = item.attr('href')
weixin_request = WeixinRequest(url=url, callback=self.parse_detail)
yield weixin_request
next = doc('#sogou_next').attr('href')
if next:
url = self.base_url + str(next)
weixin_request = WeixinRequest(url=url,
callback=self.parse_index,
need_proxy=True)
yield weixin_request
def parse_detail(self, response):
"""
解析详情页
:param response: 响应
:return: 公众号文章
"""
doc = pq(response.text)
data = {
'title':
doc('.rich_media_title').text(),
'content':
doc('.rich_media_content').text(),
'data':
doc('#post-data').text(),
'nickname':
doc('#js_profile_qrcode > div > strong').text(),
'wechat':
doc('#js_profile_qrcode > div > p:nth-child(3) > span').text()
}
yield data
def request(self, weixin_request):
"""
执行请求
:param weixin_request: 请求
:return: 响应
"""
try:
if weixin_request.need_proxy:
propxy = self.get_proxy()
if propxy:
proxies = {
'http': 'http://' + propxy,
'https': 'https://' + propxy,
}
return self.session.send(weixin_request.prepare(),
timeout=weixin_request.timeout,
allow_redirects=False,
proxies=proxies)
return self.session.send(weixin_request.prepare(),
timeout=weixin_request.timout,
allow_redirects=False)
except (ConnectionError, ReadTimeout) as e:
print(e.args)
return False
def error(self, weixin_request):
"""
错误处理
:param weixin_request: 请求
:return:
"""
weixin_request.fail_time = weixin_request.fail_time + 1
print('Request fail', weixin_request.fail_time, 'Times',
weixin_request.url)
if weixin_request.fail_time < MAX_FAILED_TIME:
self.queue.add(weixin_request)
def schedule(self):
"""
调度请求
:return:
"""
while not self.queue.empty():
weixin_request = self.queue.pop()
callback = weixin_request.callback
print('Schedule', weixin_request.url)
response = self.request(weixin_request)
if response and response.status_code in VALID_STATUSES:
results = list(callback(response))
if results:
for result in results:
print('New Result', result)
if isinstance(result, WeixinRequest):
self.queue.add(result)
if isinstance(result, dict):
self.mysql.insert('articles', result)
else:
self.error(weixin_request)
else:
self.error(weixin_request)
def run(self):
"""
入口
:return:
"""
self.start()
self.schedule()
def delect(name: str):
rq = RedisQueue(name)
if rq.qsize:
rq.lpop(name)
return {'ret': 0, 'msg': "删除成功"}