def __init__(self, queue, thread_name, origin_ips):
ProxyManager.__init__(self)
Thread.__init__(self, name=thread_name)
self.queue = queue
self.origin_ips = origin_ips
self.log = LogHandler('useful_proxy_check')
class RawProxyCheck(ProxyManager, Thread):
def __init__(self, queue, thread_name):
ProxyManager.__init__(self)
Thread.__init__(self, name=thread_name)
self.log = LogHandler('raw_proxy_check')
self.queue = queue
def run(self):
self.log.info("RawProxyCheck - {} : start".format(self.name))
self.db.changeTable(self.useful_proxy_queue)
while True:
try:
proxy_json = self.queue.get(block=False)
except Empty:
self.log.info("RawProxyCheck - {} : exit".format(self.name))
break
proxy_obj = Proxy.newProxyFromJson(proxy_json)
proxy_obj, status = checkProxyUseful(proxy_obj)
if status:
if self.db.exists(proxy_obj.proxy):
self.log.info('RawProxyCheck - {} : {} validation exists'.format(self.name,proxy_obj.proxy.ljust(20)))
else:
self.db.put(proxy_obj)
self.log.info(
'RawProxyCheck - {} : {} validation pass'.format(self.name, proxy_obj.proxy.ljust(20)))
else:
self.log.info('RawProxyCheck - {} : {} validation fail'.format(self.name, proxy_obj.proxy.ljust(20)))
self.queue.task_done()
class DoFetchProxy(ProxyManager):
""" fetch proxy"""
def __init__(self):
ProxyManager.__init__(self)
self.log = LogHandler('fetch_proxy')
def main(self):
self.log.info("start fetch proxy")
self.fetch()
self.log.info("finish fetch proxy")
def __init__(self, debug=False):
self.name = EMAIL_CONFIG.get('ACCOUNT')
self.pwd = EMAIL_CONFIG.get('PASSWORD')
self.sender = EMAIL_CONFIG.get('SENDER')
self.receivers = EMAIL_CONFIG.get('RECEIVERS')
self.debug = debug
self.smtpObj = smtplib.SMTP()
# 打印出交互的所有内容
if self.debug:
self.smtpObj.set_debuglevel(1)
self.log = LogHandler()
class UsefulProxyCheck(ProxyManager, Thread):
def __init__(self, queue, thread_name, origin_ips):
ProxyManager.__init__(self)
Thread.__init__(self, name=thread_name)
self.queue = queue
self.origin_ips = origin_ips
self.log = LogHandler('useful_proxy_check')
def run(self):
self.log.info("UsefulProxyCheck - {} : start".format(self.name))
self.db.changeTable(self.useful_proxy_queue)
while True:
try:
proxy_str = self.queue.get(block=False)
except Empty:
self.log.info("UsefulProxyCheck - {} : exit".format(
self.name))
break
proxy_obj = Proxy.newProxyFromJson(proxy_str)
proxy_obj, status = checkProxyUseful(proxy_obj, self.origin_ips)
if status or proxy_obj.fail_count < FAIL_COUNT:
self.db.put(proxy_obj)
self.log.info(
'UsefulProxyCheck - {} : {} validation pass'.format(
self.name, proxy_obj.proxy.ljust(20)))
else:
self.log.info(
'UsefulProxyCheck - {} : {} validation fail'.format(
self.name, proxy_obj.proxy.ljust(20)))
self.db.delete(proxy_obj.proxy)
self.queue.task_done()
def runScheduler():
proxyFetchScheduler()
rawProxyScheduler()
usefulProxyScheduler()
scheduler_log = LogHandler("scheduler_log")
scheduler = BlockingScheduler(logger=scheduler_log)
scheduler.add_job(proxyFetchScheduler,
'interval',
minutes=10,
id="proxy_fetch",
name="raw_proxy定时采集")
scheduler.add_job(rawProxyScheduler,
'interval',
minutes=5,
id="raw_proxy_check",
name="raw_proxy 定时校验")
scheduler.add_job(usefulProxyScheduler,
'interval',
minutes=1,
id="useful_proxy_check",
name="useful_proxy定时检查")
scheduler.start()
class SMSbox:
def __init__(self):
self.user = setting.SMSBOX_CONFIG.get('ACCOUNT') # 短信平台账号
self.password = setting.SMSBOX_CONFIG.get('PASSWORD') # 短信平台密码
self.password = md5(self.password)
self.smsapi = "http://api.smsbao.com/sms"
self.statusStr = {
'0': '短信发送成功',
'-1': '参数不全',
'-2': '服务器空间不支持,请确认支持curl或者fsocket,联系您的空间商解决或者更换空间',
'30': '密码错误',
'40': '账号不存在',
'41': '余额不足',
'42': '账户已过期',
'43': 'IP地址限制',
'50': '内容含有敏感词'
}
self.log = LogHandler()
def send_msg(self, content, receivers=[]):
"""
:param receivers: 收信人
:param subject: 主题
:param content: 内容
:return:
"""
content = setting.SMSBOX_CONFIG.get('PATTERN').format(
time=time.strftime('%Y-%m-%d %H:%M:%S'),
name='ppsteven',
message=content)
content += '(ppsteven.github.com)'
receivers = setting.SMSBOX_CONFIG.get(
'DEFAULT_RECEIVERS') if not receivers else receivers
for phone in receivers:
params = {
'u': self.user,
'p': self.password,
'm': phone,
'c': content
}
ret = requests.get(self.smsapi, params=params).text
print(ret)
if ret == '0':
self.log.info('短信发送成功')
else:
self.log.error('短信发送失败， 原因:{0}'.format(
self.statusStr.get(ret)))
def __init__(self):
self.user = setting.SMSBOX_CONFIG.get('ACCOUNT') # 短信平台账号
self.password = setting.SMSBOX_CONFIG.get('PASSWORD') # 短信平台密码
self.password = md5(self.password)
self.smsapi = "http://api.smsbao.com/sms"
self.statusStr = {
'0': '短信发送成功',
'-1': '参数不全',
'-2': '服务器空间不支持,请确认支持curl或者fsocket,联系您的空间商解决或者更换空间',
'30': '密码错误',
'40': '账号不存在',
'41': '余额不足',
'42': '账户已过期',
'43': 'IP地址限制',
'50': '内容含有敏感词'
}
self.log = LogHandler()
def runScheduler():
rawProxyScheduler()
usefulProxyScheduler()
scheduler_log = LogHandler("scheduler_log")
scheduler = BlockingScheduler(logger=scheduler_log, timezone="Asia/Shanghai")
scheduler.add_job(rawProxyScheduler, 'interval', minutes=5, id="raw_proxy_check", name="raw_proxy定时采集")
scheduler.add_job(usefulProxyScheduler, 'interval', minutes=1, id="useful_proxy_check", name="useful_proxy定时检查")
scheduler.start()
def tv_post_data():
"""
request 获取参数参考：
https://stackoverflow.com/questions/10434599/get-the-data-received-in-a-flask-request/16664376#16664376?newreg=11b034b8c3ed488eb4fe0b6eba4097db
1. 通过 application/json 提交数据
>>> url = 'http://127.0.0.1:5566/api/v1.0/tradingview?token=XXXX'
>>> data = {'text': 'xxx', 'subject': 'xxx', 'token': 'xxx'}
>>> request.post(url, data=data)
2. 通过 plain/text 方式提交数据
>>> url = 'http://127.0.0.1:5566/api/v1.0/tradingview?token=XXX'
>>> data = '12345'
>>> headers = {'Content-Type': 'plain/text'}
>>> request.post(url, headers=headers, data=data)
http://127.0.0.1:5566/api/v1.0/tradingview
返回结果:{'code': 0, 'msg': '成功'}, {'code': 1, 'msg': 'token err'}
:return:
"""
#
log = LogHandler()
token, text, subject = None, None, ''
json_data = request.get_json()
if json_data:
token = json_data.get('token')
text = json_data.get('text')
subject = json_data.get('subject')
token = request.values.get('token') if not token else token
if not text or 'plain/text' in request.headers.keys().get('Content-Type'):
text = request.data.decode('utf-8')
if not subject: # 没有输入主题
subject = text[:10] + '...' if len(text) > 10 else text
log.debug('token:{0}, text:{1}, subject:{2}'.format(token, text, subject))
if token not in TOKEN:
return {'code': 1, 'msg': 'token err'}, 200
MessageManger(email=False).seng_msg(subject, text)
return {'code': 0, 'msg': 'success'}, 200
def runSchedule():
getBook = GetBook()
# getBook.getCip()
# getBook.getDoubanBook()
scheduler_log = LogHandler("scheduler_log")
scheduler = BlockingScheduler(logger=scheduler_log)
# scheduler.add_job(getBook.DownCover, 'interval', minutes=1)
# scheduler.add_job(getBook.getCip, 'interval', minutes=1)
scheduler.add_job(getBook.getDoubanBook, 'interval', minutes=1)
scheduler.start()
def runScheduler():
origin_ips = get_origin_ips()
rawProxyScheduler(origin_ips)
usefulProxyScheduler(origin_ips)
scheduler_log = LogHandler("scheduler_log")
scheduler = BlockingScheduler(logger=scheduler_log)
scheduler.add_job(rawProxyScheduler,
'interval',
args=(origin_ips, ),
minutes=5,
id="raw_proxy_check",
name="raw_proxy定时采集")
scheduler.add_job(usefulProxyScheduler,
'interval',
args=(origin_ips, ),
minutes=1,
id="useful_proxy_check",
name="useful_proxy定时检查")
scheduler.start()
class BaseMail(object):
def __init__(self, debug=False):
self.name = EMAIL_CONFIG.get('ACCOUNT')
self.pwd = EMAIL_CONFIG.get('PASSWORD')
self.sender = EMAIL_CONFIG.get('SENDER')
self.receivers = EMAIL_CONFIG.get('RECEIVERS')
self.debug = debug
self.smtpObj = smtplib.SMTP()
# 打印出交互的所有内容
if self.debug:
self.smtpObj.set_debuglevel(1)
self.log = LogHandler()
def login(self):
self.smtpObj.connect("smtp.126.com", 25)
self.smtpObj.login(self.name, self.pwd)
self.log.info('log in email account')
def send_mail(self, subject, content):
self.login()
self.message = MIMEMultipart()
self.message.attach(MIMEText(content, 'plain', 'utf-8')) # 内容，文本，编码
self.message['From'] = self.sender
self.message['To'] = ';'.join(self.receivers)
self.message['Subject'] = Header(subject, 'utf-8')
try:
self.smtpObj.sendmail(self.sender, self.receivers,
self.message.as_string())
self.log.info(
'sender: {0}, receivers, {1}, send email success'.format(
self.sender, self.receivers))
except Exception as e:
self.log.error('send mail fail!, error msg:{0}'.format(e))
# 发送完毕，关闭连接
self.smtpObj.quit()
def __init__(self):
ProxyManager.__init__(self)
self.log = LogHandler('fetch_proxy')
def __init__(self):
self.access_token_dict = {}
self.log = LogHandler()
self.post_data = WECHAT_CONFIG.get('WECHET_DEFAULT_TEXT_DATA').copy()
def __init__(self, queue, thread_name):
ProxyManager.__init__(self)
Thread.__init__(self, name=thread_name)
self.log = LogHandler('raw_proxy_check')
self.queue = queue
class AsyncDoFetchProxy(ProxyManager):
""" fetch proxy"""
def __init__(self):
ProxyManager.__init__(self)
self.log = LogHandler('fetch_proxy')
async def fetch_free_proxy_list_net(self):
"""
fetch proxy into db by ProxyGetter
:return:
"""
self.db.changeTable(self.raw_proxy_queue)
proxy_set = set()
self.log.info("ProxyFetch : fetch_free_proxy_list_net start")
self.log.info("ProxyFetch - {func}: start".format(func="fetch_free_proxy_list_net"))
try:
proxy_list = await scrape_free_proxy_list_net()
if proxy_list is not None:
for proxy in proxy_list:
proxy = proxy.strip()
if not proxy or not verifyProxyFormat(proxy):
self.log.error('ProxyFetch - {func}: {proxy} illegal'.format(func="fetch_free_proxy_list_net", proxy=proxy.ljust(20)))
continue
elif proxy in proxy_set:
self.log.info('ProxyFetch - {func}: {proxy} exist'.format(func="fetch_free_proxy_list_net", proxy=proxy.ljust(20)))
continue
else:
self.log.info('ProxyFetch - {func}: {proxy} success'.format(func="fetch_free_proxy_list_net", proxy=proxy.ljust(20)))
self.db.put(Proxy(proxy, source="fetch_free_proxy_list_net"))
proxy_set.add(proxy)
except Exception as e:
self.log.error("ProxyFetch - {func}: error".format(func="fetch_free_proxy_list_net"))
self.log.error(str(e))
self.log.error(traceback.format_exc())
async def fetch_spys_one(self):
"""
fetch proxy into db by ProxyGetter
:return:
"""
proxy_set = set()
self.db.changeTable(self.raw_proxy_queue)
self.log.info("ProxyFetch : start")
self.log.info("ProxyFetch - {func}: start".format(func="fetch_spys_one"))
try:
proxy_list = await scrape_spys_one()
if proxy_list is not None:
for proxy in proxy_list:
proxy = proxy.strip()
if not proxy or not verifyProxyFormat(proxy):
self.log.error('ProxyFetch - {func}: {proxy} illegal'.format(func="fetch_spys_one", proxy=proxy.ljust(20)))
continue
elif proxy in proxy_set:
self.log.info('ProxyFetch - {func}: {proxy} exist'.format(func="fetch_spys_one", proxy=proxy.ljust(20)))
continue
else:
self.log.info('ProxyFetch - {func}: {proxy} success'.format(func="fetch_spys_one", proxy=proxy.ljust(20)))
self.db.put(Proxy(proxy, source="fetch_spys_one"))
proxy_set.add(proxy)
except Exception as e:
self.log.error("ProxyFetch - {func}: error".format(func="fetch_spys_one"))
self.log.error(str(e))
self.log.error(traceback.format_exc())
def main_free_proxy_list_new(self):
self.log.info("start fetch free_proxy_list_new proxy")
asyncio.run(self.fetch_free_proxy_list_net())
self.log.info("finish fetch free_proxy_list_new proxy")
def main_spys_one(self):
self.log.info("start fetch main_spys_one proxy")
asyncio.run(self.fetch_spys_one())
self.log.info("finish fetch main_spys_one proxy")
def runFlask():
app.run(host=config.host_ip, port=config.host_port, threaded=True)
def runFlaskWithGunicorn():
_options = {
'bind': '%s:%s' % (config.host_ip, config.host_port),
'workers': 4,
'accesslog': '-', # log to stdout
'access_log_format': '%(h)s %(l)s %(t)s "%(r)s" %(s)s "%(a)s"'
}
StandaloneApplication(app, _options).run()
api_log = LogHandler("api")
def main():
app.debug = True
# for handler in api_log.handlers:
# app.logger.addHandler(handler)
api_log.info("api start")
if platform.system() == "Windows":
runFlask()
else:
runFlaskWithGunicorn()
if __name__ == '__main__':
main()
class WeChatWork(object):
def __init__(self):
self.access_token_dict = {}
self.log = LogHandler()
self.post_data = WECHAT_CONFIG.get('WECHET_DEFAULT_TEXT_DATA').copy()
@classmethod
def access_token_api(cls):
access_token_api = WECHAT_CONFIG.get('ACCESS_TOKEN_API')
corpid = WECHAT_CONFIG.get('CORPID')
secret = WECHAT_CONFIG.get('CORPSECRET')
return access_token_api.format(ID=corpid, SECRET=secret)
@property
def message_send_api(self):
return WECHAT_CONFIG.get('MESSAGE_SEND_API').format(
ACCESS_TOKEN=self.access_token)
@property
def access_token(self):
if self.access_token_dict and \
time.time() - self.access_token_dict.get('expire_time', 0) <= 7200:
self.log.debug('get access token from cookies')
return self.access_token_dict.get('access_token')
ret = requests.get(WeChatWork.access_token_api()).json()
if not ret.get('errcode'):
access_token = ret.get('access_token')
self.access_token_dict['access_token'] = access_token
self.access_token_dict['expire_time'] = time.time()
self.log.debug('get access token: {0}'.format(access_token))
return access_token
else:
self.log.error('get access token fail !!')
def send_msg(self, subject, content):
"""
api 格式
{
"touser" : "UserID1|UserID2|UserID3",
"toparty" : "PartyID1|PartyID2",
"totag" : "TagID1 | TagID2",
"msgtype" : "text",
"agentid" : 1,
"text" : {
"content" : "你的快递已到，请携带工卡前往邮件中心领取。\n出发前可查看<a href=\"http://work.weixin.qq.com\">邮件中心视频实况</a>，聪明避开排队。"
},
"safe":0,
"enable_id_trans": 0,
"enable_duplicate_check": 0,
"duplicate_check_interval": 1800
}
:param subject:
:param content:
:return:
"""
self.post_data['text'][
'content'] = "subject: {0} \n content: {1}".format(
subject, content)
ret = requests.post(self.message_send_api, json=self.post_data).json()
if 0 == ret.get('errcode'):
self.log.info('send wechat msg success')
else:
self.log.error(
'send wechat msg fail!! error code:{0}, error msg:{1}'.format(
ret.get('errcode'), ret.get('errmsg')))