class RawProxyCheck(ProxyManager, Thread):
def __init__(self, queue, thread_name):
ProxyManager.__init__(self)
Thread.__init__(self, name=thread_name)
self.log = LogHandler('raw_proxy_check')
self.queue = queue
def run(self):
self.log.info("RawProxyCheck - {} : start".format(self.name))
self.db.changeTable(self.useful_proxy_queue)
while True:
try:
proxy_json = self.queue.get(block=False)
except Empty:
self.log.info("RawProxyCheck - {} : exit".format(self.name))
break
proxy_obj = Proxy.newProxyFromJson(proxy_json)
proxy_obj, status = checkProxyUseful(proxy_obj)
if status:
if self.db.exists(proxy_obj.proxy):
self.log.info('RawProxyCheck - {} : {} validation exists'.format(self.name,proxy_obj.proxy.ljust(20)))
else:
self.db.put(proxy_obj)
self.log.info(
'RawProxyCheck - {} : {} validation pass'.format(self.name, proxy_obj.proxy.ljust(20)))
else:
self.log.info('RawProxyCheck - {} : {} validation fail'.format(self.name, proxy_obj.proxy.ljust(20)))
self.queue.task_done()
class DoFetchProxy(ProxyManager):
""" fetch proxy"""
def __init__(self):
ProxyManager.__init__(self)
self.log = LogHandler('fetch_proxy')
def main(self):
self.log.info("start fetch proxy")
self.fetch()
self.log.info("finish fetch proxy")
class UsefulProxyCheck(ProxyManager, Thread):
def __init__(self, queue, thread_name, origin_ips):
ProxyManager.__init__(self)
Thread.__init__(self, name=thread_name)
self.queue = queue
self.origin_ips = origin_ips
self.log = LogHandler('useful_proxy_check')
def run(self):
self.log.info("UsefulProxyCheck - {} : start".format(self.name))
self.db.changeTable(self.useful_proxy_queue)
while True:
try:
proxy_str = self.queue.get(block=False)
except Empty:
self.log.info("UsefulProxyCheck - {} : exit".format(
self.name))
break
proxy_obj = Proxy.newProxyFromJson(proxy_str)
proxy_obj, status = checkProxyUseful(proxy_obj, self.origin_ips)
if status or proxy_obj.fail_count < FAIL_COUNT:
self.db.put(proxy_obj)
self.log.info(
'UsefulProxyCheck - {} : {} validation pass'.format(
self.name, proxy_obj.proxy.ljust(20)))
else:
self.log.info(
'UsefulProxyCheck - {} : {} validation fail'.format(
self.name, proxy_obj.proxy.ljust(20)))
self.db.delete(proxy_obj.proxy)
self.queue.task_done()
class SMSbox:
def __init__(self):
self.user = setting.SMSBOX_CONFIG.get('ACCOUNT') # 短信平台账号
self.password = setting.SMSBOX_CONFIG.get('PASSWORD') # 短信平台密码
self.password = md5(self.password)
self.smsapi = "http://api.smsbao.com/sms"
self.statusStr = {
'0': '短信发送成功',
'-1': '参数不全',
'-2': '服务器空间不支持,请确认支持curl或者fsocket,联系您的空间商解决或者更换空间',
'30': '密码错误',
'40': '账号不存在',
'41': '余额不足',
'42': '账户已过期',
'43': 'IP地址限制',
'50': '内容含有敏感词'
}
self.log = LogHandler()
def send_msg(self, content, receivers=[]):
"""
:param receivers: 收信人
:param subject: 主题
:param content: 内容
:return:
"""
content = setting.SMSBOX_CONFIG.get('PATTERN').format(
time=time.strftime('%Y-%m-%d %H:%M:%S'),
name='ppsteven',
message=content)
content += '(ppsteven.github.com)'
receivers = setting.SMSBOX_CONFIG.get(
'DEFAULT_RECEIVERS') if not receivers else receivers
for phone in receivers:
params = {
'u': self.user,
'p': self.password,
'm': phone,
'c': content
}
ret = requests.get(self.smsapi, params=params).text
print(ret)
if ret == '0':
self.log.info('短信发送成功')
else:
self.log.error('短信发送失败， 原因:{0}'.format(
self.statusStr.get(ret)))
class BaseMail(object):
def __init__(self, debug=False):
self.name = EMAIL_CONFIG.get('ACCOUNT')
self.pwd = EMAIL_CONFIG.get('PASSWORD')
self.sender = EMAIL_CONFIG.get('SENDER')
self.receivers = EMAIL_CONFIG.get('RECEIVERS')
self.debug = debug
self.smtpObj = smtplib.SMTP()
# 打印出交互的所有内容
if self.debug:
self.smtpObj.set_debuglevel(1)
self.log = LogHandler()
def login(self):
self.smtpObj.connect("smtp.126.com", 25)
self.smtpObj.login(self.name, self.pwd)
self.log.info('log in email account')
def send_mail(self, subject, content):
self.login()
self.message = MIMEMultipart()
self.message.attach(MIMEText(content, 'plain', 'utf-8')) # 内容，文本，编码
self.message['From'] = self.sender
self.message['To'] = ';'.join(self.receivers)
self.message['Subject'] = Header(subject, 'utf-8')
try:
self.smtpObj.sendmail(self.sender, self.receivers,
self.message.as_string())
self.log.info(
'sender: {0}, receivers, {1}, send email success'.format(
self.sender, self.receivers))
except Exception as e:
self.log.error('send mail fail!, error msg:{0}'.format(e))
# 发送完毕，关闭连接
self.smtpObj.quit()
class AsyncDoFetchProxy(ProxyManager):
""" fetch proxy"""
def __init__(self):
ProxyManager.__init__(self)
self.log = LogHandler('fetch_proxy')
async def fetch_free_proxy_list_net(self):
"""
fetch proxy into db by ProxyGetter
:return:
"""
self.db.changeTable(self.raw_proxy_queue)
proxy_set = set()
self.log.info("ProxyFetch : fetch_free_proxy_list_net start")
self.log.info("ProxyFetch - {func}: start".format(func="fetch_free_proxy_list_net"))
try:
proxy_list = await scrape_free_proxy_list_net()
if proxy_list is not None:
for proxy in proxy_list:
proxy = proxy.strip()
if not proxy or not verifyProxyFormat(proxy):
self.log.error('ProxyFetch - {func}: {proxy} illegal'.format(func="fetch_free_proxy_list_net", proxy=proxy.ljust(20)))
continue
elif proxy in proxy_set:
self.log.info('ProxyFetch - {func}: {proxy} exist'.format(func="fetch_free_proxy_list_net", proxy=proxy.ljust(20)))
continue
else:
self.log.info('ProxyFetch - {func}: {proxy} success'.format(func="fetch_free_proxy_list_net", proxy=proxy.ljust(20)))
self.db.put(Proxy(proxy, source="fetch_free_proxy_list_net"))
proxy_set.add(proxy)
except Exception as e:
self.log.error("ProxyFetch - {func}: error".format(func="fetch_free_proxy_list_net"))
self.log.error(str(e))
self.log.error(traceback.format_exc())
async def fetch_spys_one(self):
"""
fetch proxy into db by ProxyGetter
:return:
"""
proxy_set = set()
self.db.changeTable(self.raw_proxy_queue)
self.log.info("ProxyFetch : start")
self.log.info("ProxyFetch - {func}: start".format(func="fetch_spys_one"))
try:
proxy_list = await scrape_spys_one()
if proxy_list is not None:
for proxy in proxy_list:
proxy = proxy.strip()
if not proxy or not verifyProxyFormat(proxy):
self.log.error('ProxyFetch - {func}: {proxy} illegal'.format(func="fetch_spys_one", proxy=proxy.ljust(20)))
continue
elif proxy in proxy_set:
self.log.info('ProxyFetch - {func}: {proxy} exist'.format(func="fetch_spys_one", proxy=proxy.ljust(20)))
continue
else:
self.log.info('ProxyFetch - {func}: {proxy} success'.format(func="fetch_spys_one", proxy=proxy.ljust(20)))
self.db.put(Proxy(proxy, source="fetch_spys_one"))
proxy_set.add(proxy)
except Exception as e:
self.log.error("ProxyFetch - {func}: error".format(func="fetch_spys_one"))
self.log.error(str(e))
self.log.error(traceback.format_exc())
def main_free_proxy_list_new(self):
self.log.info("start fetch free_proxy_list_new proxy")
asyncio.run(self.fetch_free_proxy_list_net())
self.log.info("finish fetch free_proxy_list_new proxy")
def main_spys_one(self):
self.log.info("start fetch main_spys_one proxy")
asyncio.run(self.fetch_spys_one())
self.log.info("finish fetch main_spys_one proxy")
class WeChatWork(object):
def __init__(self):
self.access_token_dict = {}
self.log = LogHandler()
self.post_data = WECHAT_CONFIG.get('WECHET_DEFAULT_TEXT_DATA').copy()
@classmethod
def access_token_api(cls):
access_token_api = WECHAT_CONFIG.get('ACCESS_TOKEN_API')
corpid = WECHAT_CONFIG.get('CORPID')
secret = WECHAT_CONFIG.get('CORPSECRET')
return access_token_api.format(ID=corpid, SECRET=secret)
@property
def message_send_api(self):
return WECHAT_CONFIG.get('MESSAGE_SEND_API').format(
ACCESS_TOKEN=self.access_token)
@property
def access_token(self):
if self.access_token_dict and \
time.time() - self.access_token_dict.get('expire_time', 0) <= 7200:
self.log.debug('get access token from cookies')
return self.access_token_dict.get('access_token')
ret = requests.get(WeChatWork.access_token_api()).json()
if not ret.get('errcode'):
access_token = ret.get('access_token')
self.access_token_dict['access_token'] = access_token
self.access_token_dict['expire_time'] = time.time()
self.log.debug('get access token: {0}'.format(access_token))
return access_token
else:
self.log.error('get access token fail !!')
def send_msg(self, subject, content):
"""
api 格式
{
"touser" : "UserID1|UserID2|UserID3",
"toparty" : "PartyID1|PartyID2",
"totag" : "TagID1 | TagID2",
"msgtype" : "text",
"agentid" : 1,
"text" : {
"content" : "你的快递已到，请携带工卡前往邮件中心领取。\n出发前可查看<a href=\"http://work.weixin.qq.com\">邮件中心视频实况</a>，聪明避开排队。"
},
"safe":0,
"enable_id_trans": 0,
"enable_duplicate_check": 0,
"duplicate_check_interval": 1800
}
:param subject:
:param content:
:return:
"""
self.post_data['text'][
'content'] = "subject: {0} \n content: {1}".format(
subject, content)
ret = requests.post(self.message_send_api, json=self.post_data).json()
if 0 == ret.get('errcode'):
self.log.info('send wechat msg success')
else:
self.log.error(
'send wechat msg fail!! error code:{0}, error msg:{1}'.format(
ret.get('errcode'), ret.get('errmsg')))