def my_send_msg(num):
# 从sample.cfg中读取基本配置信息
## WARNING： Please do not hard code your accessId and accesskey in next line.(more information: https://yq.aliyun.com/articles/55947)
accid, acckey, endpoint, token = MNSSampleCommon.LoadConfig()
# 初始化 my_account, my_queue
my_account = Account(endpoint, accid, acckey, token)
queue_name = "MyQueue1"
my_queue = my_account.get_queue(queue_name)
try:
# msg_body = "I am test message %s." % i
msg_body = num
msg = Message(msg_body)
re_msg = my_queue.send_message(msg)
# print
# "Send Message Succeed! MessageBody:%s MessageID:%s" % (msg_body, re_msg.message_id)
except MNSExceptionBase, e:
if e.type == "QueueNotExist":
print
"Queue not exist, please create queue before send message."
sys.exit(0)
print
"Send Message Fail! Exception:%s\n" % e
def run_produce():
account = Account(MNS_ENDPOINT, ALI_ACCESSKEYID, ALI_ACCESSKEYSECRET)
q = account.get_queue(QUEUE_NAME)
queue_meta = QueueMeta()
queue_meta.set_visibilitytimeout(100)
queue_meta.set_maximum_message_size(10240)
queue_meta.set_message_retention_period(3600)
queue_meta.set_delay_seconds(10)
queue_meta.set_polling_wait_seconds(20)
queue_meta.set_logging_enabled(True)
try:
q_url = q.create(queue_meta)
sys.stdout.write("Create Queue Succeed!\nQueueURL:%s\n\n" % q_url)
except MNSExceptionBase as e:
sys.stderr.write("Create Queue Fail!\nException:%s\n\n" % e)
sys.exit(1)
for i in range(1, 10):
msg_body = "Test Message n. {} at {}".format(
i,
datetime.now().isoformat())
message = Message(msg_body)
message.set_delayseconds(0)
message.set_priority(10)
try:
send_msg = q.send_message(message)
sys.stdout.write(
"Send Message Succeed.\nMessageBody:%s\nMessageId:%s\nMessageBodyMd5:%s\n\n"
% (msg_body, send_msg.message_id, send_msg.message_body_md5))
except MNSExceptionBase as e:
sys.stderr.write("Send Message Fail!\nException:%s\n\n" % e)
sys.exit(1)
def run_consume():
account = Account(MNS_ENDPOINT, ALI_ACCESSKEYID, ALI_ACCESSKEYSECRET)
q = account.get_queue(QUEUE_NAME)
while True:
try:
wait_seconds = 10
recv_msg = q.receive_message(wait_seconds)
sys.stdout.write(
"Receive Message Succeed! \
\nMessageId: %s\nMessageBodyMD5: %s \
\nMessageBody: %s\nDequeueCount: %s \
\nEnqueueTime: %s\nFirstDequeueTime: %s \
\nPriority: %s\nNextVisibleTime: %s \
\nReceiptHandle: %s\n\n" %
(recv_msg.message_id, recv_msg.message_body_md5,
recv_msg.message_body, recv_msg.dequeue_count,
recv_msg.enqueue_time, recv_msg.first_dequeue_time,
recv_msg.priority, recv_msg.next_visible_time,
recv_msg.receipt_handle))
q.delete_message(recv_msg.receipt_handle)
sys.stdout.write("Delete Message Succeed.\n\n")
except MNSExceptionBase as e:
sys.stderr.write("Receive Message Fail!\nException:%s\n\n" % e)
break
try:
q.delete()
sys.stdout.write("Delete Queue Succeed!\n\n")
except MNSExceptionBase as e:
sys.stderr.write("Delete Queue Fail!\nException:%s\n\n" % e)
sys.exit(1)
class aliyun_mns_api(object):
def __init__(self):
"""客户端初始化"""
cf = ConfigParser.ConfigParser()
cf.read("aliyun.conf")
host = cf.get('AccessKey_mns',"Host")
accessid = cf.get('AccessKey_mns',"Access_ID")
accesskey = cf.get('AccessKey_mns',"Access_Key")
self.cli = Account(host,accessid,accesskey)
def sendmessagerequest(self,queue_name):
"""向指定mns发送message"""
with open('SendMessage.txt','r') as f:
for line in f:
message = Message(line)
re_msg = self.cli.get_queue(queue_name).send_message(message)
print "Send Message Succeed! MessageBody:%s MessageID:%s" % (line,re_msg.message_id)
def getlistqueuenames(self):
"""查询消息列队名字"""
result = self.cli.list_queue()
name_list = []
for name in result[0]:
real_name = str(name).split('/')[-1]
name_list.append(real_name)
#print name_list
return name_list
def __init__(self):
# 读取邮箱信息
self.db = Database(
'mysql+mysqldb://spider_admin:'
'[email protected]:'
'3306/spider?charset=utf8',
encoding='utf-8')
# 初始化数据库
self.db1 = Database(
'mysql+mysqldb://%s:%s@%s:%s/%s?charset=utf8' %
(MYSQL_USER, MYSQL_PASSWD, MYSQL_HOST, MYSQL_PORT, MYSQL_DB),
encoding='utf-8')
# 初始化MNS
self.mns_account = Account(MNS_ENDPOINT, MNS_ACCID, MNS_ACCKEY,
MNS_TOKEN)
self.mns_client = self.mns_account.get_queue(MNS_QUEUE)
self.logger = Logger.timed_rt_logger()
self.queue = RedisQueue('t_mail_by_date',
host=REDIS_HOST,
port=REDIS_PORT)
self.robot = DingDingRobot(
access_token="eb749abfe9080a69da6524b77f589b8f6ddbc"
"c182c7a41bf095b095336edb0a1")
def __init__(self):
self.id = conf.access_key_id
self.secret = conf.access_key_secret
self.point = conf.queue_endpoint
self.queue_name = conf.queue_name
self.auth = oss2.Auth(conf.access_key_id, conf.access_key_secret)
self.bucket = oss2.Bucket(self.auth, conf.oss_endpoint,
conf.bucket_name)
self.oss_img_path = conf.oss_img_path
client = MongoClient(conf.mongo_host)
client['admin'].authenticate('root', 'qwer1234')
self.userdb = client['qp_account']
for index, subject in enumerate(_ALL_SUPPORT_SUBJECTS):
db_name = 'qp_' + subject
self.db[index] = client[db_name]
logging.critical('connect mongodb success:{}'.format(conf.mongo_host))
if self.id is None or self.secret is None or self.point is None or self.queue_name is None:
raise Exception('lose conf')
try:
self.account = Account(self.point, self.id, self.secret)
self.queue = self.account.get_queue(self.queue_name)
logging.critical('init_MNS success:')
logging.critical(u'id:{}, point:{}, queue_name'.format(
self.id, self.point, self.queue_name))
except:
traceback.print_exc()
raise Exception('connect MNS fail')
def rece_del_msg(queue_name, wait_seconds=3):
"""
从指定的消息队列中获取消息，默认等待3s，每次获取1个
:param queue_name: 消息队列的名称
:param wait_seconds: 等待时长
:return: (None, ""): 队列不存在；(False, ""): 获取失败；(True, "***"): 获取成功
"""
access_key_id, access_key_secret, end_point, security_token = config_parse()
my_account = Account(end_point, access_key_id, access_key_secret, security_token)
my_queue = my_account.get_queue(queue_name)
try:
receive_message = my_queue.receive_message(wait_seconds)
except MNSExceptionBase as e:
if e.type == "QueueNotExist":
result = (None, "")
elif e.type == "MessageNotExist":
result = (True, "")
else:
result = (False, "")
else:
result = (True, receive_message.message_body)
try:
my_queue.delete_message(receive_message.receipt_handle)
except Exception as e:
result = (False, "")
return result
def __init__(self):
self.set_logger()
#加载阿里云相关配置
self.mns_endpoint = get_config("MNS", "mns_endpoint")
self.mns_access_id = get_config("MNS", "mns_access_id")
self.mns_access_key = get_config("MNS", "mns_access_key")
self.mns_queue_name = get_config("MNS", "mns_queue_name")
self.oss_endpoint = get_config("OSS", "oss_endpoint")
self.oss_access_id = get_config("OSS", "oss_access_id")
self.oss_access_key = get_config("OSS", "oss_access_key")
self.oss_bucket_name = get_config("OSS", "oss_bucket_name")
#加载rtmp相关配置
self.max_stream_count = int(get_config("Base", "max_stream_count"))
self.max_retries = 3
#初始化 msn account, queue
self.mns_account = Account(self.mns_endpoint, self.mns_access_id,
self.mns_access_key)
self.mns_queue = self.mns_account.get_queue(self.mns_queue_name)
self.mns_queue.set_encoding(False)
self.publish_count = 0
self.publish_mutex = threading.Lock()
self.bucket = oss2.Bucket(
oss2.Auth(self.oss_access_id, self.oss_access_key),
self.oss_endpoint, self.oss_bucket_name)
def insert_db(self, item):
try:
self.sqlalchemy_handle.session.execute(
spider_search.ResumeRaw.__table__.insert(), item
)
self.sqlalchemy_handle.session.commit()
except Exception as e:
self.sqlalchemy_handle.session.rollback()
self.logger.exception("入库失败, 回滚： %s" % e)
raise e
sql_id = self.sqlalchemy_handle.session.execute(
"select last_insert_id()").first()[0]
self.logger.info("入库成功. %s" % str(sql_id))
try:
resume_uuid = uuid.uuid1()
mns_data = {
"channelType": "WEB",
"content": {
"content": item['content'],
"id": sql_id,
"createBy": "python",
"createTime": int(time.time() * 1000),
"ip": '',
'emailJobType': item['emailJobType'],
'emailCity': item['emailCity'],
'deliverJobName': '',
'deliverJobCity': '',
"resumeSubmitTime": '',
"resumeUpdateTime": '',
"source": item['source'],
"trackId": str(resume_uuid),
"avatarUrl": '',
},
"interfaceType": "PARSE",
"resourceDataType": "RAW",
"resourceType": "RESUME_SEARCH",
"source": item['source'],
"trackId": resume_uuid,
"traceID": resume_uuid,
"callSystemID": 'five_one_python3_scrapy'
}
dumps = json.dumps(mns_data, ensure_ascii=False)
buf = StringIO()
f = gzip.GzipFile(mode='wb', fileobj=buf)
f.write(dumps)
f.close()
msg_body = base64.b64encode(buf.getvalue())
msg = Message(msg_body)
# 初始化MNS
mns_account = Account(MNS_ENDPOINT, MNS_ACCID, MNS_ACCKEY,
MNS_TOKEN)
mns_client = mns_account.get_queue(MNS_QUEUE)
mns_client.send_message(msg)
self.logger.info("推送mns成功. %s" % str(sql_id))
except Exception as e:
self.logger.exception("推送mns失败: %s" % e)
def __init__(self):
"""客户端初始化"""
cf = ConfigParser.ConfigParser()
cf.read("aliyun.conf")
host = cf.get('AccessKey_mns',"Host")
accessid = cf.get('AccessKey_mns',"Access_ID")
accesskey = cf.get('AccessKey_mns',"Access_Key")
self.cli = Account(host,accessid,accesskey)
def __init__(self):
self.client = AcsClient(app.config.get("ALI_ACCESS_KEY"),
app.config.get("ALI_ACCESS_SECRET"),
app.config.get("BASE_REGION"))
self.account = Account(app.config.get("MNS_ENDPOINT"),
app.config.get("ALI_ACCESS_KEY"),
app.config.get("ALI_ACCESS_SECRET"), "")
self.wait_seconds = 10
self.kube = Kubernetes()
def token_and_queue_refresh(self):
""" Token过期刷新 """
if not self.token.need_refresh:
return
self.token.refresh()
if self.account:
self.account.mns_client.close_connection()
self.account = Account(endpoint, self.token.tmp_access_id, self.token.tmp_access_key, self.token.token)
self.queue = self.account.get_queue(self.queue_cfg.queue_name)
logger.info(f'{self.token.tk_desc} account and queue ready')
def get_producer(self, settings):
if self._mns_producer:
return self._mns_producer
account = Account(
host=settings['MNS_ACCOUNT_ENDPOINT'],
access_id=settings['MNS_ACCESSKEY_ID'],
access_key=settings['MNS_ACCESSKEY_SECRET'],
logger=logging.getLogger('evascrapy.pipelines.AliyunMnsPipeline'),
debug=False,
)
self._mns_producer = account.get_queue(settings['MNS_QUEUE_NAME'])
return self._mns_producer
def send_message(tel="18611846371", module="hadoop", message="error"):
# 从https://account.console.aliyun.com/#/secure获取$YourAccountid
# 从https://ak-console.aliyun.com/#/accesskey获取$YourAccessId和$YourAccessKey
# 从http://$YourAccountId.mns.cn-hangzhou.aliyuncs.com获取$YourMNSEndpoint, eg. http://1234567890123456.mns.cn-hangzhou.aliyuncs.com
YourMNSEndpoint = "http://1111641897067585.mns.cn-hangzhou-internal.aliyuncs.com/"
YourAccessId = "LTAIyhZeIPt8ivJd"
YourAccessKey = "myvAcplJMk32PHIt2DSmWhRqcz6CHv"
my_account = Account(YourMNSEndpoint, YourAccessId, YourAccessKey)
YourTopicName = "sms.topic-cn-hangzhou"
my_topic = my_account.get_topic(YourTopicName)
#print my_topic
'''
Step 2. 设置SMS消息体（必须）
注：目前暂时不支持消息内容为空，需要指定消息内容，不为空即可。
'''
msg_body1 = "报警短信消息"
'''
Step 3. 生成SMS消息属性，single=False表示每个接收者参数不一样，
single=True表示每个接收者参数一样
'''
# 3.1 设置SMSSignName和SMSTempateCode
YourSignName = "app运营助手"
YourSMSTemplateCode = "SMS_32785106"
direct_sms_attr1 = DirectSMSInfo(free_sign_name=YourSignName,
template_code=YourSMSTemplateCode,
single=True)
# 3.2 指定接收短信的手机号并指定发送给该接收人的短信中的参数值（在短信模板中定义的）
cur_time = time.strftime('%H:%M:%S', time.localtime()) #变量内容不能太长
direct_sms_attr1.add_receiver(receiver=tel)
direct_sms_attr1.set_params({
"machine": "短视频",
"module": module,
"time": cur_time,
"message": message
})
#Step 5. 生成SMS消息对象
msg1 = TopicMessage(msg_body1, direct_sms=direct_sms_attr1)
re_msg = my_topic.publish_message(msg1)
#print "Publish Message Succeed. MessageBody:%s MessageID:%s" % (msg_body1, re_msg.message_id)
return 0
def init_from_cfg(self):
parser = ConfigParser.ConfigParser()
parser.read(self.cfg_file)
for sec,op in required_ops:
if not parser.has_option(sec, op):
raise Exception("mnsclient load config file failed")
self.mns_account = Account(parser.get(BASE, END_POINT), parser.get(BASE, ACCESS_ID), parser.get(BASE, ACCESS_KEY_SECRET))
self.mns_queue = self.mns_account.get_queue(parser.get(BASE, QUEUE_NAME))
self.wait_seconds = int(parser.get(BASE, WAIT_SECOND))
def create_queue(self, broker):
"""
根据broker类型，创建相应的queue对象
"""
if broker == 'redis':
#创建redis queue
from util import redis_queue
queue = redis_queue.get_queue(settings.SUBSCRIBE_QUEUE_NAME)
elif broker == 'mns':
#创建aliyun mns queue
self.mns_account = Account(\
settings.MNS_ENDPOINT, \
settings.MNS_ACCESS_KEY_ID, \
settings.MNS_ACCESS_KEY_SECRET, \
settings.MNS_SECURITY_TOKEN)
queue = self.mns_account.get_queue(settings.SUBSCRIBE_QUEUE_NAME)
else:
queue = None
return queue
def aliyunMsg(user, *args):
code, product = args
my_account = Account(Config.MNS_ENDPOINT, Config.MNS_ACCESSID,
Config.MNS_ACCESSKEY)
my_topic = my_account.get_topic(Config.MNS_TOPIC)
msg_body1 = "sms-message1."
direct_sms_attr1 = DirectSMSInfo(free_sign_name=Config.MNS_SIGN,
template_code=Config.MNS_TEMPLATE_CODE,
single=True)
direct_sms_attr1.add_receiver(receiver=user)
direct_sms_attr1.set_params({"code": code, "product": product})
msg1 = TopicMessage(msg_body1, direct_sms=direct_sms_attr1)
try:
re_msg = my_topic.publish_message(msg1)
print "Publish Message Succeed. MessageBody:%s MessageID:%s" % (
msg_body1, re_msg.message_id)
except MNSExceptionBase, e:
if e.type == "TopicNotExist":
print "Topic not exist, please create it."
sys.exit(1)
print "Publish Message Fail. Exception:%s" % e
def send_msg(queue_name, message_body):
"""
向指定消息队列发送消息
:param queue_name: 消息队列的名称
:param message_body: 需要发送的消息
:return: True：发送成功；False：发送失败；None：该消息队列不存在
"""
access_key_id, access_key_secret, end_point, security_token = config_parse()
my_account = Account(end_point, access_key_id, access_key_secret, security_token)
my_queue = my_account.get_queue(queue_name)
try:
msg = Message(str(message_body))
re_msg = my_queue.send_message(msg)
# print re_msg.message_id
except MNSExceptionBase as e:
if e.type == "QueueNotExist":
return None
else:
return False
else:
return True
def my_recv_msg():
#从sample.cfg中读取基本配置信息
## WARNING： Please do not hard code your accessId and accesskey in next line.(more information: https://yq.aliyun.com/articles/55947)
accid,acckey,endpoint,token = MNSSampleCommon.LoadConfig()
#初始化 my_account, my_queue
my_account = Account(endpoint, accid, acckey, token)
queue_name = "MyQueue1"
base64 = False if len(sys.argv) > 2 and sys.argv[2].lower() == "false" else True
my_queue = my_account.get_queue(queue_name)
my_queue.set_encoding(base64)
#循环读取删除消息直到队列空
#receive message请求使用long polling方式，通过wait_seconds指定长轮询时间为3秒
## long polling 解析:
### 当队列中有消息时，请求立即返回；
### 当队列中没有消息时，请求在MNS服务器端挂3秒钟，在这期间，有消息写入队列，请求会立即返回消息，3秒后，请求返回队列没有消息；
wait_seconds = 3
# print "%sReceive And Delete Message From Queue%s\nQueueName:%s\nWaitSeconds:%s\n" % (10*"=", 10*"=", queue_name, wait_seconds)
#读取消息
try:
recv_msg = my_queue.receive_message(wait_seconds)
# print "Receive Message Succeed! ReceiptHandle:%s MessageBody:%s MessageID:%s" % (recv_msg.receipt_handle, recv_msg.message_body, recv_msg.message_id)
except MNSExceptionBase, e:
if e.type == "QueueNotExist":
print
"Queue not exist, please create queue before receive message."
sys.exit(0)
elif e.type == "MessageNotExist":
print
"Queue is empty!"
sys.exit(0)
print
"Receive Message Fail! Exception:%s\n" % e
continue
class MNSReader(object):
id = None
secret = None
point = None
message = None
queue_name = None
db = dict()
def __init__(self):
self.id = conf.access_key_id
self.secret = conf.access_key_secret
self.point = conf.queue_endpoint
self.queue_name = conf.queue_name
self.auth = oss2.Auth(conf.access_key_id, conf.access_key_secret)
self.bucket = oss2.Bucket(self.auth, conf.oss_endpoint,
conf.bucket_name)
self.oss_img_path = conf.oss_img_path
client = MongoClient(conf.mongo_host)
client['admin'].authenticate('root', 'qwer1234')
self.userdb = client['qp_account']
for index, subject in enumerate(_ALL_SUPPORT_SUBJECTS):
db_name = 'qp_' + subject
self.db[index] = client[db_name]
logging.critical('connect mongodb success:{}'.format(conf.mongo_host))
if self.id is None or self.secret is None or self.point is None or self.queue_name is None:
raise Exception('lose conf')
try:
self.account = Account(self.point, self.id, self.secret)
self.queue = self.account.get_queue(self.queue_name)
logging.critical('init_MNS success:')
logging.critical(u'id:{}, point:{}, queue_name'.format(
self.id, self.point, self.queue_name))
except:
traceback.print_exc()
raise Exception('connect MNS fail')
def pop(self):
retry = 2
while retry > 0:
try:
self.queue.delete_message(self.message.receipt_handle)
except MNSExceptionBase, e:
logging.fatal('release message fail: ' + str(e))
retry -= 1
time.sleep(1)
else:
logging.info('release message success')
break
self.message = None
class MnsClient():
"""mns client
"""
def __init__(self, cfg_file):
self.cfg_file = cfg_file
self.init_from_cfg()
self.last_recv_msg = None
def init_from_cfg(self):
parser = ConfigParser.ConfigParser()
parser.read(self.cfg_file)
for sec,op in required_ops:
if not parser.has_option(sec, op):
raise Exception("mnsclient load config file failed")
self.mns_account = Account(parser.get(BASE, END_POINT), parser.get(BASE, ACCESS_ID), parser.get(BASE, ACCESS_KEY_SECRET))
self.mns_queue = self.mns_account.get_queue(parser.get(BASE, QUEUE_NAME))
self.wait_seconds = int(parser.get(BASE, WAIT_SECOND))
def recv_msg(self, visible_timeout=60):
"""
@return: None means recv msg failed, otherwise, return the msg_body(means the msg that you send)
"""
ret_flag = None
try:
_msg = self.mns_queue.receive_message(self.wait_seconds)
try:
change_msg_vis = self.mns_queue.change_message_visibility(_msg.receipt_handle, visible_timeout)
logger.debug("Change Message Visibility Succeed!\nReceiptHandle:%s\nNextVisibleTime:%s\n\n" %
(change_msg_vis.receipt_handle, change_msg_vis.next_visible_time))
ret_flag = _msg.message_body
self.last_recv_msg = change_msg_vis
except MNSExceptionBase, e:
logger.warn("Change Message Visibility Fail!\nException:%s\n\n" % e)
except MNSExceptionBase as e:
if e.type == 'MessageNotExist':
logger.debug("no message left in queue [%s]" %self.mns_queue.queue_name)
else:
logger.warn("Receive Message Fail!\nException:%s\n\n" % e)
return ret_flag
def del_msg(self, retry=3):
"""
@return:
"""
num = retry
ret_flag = False
if self.last_recv_msg is None:
logger.debug("last recv msg is None, nothing to do")
return True
while num > 0:
try:
self.mns_queue.delete_message(self.last_recv_msg.receipt_handle)
logger.debug("Delete Message Succeed.\n\n")
ret_flag = True
self.last_recv_msg = None
return ret_flag
except MNSExceptionBase, e:
logger.warn("Del Message from %s Fail!\nException:%s\n\n, retry again" % (self.mns_queue.queue_name,e))
time.sleep(1.0)
logger.warn("try to delete Message %s time still Failed, message will be push back to queue [%s]" %(retry, self.mns_queue.queue_name))
return ret_flag
self.__tmp_access_key = response_body.get("MessageTokenDTO").get(
"AccessKeySecret")
self.__tmp_access_id = response_body.get("MessageTokenDTO").get(
"AccessKeyId")
self.__expire_time = response_body.get("MessageTokenDTO").get(
"ExpireTime")
self.__token = response_body.get("MessageTokenDTO").get(
"SecurityToken")
print "finsh refresh token..."
# 初始化 my_account, my_queue
token = Token()
token.refresh()
my_account = Account(endpoint, token.get_tmp_access_id(),
token.get_tmp_access_key(), token.get_token())
my_queue = my_account.get_queue(qname)
# my_queue.set_encoding(False)
# 循环读取删除消息直到队列空
# receive message请求使用long polling方式，通过wait_seconds指定长轮询时间为3秒
## long polling 解析:
### 当队列中有消息时，请求立即返回；
### 当队列中没有消息时，请求在MNS服务器端挂3秒钟，在这期间，有消息写入队列，请求会立即返回消息，3秒后，请求返回队列没有消息；
wait_seconds = 3
print "%sReceive And Delete Message From Queue%s\nQueueName:%s\nWaitSeconds:%s\n" % (
10 * "=", 10 * "=", qname, wait_seconds)
while True:
# 读取消息
try:
import sys
import os
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)) + "/..")
import time
from sample.sample_common import MNSSampleCommon
from mns.account import Account
from mns.queue import *
# 从sample.cfg中读取基本配置信息 # WARNING： Please do not hard code your accessId and accesskey in next line.(more information:
# https://yq.aliyun.com/articles/55947)
accid, acckey, endpoint, token = MNSSampleCommon.LoadConfig()
#初始化 my_account, my_queue
my_account = Account(endpoint, accid, acckey, token)
queue_name = sys.argv[1] if len(sys.argv) > 1 else "MySampleQueue"
base64 = False if len(
sys.argv) > 2 and sys.argv[2].lower() == "false" else True
my_queue = my_account.get_queue(queue_name)
my_queue.set_encoding(base64)
#循环读取删除消息直到队列空
#receive message请求使用long polling方式，通过wait_seconds指定长轮询时间为3秒
## long polling 解析:
### 当队列中有消息时，请求立即返回；
### 当队列中没有消息时，请求在MNS服务器端挂3秒钟，在这期间，有消息写入队列，请求会立即返回消息，3秒后，请求返回队列没有消息；
wait_seconds = 3
print(
#获取配置信息
## AccessKeyId 阿里云官网获取
## AccessKeySecret 阿里云官网获取
## Endpoint 阿里云消息和通知服务官网获取, Example: http://$AccountId.mns.cn-hangzhou.aliyuncs.com
## WARNING： Please do not hard code your accessId and accesskey in next line.(more information: https://yq.aliyun.com/articles/55947)
accessKeyId = parser.get("Base", "AccessKeyId")
accessKeySecret = parser.get("Base", "AccessKeySecret")
endpoint = parser.get("Base", "Endpoint")
securityToken = ""
if parser.has_option("Optional", "SecurityToken") and parser.get(
"Optional", "SecurityToken") != "$SecurityToken":
securityToken = parser.get("Optional", "SecurityToken")
#初始化my_account
my_account = Account(endpoint, accessKeyId, accessKeySecret, securityToken)
##############Queue 相关操作#####################
my_queue = my_account.get_queue(
"MyQueue-%s" % time.strftime("%y%m%d-%H%M%S", time.localtime()))
#创建队列
## message被receive后，持续不可消费的时间 100秒
## message body的最大长度 10240Byte
## message最长存活时间 3600秒
## 新message可消费的默认延迟时间 10秒
## receive message时，长轮询时间 20秒
queue_meta = QueueMeta()
queue_meta.set_visibilitytimeout(100)
queue_meta.set_maximum_message_size(10240)
queue_meta.set_message_retention_period(3600)
#coding=utf8
import sys
import os
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)) + "/..")
import time
from sample_common import MNSSampleCommon
from mns.account import Account
from mns.queue import *
#从sample.cfg中读取基本配置信息
## WARNING： Please do not hard code your accessId and accesskey in next line.(more information: https://yq.aliyun.com/articles/55947)
accid,acckey,endpoint,token = MNSSampleCommon.LoadConfig()
#初始化 my_account, my_queue
my_account = Account(endpoint, accid, acckey, token)
queue_name = sys.argv[1] if len(sys.argv) > 1 else "MySampleQueue"
my_queue = my_account.get_queue(queue_name)
#创建队列, 具体属性请参考mns/queue.py中的QueueMeta结构
queue_meta = QueueMeta()
try:
queue_url = my_queue.create(queue_meta)
print "Create Queue Succeed! QueueName:%s\n" % queue_name
except MNSExceptionBase, e:
if e.type == "QueueAlreadyExist":
print "Queue already exist, please delete it before creating or use it directly."
sys.exit(0)
print "Create Queue Fail! Exception:%s\n" % e
class ResumeExchange(Spider):
def __init__(self):
super(ResumeExchange, self).__init__()
self.mns_account = Account(MNS_ENDPOINT, MNS_ACCID, MNS_ACCKEY,
MNS_TOKEN)
self.mns_client = self.mns_account.get_queue(MNS_QUEUE)
def get_targetJobType(self, emailJobType):
for item in job_type_mapping:
if item.upper() in emailJobType:
return job_type_mapping[item]
return
def save2db(self, sql, data):
# conn = pymysql.Connect(**MYSQL_CONF_TEST)
conn = pymysql.Connect(**MYSQL_CONF)
cur = conn.cursor()
try:
cur.execute(sql, data)
conn.commit()
cur.execute("select last_insert_id()")
sql_id = cur.fetchone()[0]
self.logger.info("入库成功. %s" % sql_id)
return sql_id
except Exception as e:
self.logger.exception(e)
return
finally:
cur.close()
conn.close()
def save2mns(self, sql_id, data_for_mns, trackId):
try:
kafka_data = {
"content": data_for_mns,
"channelType": "WEB",
"interfaceType": "NORMAL",
"resourceDataType": "PARSED",
"resourceType": "RESUME_INBOX",
"source": data_for_mns['source'],
"traceID": trackId,
"callSystemID": PROJECT_NAME
}
dumps = json.dumps(kafka_data, ensure_ascii=False)
buf = StringIO()
f = gzip.GzipFile(mode='wb', fileobj=buf)
f.write(dumps)
f.close()
msg_body = base64.b64encode(buf.getvalue())
msg = Message(msg_body)
self.mns_client.send_message(msg)
self.logger.info("推送mns成功. %s" % str(sql_id))
except Exception as e:
self.logger.exception("推送mns失败: %s" % e)
def run(self, file_path):
try:
with open(file_path, 'r') as file:
self.logger.info("开始上载简历.")
for index, line in enumerate(file):
# self.process1(index=index,line=line)
self.process2(index=index, line=line)
except Exception as e:
self.logger.exception(e)
def process2(self, index, line):
try:
self.logger.info('当前处理的内如：%s %s' % (index, line))
name, mobile, gender, degree, age, work_exp, target_job_type, register_location = line.strip(
'\n').split('\t')
except ValueError:
self.logger.warning("该行读取错误: " + str(index))
return
now_year = datetime.datetime.now().year
birthday = ''
if age:
birthday = str(now_year - int(age)) + '-05-15'
uuid_ = str(uuid.uuid1())
data = (name, mobile, gender, degree, birthday, work_exp,
target_job_type, register_location,
datetime.datetime.strftime(datetime.datetime.now(),
'%Y-%m-%d'),
datetime.datetime.strftime(datetime.datetime.now(),
'%Y-%m-%d %H:%M:%S'),
datetime.datetime.strftime(datetime.datetime.now(),
'%Y-%m-%d %H:%M:%S'), uuid_)
sql = '''
INSERT INTO spider.resume_parsed
(name,mobile,gender,degree,birthday,workExperience,targetJobType,registerLocation,targetWorkLocation,workType,status,source,targetSalary,resumeUpdateTime,createTime,updateTime,trackId)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s,'西安','全职','正在找工作','RESUME_EXCHANGE','面议',%s,%s,%s,%s)
'''
sql_id = self.save2db(sql=sql, data=data)
self.logger.info('解析入库成功 %s' % sql_id)
data_for_mns = {
'id':
sql_id,
'name':
name,
'mobile':
mobile,
'gender':
gender,
'degree':
degree,
'birthday':
birthday,
'workExperience':
work_exp,
'targetJobType':
target_job_type,
'registerLocation':
register_location,
'targetWorkLocation':
'西安',
'workType':
'全职',
'status':
'正在找工作',
'source':
'RESUME_EXCHANGE',
'targetSalary':
'面议',
'resumeUpdateTime':
datetime.datetime.strftime(datetime.datetime.now(),
'%Y-%m-%d %H:%M:%S'),
'createTime':
datetime.datetime.strftime(datetime.datetime.now(),
'%Y-%m-%d %H:%M:%S'),
'updateTime':
datetime.datetime.strftime(datetime.datetime.now(),
'%Y-%m-%d %H:%M:%S'),
'trackId':
uuid_,
}
self.save2mns(sql_id, data_for_mns, uuid_)
self.logger.info('处理完毕 %s' % sql_id)
def process1(self, index, line):
try:
name, emailJobType, targetJobType, gender, birthday, mobile, \
workExperience, email, currentLocation, registerLocation, \
career, education, degree, targetSalary = line.strip(
'\n').split('\t')
except ValueError:
self.logger.warning("该行读取错误: " + str(index))
return
try:
emailJobType = emailJobType.replace('北京', '').replace('求职',
'').replace(
'/', ',')
targetJobType = self.get_targetJobType(emailJobType)
if not targetJobType:
targetJobType = emailJobType
birthday = birthday.replace('/', '-')
career_lst = []
if career:
for item in career.split('|$|'):
career_lst.append({"companyName": item})
career = json.dumps(career_lst, ensure_ascii=False)
education_lst = []
if education:
for item in education.split('|$|'):
education_lst.append({"schoolName": item})
education = json.dumps(education_lst, ensure_ascii=False)
now = datetime.datetime.now()
trackId = str(uuid.uuid1())
data = (name, emailJobType, targetJobType, gender, birthday,
mobile, workExperience, email, currentLocation,
registerLocation, career, education, degree, targetSalary,
now, '2017-11-01 00:00:00', trackId)
sql = """
INSERT INTO spider.resume_parsed(name, emailJobType,
targetJobType, gender, birthday, mobile, \
workExperience, email, currentLocation, registerLocation, \
career, education, degree, targetSalary, createTime,
updateTime, trackId,
createBy, source, workType, status) VALUES (%s, %s, %s,
%s, %s, %s, %s, %s, %s,
%s, %s, %s, %s, %s, %s, %s, %s, 'PYTHON',
'RESUME_EXCHANGE',
'全职', '正在找工作')
"""
sql_id = self.save2db(sql=sql, data=data)
if not sql_id:
return
data_for_mns = {
'id': sql_id,
'name': name,
'emailJobType': emailJobType,
'targetJobType': targetJobType,
'gender': gender,
'birthday': birthday,
'mobile': mobile,
'workExperience': workExperience,
'email': email,
'currentLocation': currentLocation,
'registerLocation': registerLocation,
'career': career_lst,
'education': education_lst,
'degree': degree,
'targetSalary': targetSalary,
'createTime':
datetime.datetime.strftime(now, '%Y-%m-%d %H:%M:%S'),
'updateTime': '2017-11-01 00:00:00',
'trackId': trackId,
'source': 'RESUME_EXCHANGE',
'workType': '全职',
'status': '正在找工作'
}
self.save2mns(sql_id, data_for_mns, trackId)
self.logger.info(
'%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s' %
(index, name, emailJobType, targetJobType, gender, birthday,
mobile, workExperience, email, currentLocation,
registerLocation, career, education, degree, targetSalary))
except Exception as e:
self.logger.exception(e)
return
class AliAPI:
def __init__(self):
self.client = AcsClient(app.config.get("ALI_ACCESS_KEY"),
app.config.get("ALI_ACCESS_SECRET"),
app.config.get("BASE_REGION"))
self.account = Account(app.config.get("MNS_ENDPOINT"),
app.config.get("ALI_ACCESS_KEY"),
app.config.get("ALI_ACCESS_SECRET"), "")
self.wait_seconds = 10
self.kube = Kubernetes()
def reboot_ecs(self, ecs_id):
"""
reboot ecs
:param ecs_id:
:return:
"""
try:
request = RebootInstanceRequest()
request.set_accept_format('json')
request.set_InstanceId(ecs_id)
response = self.client.do_action_with_exception(request)
logs.info("reboot ecs {} successed result is {}".format(
ecs_id, response))
return True
except ServerException as exc:
logs.info("reboot ecs failed msg is {}".format(exc))
def set_ecs(self, ecs_id, ecs_hostname, ecs_instancename):
"""
set ecs hostname
:param ecs_id:
:param ecs_hostname:
:param ecs_instancename:
:return:
"""
try:
request = ModifyInstanceAttributeRequest()
request.set_accept_format('json')
request.set_InstanceId(ecs_id)
request.set_InstanceName(ecs_instancename)
response = self.client.do_action_with_exception(request)
logs.info(
"set ecs ecs_id is {} ecs_hostname is {} ecs_instancename is {} response is {}"
.format(ecs_id, ecs_hostname, ecs_instancename, response))
return True
except ServerException as exc:
logs.error("set ecs failed msg is {}".format(exc))
def get_ecs_info(self, ecs_id):
"""
get ecs ip and hostname
:param ecs_id:
:return:
"""
ip, hostname = None, None
try:
request = DescribeInstancesRequest()
request.set_accept_format('json')
request.set_InstanceIds([
ecs_id,
])
response = json.loads(
self.client.do_action_with_exception(request))
# 私有网络IP
ip = response.get("Instances").get("Instance")[0].get(
"VpcAttributes").get("PrivateIpAddress").get("IpAddress")[0]
hostname = response.get("Instances").get("Instance")[0].get(
"HostName")
except ServerException as exc:
logs.error("get ecs info failed msg is {}".format(exc))
finally:
return ip, hostname
def handlers(self, transiton, ecs_id, hostname, ip, scaling_groupname):
"""
响应伸缩组活动
:param transiton:
:param ecs_id:
:param hostname:
:param ip:
:param scaling_groupname:
:return:
"""
assert self.set_ecs(ecs_id, hostname,
hostname), AssertionError('set ecs failed.')
if ip and hostname:
if hasattr(self.kube, transiton.lower()):
result = getattr(self.kube, transiton.lower())(ip, hostname)
if not result:
logs.info("Reboot Ecs id is {} hostname is {}".format(
ecs_id, hostname))
self.reboot_ecs(ecs_id)
# 发送钉钉通知
self.kube.notice.send_mardown(
title='伸缩组活动',
text='#### 伸缩组活动\n'
'> \n\n'
'> - 伸缩组名称 {}\n'
'> - 动作类型 **{}**\n'
'> - ECS服务器名称 {}\n'
'> - ECS服务器IP {}\n'
'> ###### {} \n'.format(
scaling_groupname,
"缩容机器" if transiton == "SCALE_IN" else "扩容机器", hostname,
ip, time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
def receive_msg(self, queue_name=app.config.get("MNS_QUEUENAME")):
queue = self.account.get_queue(queue_name)
while 1:
try:
recv_msg = queue.receive_message(self.wait_seconds)
message_body = bytes.decode(recv_msg.message_body)
change_msg_vis = queue.change_message_visibility(
recv_msg.receipt_handle, 35)
self.delete_msg(change_msg_vis)
try:
message_body = json.loads(message_body)
except json.decoder.JSONDecodeError:
logs.debug("json decoder failes.")
# 只处理dict类型数据
if type(message_body) is dict:
logs.info("recv msg is {}".format(message_body))
transiton = message_body.get("content").get(
"lifecycleTransition")
if transiton not in storage.cache:
storage.cache[transiton] = []
scaling_groupname = message_body.get("content").get(
"scalingGroupName")
for ecs_id in message_body.get("content").get(
"instanceIds"):
if ecs_id not in storage.cache[transiton]:
storage.cache[transiton].append(ecs_id)
ip, _ = self.get_ecs_info(ecs_id)
if ip in storage.host_map:
hostname = storage.host_map[ip]
else:
# 拼接主机名
hostname = "{}{}".format(
app.config.get("HOSTNAME_PREFIX"),
("%03d" % get_counter()))
storage.host_map[ip] = hostname
# 线程处理
threading.Thread(target=self.handlers,
args=(transiton, ecs_id, hostname,
ip,
scaling_groupname)).start()
except MNSExceptionBase as e:
logs.error("recv msg failed {}".format(e))
def delete_msg(self,
change_msg_vis,
queue_name=app.config.get("MNS_QUEUENAME")):
queue = self.account.get_queue(queue_name)
queue.delete_message(change_msg_vis.receipt_handle)
logs.info("del msg {}".format(change_msg_vis.receipt_handle))
#DB_PASSWORD = "******"
DB_HOST = "47.95.232.88"
DB_USERNAME = "******"
DB_PASSWORD = "******"
DB_DATABASE = "greengrass"
# Read the basic configurations from sample.cfg
# WARNING: Please do not hard code your accessId and accesskey in next line.
# (more information: https://yq.aliyun.com/articles/55947)
accessKeyId, accesskeySecret, endpoint, token = MNSSampleCommon.LoadConfig()
# create the connect for IoT Kit
clt = client.AcsClient(accessKeyId, accesskeySecret, 'cn-shanghai')
# Init my_account, my_queue
my_account = Account(endpoint, accessKeyId, accesskeySecret, token)
queue_name = sys.argv[1] if len(sys.argv) > 1 else queueName
boolbase64 = False if len(
sys.argv) > 2 and sys.argv[2].lower() == "false" else True
my_queue = my_account.get_queue(queue_name)
my_queue.set_encoding(boolbase64)
# get the connection to Amazon Greengrass RDS
db = MySQLdb.connect(DB_HOST, DB_USERNAME, DB_PASSWORD, DB_DATABASE)
# Read and delete the message from queue until the queue is empty
# receive message uses long polling mode, specify the long loop time to 3 second through wait_seconds
# long polling parsing
# Return the message when there are messages
# When queue is empty, request will wait for 3 second on server side.
#获取配置信息
## AccessKeyId 阿里云官网获取
## AccessKeySecret 阿里云官网获取
## Endpoint 阿里云消息和通知服务官网获取, Example: http://$AccountId.mns.cn-hangzhou.aliyuncs.com
## WARNING： Please do not hard code your accessId and accesskey in next line.(more information: https://yq.aliyun.com/articles/55947)
accessKeyId = parser.get("Base", "AccessKeyId")
accessKeySecret = parser.get("Base", "AccessKeySecret")
endpoint = parser.get("Base", "Endpoint")
securityToken = ""
if parser.has_option("Optional", "SecurityToken") and parser.get("Optional", "SecurityToken") != "$SecurityToken":
securityToken = parser.get("Optional", "SecurityToken")
#初始化my_account
my_account = Account(endpoint, accessKeyId, accessKeySecret, securityToken)
##############Queue 相关操作#####################
my_queue = my_account.get_queue("MyQueue-%s" % time.strftime("%y%m%d-%H%M%S", time.localtime()))
#创建队列
## message被receive后，持续不可消费的时间 100秒
## message body的最大长度 10240Byte
## message最长存活时间 3600秒
## 新message可消费的默认延迟时间 10秒
## receive message时，长轮询时间 20秒
queue_meta = QueueMeta()
queue_meta.set_visibilitytimeout(100)
queue_meta.set_maximum_message_size(10240)
queue_meta.set_message_retention_period(3600)
queue_meta.set_delay_seconds(10)
#! -*- coding: utf-8 -*-
import json
from mns.account import Account
from mns.mns_exception import MNSExceptionBase
from utils import post_slack
from config import QUEUE_NAME, END_POINT, ACCESS_KEY_ID, ACCESS_KEY_SECRET
account = Account(END_POINT, ACCESS_KEY_ID, ACCESS_KEY_SECRET)
queue = account.get_queue(QUEUE_NAME)
def run():
while True:
try:
recv_msg = queue.receive_message(wait_seconds=10)
message = json.loads(recv_msg.message_body)['message']
print message
if message['levelDescription'] == u'恢复正常':
color = 'good'
else:
color = 'danger'
post_slack(message['dimensions'], message['metricName'],
message['expression'], color)
queue.delete_message(recv_msg.receipt_handle)
except MNSExceptionBase, e:
print e.message
if __name__ == '__main__':
import sys
import os
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)) + "/..")
import time
from sample_common import MNSSampleCommon
from mns.account import Account
from mns.topic import *
#从sample.cfg中读取基本配置信息
## WARNING： Please do not hard code your accessId and accesskey in next line.(more information: https://yq.aliyun.com/articles/55947)
accid,acckey,endpoint,token = MNSSampleCommon.LoadConfig()
#初始化 my_account, my_topic
my_account = Account(endpoint, accid, acckey, token)
topic_name = sys.argv[1] if len(sys.argv) > 1 else "MySampleTopic"
my_topic = my_account.get_topic(topic_name)
#循环发布多条消息
msg_count = 3
print "%sPublish Message To Topic%s\nTopicName:%s\nMessageCount:%s\n" % (10*"=", 10*"=", topic_name, msg_count)
for i in range(msg_count):
try:
msg_body = "I am test message %s." % i
msg = TopicMessage(msg_body)
re_msg = my_topic.publish_message(msg)
print "Publish Message Succeed. MessageBody:%s MessageID:%s" % (msg_body, re_msg.message_id)
except MNSExceptionBase,e:
if e.type == "TopicNotExist":
GPIO_clean()
if __name__ == '__main__':
print "start deamon!"
#init my_account, my_topic
endpoint = "http://1402010103453031.mns.cn-hangzhou.aliyuncs.com/"
access_id = "LTAIfpx57LeEmMKs"
access_key = "nUqBzUFc48cTVIZh3mkpcnmosxvGTz"
token = ""
quit = Queue()
dr = DryingRack()
GPIO_init()
#init my_account, my_topic
my_account = Account(endpoint, access_id, access_key, token)
# queue_name = "control-queue"
# my_queue = my_account.get_queue(queue_name)
#Thread(target=control_thread,args=(my_account,)).start()
#Thread(target=ioread_thread,args=(my_queue,)).start()
Process(target=ioread_thread, args=(my_account, quit)).start()
Process(target=control_thread, args=(my_account, quit)).start()
#ioread_thread(my_account)
while True:
s = raw_input("type quit to quit:")
if s == "quit":
quiting = True
quit.put("quit")
#coding:utf-8
import os
from flask import Flask, jsonify, request
from mns.account import Account
from mns.topic import DirectSMSInfo, TopicMessage, MNSExceptionBase
app = Flask(__name__)
account = Account(os.getenv('END_POINT'), os.getenv('ACCESS_ID'),
os.getenv('ACCESS_KEY'))
topic = account.get_topic(os.getenv('TOPIC'))
accessToken = os.getenv('ACCESS_TOKEN')
class InvalidUsage(Exception):
status_code = 400
def __init__(self, message, status_code=None, payload=None):
Exception.__init__(self)
self.message = message
if status_code is not None:
self.status_code = status_code
self.payload = payload
def to_dict(self):
rv = dict(self.payload or ())
rv['message'] = self.message
return rv
@app.errorhandler(InvalidUsage)
def handle_invalid_usage(error):
# coding: utf-8
import sys
from mns.account import Account
from mns.topic import DirectSMSInfo, TopicMessage
from mns.mns_exception import MNSExceptionBase
accid = 'LTAIf6atnIup14AK'
acckey = 'zfoeCi2Au0le7vUbsFOmEVtPrtlfZf'
endpoint = 'http://1771999519197750.mns.cn-hangzhou.aliyuncs.com/'
topic_name = 'sms.topic-cn-hangzhou'
my_account = Account(endpoint, accid, acckey)
my_topic = my_account.get_topic(topic_name)
'''
Step 2. 设置SMS消息体（必须）
注：目前暂时不支持消息内容为空，需要指定消息内容，不为空即可。
'''
msg_body1 = "sms-message1."
'''
Step 3. 生成SMS消息属性，single=False表示每个接收者参数不一样，
'''
# 3.1 设置SMSSignName和SMSTemplateCode
direct_sms_attr1 = DirectSMSInfo(free_sign_name='宁师生活网', template_code='SMS_69460049', single=False)
# 3.2 指定接收短信的手机号并指定发送给该接收人的短信中的参数值（在短信模板中定义的）
direct_sms_attr1.add_receiver(receiver='18150850025', params={'code': '123456'})
'''
#Step 5. 生成SMS消息对象
'''
msg1 = TopicMessage(msg_body1, direct_sms=direct_sms_attr1)
print(direct_sms_attr1.sms_params)
try:
'''
Step 6. 发布SMS消息
class Runner(object):
def __init__(self):
# 读取邮箱信息
self.db = Database(
'mysql+mysqldb://spider_admin:'
'[email protected]:'
'3306/spider?charset=utf8',
encoding='utf-8')
# 初始化数据库
self.db1 = Database(
'mysql+mysqldb://%s:%s@%s:%s/%s?charset=utf8' %
(MYSQL_USER, MYSQL_PASSWD, MYSQL_HOST, MYSQL_PORT, MYSQL_DB),
encoding='utf-8')
# 初始化MNS
self.mns_account = Account(MNS_ENDPOINT, MNS_ACCID, MNS_ACCKEY,
MNS_TOKEN)
self.mns_client = self.mns_account.get_queue(MNS_QUEUE)
self.logger = Logger.timed_rt_logger()
self.queue = RedisQueue('t_mail_by_date',
host=REDIS_HOST,
port=REDIS_PORT)
self.robot = DingDingRobot(
access_token="eb749abfe9080a69da6524b77f589b8f6ddbc"
"c182c7a41bf095b095336edb0a1")
def deliver(self, subject):
try:
if '51job.com' in subject:
source = "FIVE_ONE"
deliverJobName = re.search('(?<=申请贵公司).*(?=（)',
subject).group().strip()
subject_ = subject.replace('（', '(').replace('）', ')')
deliverCity = re.findall('(?<=\().*?(?=\))',
subject_)[-1].strip()
elif '58.com' in subject:
# (58.com)应聘贵公司供应链管理8k六险一金-北京 朝阳-刘春雨
source = "FIVE_EIGHT"
deliverJobName = re.search('(?<=应聘贵公司).*?(?=-)',
subject).group().strip()
deliverCity = re.findall('(?<=-).*?(?=\s+)',
subject)[-1].strip()
elif 'Zhaopin.com' in subject:
source = "ZHI_LIAN"
deliverJobName = re.search('(?<=应聘).*?(?=-)',
subject).group().strip()
deliverCity = re.findall('(?<=-).*?(?=-)', subject)[-1].strip()
else:
return None, None, None
return source, deliverJobName, deliverCity
except:
return None, None, None
def save2db(self, source, email, subject, content, emailJobType, emailCity,
deliverJobName, deliverJobCity, trackId):
try:
createTime = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
data = {
'source': source,
'email': email,
'subject': subject,
'content': content,
'emailJobType': emailJobType,
'emailCity': emailCity,
'createBy': email,
'updateBy': 'PYTHON-RECEIVER',
'deliverJobName': deliverJobName,
'deliverJobCity': deliverJobCity,
'createTime': createTime,
'trackId': trackId
}
try:
self.db1.session.execute(ResumeRaw.__table__.insert(), data)
self.db1.session.commit()
except Exception as e:
self.db1.session.rollback()
self.logger.exception("入库失败, 回滚： %s" % e)
self.robot.send_text("邮箱采集，入库失败： %s" % email)
time.sleep(600)
return
sql_id = self.db1.session.execute(
"select last_insert_id()").first()[0]
# print(sql_id)
self.logger.info("入库成功. %s" % str(sql_id))
return sql_id
except Exception as e:
self.logger.exception("入库失败： %s" % e)
return
def save2mns(self, sql_id, source, content, emailJobType, emailCity,
deliverJobName, deliverJobCity, trackId):
try:
# resume_uuid = uuid.uuid1()
kafka_data = {
"channelType": "WEB",
"content": {
"content": content,
"id": sql_id,
"createBy": "python",
"createTime": int(time.time() * 1000),
"ip": '',
'emailJobType': emailJobType,
'emailCity': emailCity,
'deliverJobName': deliverJobName,
'deliverJobCity': deliverJobCity,
"resumeSubmitTime": '',
"resumeUpdateTime": '',
"source": source,
"trackId": str(trackId),
"avatarUrl": '',
},
"interfaceType": "PARSE",
"resourceDataType": "RAW",
"resourceType": "RESUME_EMAIL",
"source": source,
"trackId": str(trackId),
"callSystemID": project_name,
"traceID": str(trackId)
}
dumps = json.dumps(kafka_data, ensure_ascii=False)
buf = StringIO()
f = gzip.GzipFile(mode='wb', fileobj=buf)
f.write(dumps)
f.close()
msg_body = base64.b64encode(buf.getvalue())
msg = Message(msg_body)
self.mns_client.send_message(msg)
self.logger.info("推送mns成功. %s" % str(sql_id))
except Exception as e:
self.logger.exception("推送mns失败: %s" % e)
def producer(self):
self.queue.clean()
# res = self.db.session.query(TEmail).filter_by(valid=1).all()
res = self.db.session.query(TEmail).filter(
TEmail.email.in_((
# '*****@*****.**',
# '*****@*****.**',
# '*****@*****.**',
# '*****@*****.**',
# '*****@*****.**',
# '*****@*****.**',
'*****@*****.**',
# '*****@*****.**',
# '*****@*****.**',
)))
if not self.queue.empty():
self.logger.info("剩余%s个企业邮箱未处理！" % str(self.queue.qsize()))
sys.exit()
# self.queue.clean()
for item in res:
self.queue.put(
(item.pop3_host.encode('utf-8'), item.username.encode('utf-8'),
item.password.encode('utf-8'),
item.position_type.encode('utf-8'),
item.preferred_city.encode('utf-8'),
item.source.encode('utf-8')))
self.logger.info("总计加载%s个企业邮箱！" % str(self.queue.qsize()))
def consumer(self):
while True:
try:
if not self.queue.empty():
host, username, password, position_type, preferred_city, \
source1 = eval(self.queue.get())
# print(host, username, password)
imap = MailReceiver(host, username, password)
try:
imap.server.login(username, password)
self.logger.info("%s: 登录成功 <%s>" %
(username, self.queue.qsize()))
except Exception as e:
if "LOGIN Login error" in e:
self.robot.send_text("邮箱登录异常： %s %s %s" %
(username, password, host))
self.logger.exception(
"%s: 登录失败 %s <%s>" %
(username, e, self.queue.qsize()))
continue
try:
folder_list = imap.folder_list()
except Exception as e:
self.logger.exception("%s: 邮件箱加载失败 %s" % (username, e))
continue
if not folder_list:
continue
# print(folder_list)
if "Deleted Messages" in folder_list:
delete_folder = u"Deleted Messages"
elif "已删除邮件" in folder_list:
delete_folder = u"已删除邮件"
else:
delete_folder = u"已删除"
self.logger.info("%s: 开始收取邮件." % username)
today = datetime.datetime.now()
days = 180
for day in xrange(1, days):
date = today - datetime.timedelta(days=day)
fmt_date = datetime.datetime.strftime(date, "%d-%b-%Y")
self.logger.info('%s开始收取：%s 邮件' % (username, fmt_date))
try:
message_list, msg_data = imap.message_list(
search_='ON %s' % fmt_date)
except Exception as e:
self.logger.exception("%s: 邮件list加载失败 %s" %
(username, e))
continue
if not msg_data:
self.logger.warning("%s: 没有收到新邮件." % username)
continue
self.logger.info("%s: 收到%s个新邮件" %
(username, str(len(message_list))))
# print(message_list)
for msg_id, message in msg_data.items():
res = imap.get_content(message)
if not res:
# message_list.append(message)
continue
try:
subject = res.get('subject').encode(
'utf-8', 'ignore')
except UnicodeDecodeError:
subject = res.get('subject')
content = res.get('body')
emoji_pattern = re.compile(
u"(\ud83d[\ude00-\ude4f])|(\ud83c[\udf00-\uffff])|(\ud83d[\u0000-\uddff])|(\ud83d[\ude80-\udeff])|(\ud83c[\udde0-\uddff])+",
flags=re.UNICODE)
content = emoji_pattern.sub(r'', content)
source, deliverJobName, deliverJobCity = self.deliver(
subject)
if not deliverJobName:
self.logger.warning(
"%s: 收到非期待邮件. subject: %s" %
(username, subject))
try:
imap.delete_message([msg_id],
delete_folder)
self.logger.info(
"%s: 非期待邮件<%s>成功转移到“已删除”" %
(username, msg_id))
except Exception as e:
self.logger.exception(
"%s: 非期待邮件<%s>转移到“已删除”失败 %s" %
(username, msg_id, e))
continue
self.logger.info(
"%s\t%s\t%s\t" %
(source, deliverJobName, deliverJobCity))
trackId = uuid.uuid4()
sql_id = self.save2db(source, username, subject,
content, position_type,
preferred_city,
deliverJobName,
deliverJobCity, trackId)
if not sql_id:
continue
self.save2mns(sql_id, source.encode('utf-8'),
content.encode('utf-8'),
position_type, preferred_city,
deliverJobName, deliverJobCity,
trackId)
try:
imap.delete_message([msg_id], delete_folder)
self.logger.info("%s: 邮件<%s>成功转移到“已删除”" %
(username, msg_id))
except Exception as e:
self.logger.exception(
"%s: 邮件<%s>转移到“已删除”失败 %s" %
(username, msg_id, e))
imap.close()
# else:
# self.logger.info("finished!")
# break
except Exception as e:
self.logger.exception(e)
def get_mns_client():
mns_account = Account(common_settings.ENDPOINT, common_settings.ACCID,
common_settings.ACCKEY, common_settings.TOKEN)
mns_client = mns_account.get_queue(common_settings.MNS_QUEUE)
return mns_client
def __init__(self):
super(ResumeExchange, self).__init__()
self.mns_account = Account(MNS_ENDPOINT, MNS_ACCID, MNS_ACCKEY,
MNS_TOKEN)
self.mns_client = self.mns_account.get_queue(MNS_QUEUE)
