def my_send_msg(num):
# 从sample.cfg中读取基本配置信息
## WARNING： Please do not hard code your accessId and accesskey in next line.(more information: https://yq.aliyun.com/articles/55947)
accid, acckey, endpoint, token = MNSSampleCommon.LoadConfig()
# 初始化 my_account, my_queue
my_account = Account(endpoint, accid, acckey, token)
queue_name = "MyQueue1"
my_queue = my_account.get_queue(queue_name)
try:
# msg_body = "I am test message %s." % i
msg_body = num
msg = Message(msg_body)
re_msg = my_queue.send_message(msg)
# print
# "Send Message Succeed! MessageBody:%s MessageID:%s" % (msg_body, re_msg.message_id)
except MNSExceptionBase, e:
if e.type == "QueueNotExist":
print
"Queue not exist, please create queue before send message."
sys.exit(0)
print
"Send Message Fail! Exception:%s\n" % e
def rece_del_msg(queue_name, wait_seconds=3):
"""
从指定的消息队列中获取消息，默认等待3s，每次获取1个
:param queue_name: 消息队列的名称
:param wait_seconds: 等待时长
:return: (None, ""): 队列不存在；(False, ""): 获取失败；(True, "***"): 获取成功
"""
access_key_id, access_key_secret, end_point, security_token = config_parse()
my_account = Account(end_point, access_key_id, access_key_secret, security_token)
my_queue = my_account.get_queue(queue_name)
try:
receive_message = my_queue.receive_message(wait_seconds)
except MNSExceptionBase as e:
if e.type == "QueueNotExist":
result = (None, "")
elif e.type == "MessageNotExist":
result = (True, "")
else:
result = (False, "")
else:
result = (True, receive_message.message_body)
try:
my_queue.delete_message(receive_message.receipt_handle)
except Exception as e:
result = (False, "")
return result
def run_consume():
account = Account(MNS_ENDPOINT, ALI_ACCESSKEYID, ALI_ACCESSKEYSECRET)
q = account.get_queue(QUEUE_NAME)
while True:
try:
wait_seconds = 10
recv_msg = q.receive_message(wait_seconds)
sys.stdout.write(
"Receive Message Succeed! \
\nMessageId: %s\nMessageBodyMD5: %s \
\nMessageBody: %s\nDequeueCount: %s \
\nEnqueueTime: %s\nFirstDequeueTime: %s \
\nPriority: %s\nNextVisibleTime: %s \
\nReceiptHandle: %s\n\n" %
(recv_msg.message_id, recv_msg.message_body_md5,
recv_msg.message_body, recv_msg.dequeue_count,
recv_msg.enqueue_time, recv_msg.first_dequeue_time,
recv_msg.priority, recv_msg.next_visible_time,
recv_msg.receipt_handle))
q.delete_message(recv_msg.receipt_handle)
sys.stdout.write("Delete Message Succeed.\n\n")
except MNSExceptionBase as e:
sys.stderr.write("Receive Message Fail!\nException:%s\n\n" % e)
break
try:
q.delete()
sys.stdout.write("Delete Queue Succeed!\n\n")
except MNSExceptionBase as e:
sys.stderr.write("Delete Queue Fail!\nException:%s\n\n" % e)
sys.exit(1)
def run_produce():
account = Account(MNS_ENDPOINT, ALI_ACCESSKEYID, ALI_ACCESSKEYSECRET)
q = account.get_queue(QUEUE_NAME)
queue_meta = QueueMeta()
queue_meta.set_visibilitytimeout(100)
queue_meta.set_maximum_message_size(10240)
queue_meta.set_message_retention_period(3600)
queue_meta.set_delay_seconds(10)
queue_meta.set_polling_wait_seconds(20)
queue_meta.set_logging_enabled(True)
try:
q_url = q.create(queue_meta)
sys.stdout.write("Create Queue Succeed!\nQueueURL:%s\n\n" % q_url)
except MNSExceptionBase as e:
sys.stderr.write("Create Queue Fail!\nException:%s\n\n" % e)
sys.exit(1)
for i in range(1, 10):
msg_body = "Test Message n. {} at {}".format(
i,
datetime.now().isoformat())
message = Message(msg_body)
message.set_delayseconds(0)
message.set_priority(10)
try:
send_msg = q.send_message(message)
sys.stdout.write(
"Send Message Succeed.\nMessageBody:%s\nMessageId:%s\nMessageBodyMd5:%s\n\n"
% (msg_body, send_msg.message_id, send_msg.message_body_md5))
except MNSExceptionBase as e:
sys.stderr.write("Send Message Fail!\nException:%s\n\n" % e)
sys.exit(1)
class aliyun_mns_api(object):
def __init__(self):
"""客户端初始化"""
cf = ConfigParser.ConfigParser()
cf.read("aliyun.conf")
host = cf.get('AccessKey_mns',"Host")
accessid = cf.get('AccessKey_mns',"Access_ID")
accesskey = cf.get('AccessKey_mns',"Access_Key")
self.cli = Account(host,accessid,accesskey)
def sendmessagerequest(self,queue_name):
"""向指定mns发送message"""
with open('SendMessage.txt','r') as f:
for line in f:
message = Message(line)
re_msg = self.cli.get_queue(queue_name).send_message(message)
print "Send Message Succeed! MessageBody:%s MessageID:%s" % (line,re_msg.message_id)
def getlistqueuenames(self):
"""查询消息列队名字"""
result = self.cli.list_queue()
name_list = []
for name in result[0]:
real_name = str(name).split('/')[-1]
name_list.append(real_name)
#print name_list
return name_list
def insert_db(self, item):
try:
self.sqlalchemy_handle.session.execute(
spider_search.ResumeRaw.__table__.insert(), item
)
self.sqlalchemy_handle.session.commit()
except Exception as e:
self.sqlalchemy_handle.session.rollback()
self.logger.exception("入库失败, 回滚： %s" % e)
raise e
sql_id = self.sqlalchemy_handle.session.execute(
"select last_insert_id()").first()[0]
self.logger.info("入库成功. %s" % str(sql_id))
try:
resume_uuid = uuid.uuid1()
mns_data = {
"channelType": "WEB",
"content": {
"content": item['content'],
"id": sql_id,
"createBy": "python",
"createTime": int(time.time() * 1000),
"ip": '',
'emailJobType': item['emailJobType'],
'emailCity': item['emailCity'],
'deliverJobName': '',
'deliverJobCity': '',
"resumeSubmitTime": '',
"resumeUpdateTime": '',
"source": item['source'],
"trackId": str(resume_uuid),
"avatarUrl": '',
},
"interfaceType": "PARSE",
"resourceDataType": "RAW",
"resourceType": "RESUME_SEARCH",
"source": item['source'],
"trackId": resume_uuid,
"traceID": resume_uuid,
"callSystemID": 'five_one_python3_scrapy'
}
dumps = json.dumps(mns_data, ensure_ascii=False)
buf = StringIO()
f = gzip.GzipFile(mode='wb', fileobj=buf)
f.write(dumps)
f.close()
msg_body = base64.b64encode(buf.getvalue())
msg = Message(msg_body)
# 初始化MNS
mns_account = Account(MNS_ENDPOINT, MNS_ACCID, MNS_ACCKEY,
MNS_TOKEN)
mns_client = mns_account.get_queue(MNS_QUEUE)
mns_client.send_message(msg)
self.logger.info("推送mns成功. %s" % str(sql_id))
except Exception as e:
self.logger.exception("推送mns失败: %s" % e)
def get_producer(self, settings):
if self._mns_producer:
return self._mns_producer
account = Account(
host=settings['MNS_ACCOUNT_ENDPOINT'],
access_id=settings['MNS_ACCESSKEY_ID'],
access_key=settings['MNS_ACCESSKEY_SECRET'],
logger=logging.getLogger('evascrapy.pipelines.AliyunMnsPipeline'),
debug=False,
)
self._mns_producer = account.get_queue(settings['MNS_QUEUE_NAME'])
return self._mns_producer
class MNSReader(object):
id = None
secret = None
point = None
message = None
queue_name = None
db = dict()
def __init__(self):
self.id = conf.access_key_id
self.secret = conf.access_key_secret
self.point = conf.queue_endpoint
self.queue_name = conf.queue_name
self.auth = oss2.Auth(conf.access_key_id, conf.access_key_secret)
self.bucket = oss2.Bucket(self.auth, conf.oss_endpoint,
conf.bucket_name)
self.oss_img_path = conf.oss_img_path
client = MongoClient(conf.mongo_host)
client['admin'].authenticate('root', 'qwer1234')
self.userdb = client['qp_account']
for index, subject in enumerate(_ALL_SUPPORT_SUBJECTS):
db_name = 'qp_' + subject
self.db[index] = client[db_name]
logging.critical('connect mongodb success:{}'.format(conf.mongo_host))
if self.id is None or self.secret is None or self.point is None or self.queue_name is None:
raise Exception('lose conf')
try:
self.account = Account(self.point, self.id, self.secret)
self.queue = self.account.get_queue(self.queue_name)
logging.critical('init_MNS success:')
logging.critical(u'id:{}, point:{}, queue_name'.format(
self.id, self.point, self.queue_name))
except:
traceback.print_exc()
raise Exception('connect MNS fail')
def pop(self):
retry = 2
while retry > 0:
try:
self.queue.delete_message(self.message.receipt_handle)
except MNSExceptionBase, e:
logging.fatal('release message fail: ' + str(e))
retry -= 1
time.sleep(1)
else:
logging.info('release message success')
break
self.message = None
def send_msg(queue_name, message_body):
"""
向指定消息队列发送消息
:param queue_name: 消息队列的名称
:param message_body: 需要发送的消息
:return: True：发送成功；False：发送失败；None：该消息队列不存在
"""
access_key_id, access_key_secret, end_point, security_token = config_parse()
my_account = Account(end_point, access_key_id, access_key_secret, security_token)
my_queue = my_account.get_queue(queue_name)
try:
msg = Message(str(message_body))
re_msg = my_queue.send_message(msg)
# print re_msg.message_id
except MNSExceptionBase as e:
if e.type == "QueueNotExist":
return None
else:
return False
else:
return True
def my_recv_msg():
#从sample.cfg中读取基本配置信息
## WARNING： Please do not hard code your accessId and accesskey in next line.(more information: https://yq.aliyun.com/articles/55947)
accid,acckey,endpoint,token = MNSSampleCommon.LoadConfig()
#初始化 my_account, my_queue
my_account = Account(endpoint, accid, acckey, token)
queue_name = "MyQueue1"
base64 = False if len(sys.argv) > 2 and sys.argv[2].lower() == "false" else True
my_queue = my_account.get_queue(queue_name)
my_queue.set_encoding(base64)
#循环读取删除消息直到队列空
#receive message请求使用long polling方式，通过wait_seconds指定长轮询时间为3秒
## long polling 解析:
### 当队列中有消息时，请求立即返回；
### 当队列中没有消息时，请求在MNS服务器端挂3秒钟，在这期间，有消息写入队列，请求会立即返回消息，3秒后，请求返回队列没有消息；
wait_seconds = 3
# print "%sReceive And Delete Message From Queue%s\nQueueName:%s\nWaitSeconds:%s\n" % (10*"=", 10*"=", queue_name, wait_seconds)
#读取消息
try:
recv_msg = my_queue.receive_message(wait_seconds)
# print "Receive Message Succeed! ReceiptHandle:%s MessageBody:%s MessageID:%s" % (recv_msg.receipt_handle, recv_msg.message_body, recv_msg.message_id)
except MNSExceptionBase, e:
if e.type == "QueueNotExist":
print
"Queue not exist, please create queue before receive message."
sys.exit(0)
elif e.type == "MessageNotExist":
print
"Queue is empty!"
sys.exit(0)
print
"Receive Message Fail! Exception:%s\n" % e
continue
class Command(BaseCommand):
help = "python manage.py service_runner"
args = ''
def get_message_broker(self):
"""
获取message broker
如果是无效的broker，则返回None
"""
if hasattr(settings, 'MESSAGE_BROKER'):
broker = getattr(settings, 'MESSAGE_BROKER')
else:
if hasattr(settings,
'MESSAGE_DEBUG_MODE') and settings.MESSAGE_DEBUG_MODE:
broker = 'redis'
else:
broker = 'mns'
if not broker in VALID_BROKERS:
logging.error('invalid message broker: %s' % broker)
broker = None
return broker
def create_queue(self, broker):
"""
根据broker类型，创建相应的queue对象
"""
if broker == 'redis':
#创建redis queue
from util import redis_queue
queue = redis_queue.get_queue(settings.SUBSCRIBE_QUEUE_NAME)
elif broker == 'mns':
#创建aliyun mns queue
self.mns_account = Account(\
settings.MNS_ENDPOINT, \
settings.MNS_ACCESS_KEY_ID, \
settings.MNS_ACCESS_KEY_SECRET, \
settings.MNS_SECURITY_TOKEN)
queue = self.mns_account.get_queue(settings.SUBSCRIBE_QUEUE_NAME)
else:
queue = None
return queue
def handle(self, *args, **options):
message_broker = self.get_message_broker()
if not message_broker:
logging.error('abort!!')
return
logging.info("message broker: {}".format(message_broker))
queue = self.create_queue(message_broker)
if not queue:
logging.error('invalid queue, abort!!')
return
logging.info("receive message from queue: {}".format(
settings.SUBSCRIBE_QUEUE_NAME))
# TODO: 改成LongPoll更好
while True:
message_data = {}
handler_func = None
handle_success = False
#读取消息
message_name = ''
try:
recv_msg = queue.receive_message(WAIT_SECONDS)
logging.info(
"Receive Message Succeed! ReceiptHandle:%s MessageBody:%s MessageID:%s"
% (recv_msg.receipt_handle, recv_msg.message_body,
recv_msg.message_id))
# 处理消息(consume)
data = json.loads(recv_msg.message_body)
message_name = data['name']
handler_func = handler_register.find_message_handler(
message_name)
if handler_func:
try:
message_data = data['data']
response = handler_func(message_data, recv_msg)
logging.info("service response: {}".format(response))
handle_success = True
#只有正常才能删除消息，否则消息仍然在队列中
try:
queue.delete_message(recv_msg.receipt_handle)
logging.debug(
"Delete Message Succeed! ReceiptHandle:%s" %
recv_msg.receipt_handle)
except MNSExceptionBase, e:
logging.debug(
"Delete Message Fail! Exception:%s\n" % e)
except:
traceback = unicode_full_stack()
logging.info(
u"Service Exception: {}".format(traceback))
message = {
'message_id': recv_msg.message_id,
'message_body_md5': '',
'data': message_data,
'queue_name': settings.SUBSCRIBE_QUEUE_NAME,
'msg_name': message_name,
'handel_success': handle_success,
}
uncaught_exception_data = get_uncaught_exception_data(
None, message)
if settings.MODE == 'deploy':
watchdog.critical(uncaught_exception_data,
'Uncaught_Exception')
else:
print('**********Uncaught_Exception**********')
print(json.dumps(uncaught_exception_data,
indent=2))
print('**********Uncaught_Exception**********\n')
else:
logging.warn(u"Warn: no such service found : {}".format(
message_name))
try:
queue.delete_message(recv_msg.receipt_handle)
logging.debug(
"Delete Message Succeed! ReceiptHandle:%s" %
recv_msg.receipt_handle)
except MNSExceptionBase, e:
logging.debug("Delete Message Fail! Exception:%s\n" %
e)
except MNSExceptionBase as e:
if e.type == "QueueNotExist":
logging.debug(
"Queue not exist, please create queue before receive message."
)
break
elif e.type == "MessageNotExist":
logging.debug("Queue is empty! Waiting...")
else:
logging.debug("Receive Message Fail! Exception:%s\n" % e)
time.sleep(SLEEP_SECONDS)
continue
except Exception as e:
traceback = unicode_full_stack()
logging.info(u"Service Exception: {}".format(traceback))
try:
if handler_func:
message = {
'message_id': recv_msg.message_id,
'message_body_md5': '',
'data': message_data,
'queue_name': settings.SUBSCRIBE_QUEUE_NAME,
'msg_name': message_name,
'handel_success': handle_success,
}
if handle_success:
watchdog.info(message, log_type='MNS_RECEIVE_LOG')
else:
watchdog.critical(message, log_type='MNS_RECEIVE_LOG')
except:
print(unicode_full_stack())
return
class Command(BaseCommand):
help = "python manage.py service_runner"
args = ''
# topic-queue模型中的queue
def handle(self, *args, **options):
global _SERVICE_LIST
# 准备访问MNS
self.mns_account = Account(\
settings.MNS_ENDPOINT, \
settings.MNS_ACCESS_KEY_ID, \
settings.MNS_ACCESS_KEY_SECRET, \
settings.MNS_SECURITY_TOKEN)
if hasattr(settings,
'MESSAGE_DEBUG_MODE') and settings.MESSAGE_DEBUG_MODE:
import redis_queue
queue = redis_queue.get_queue(settings.SUBSCRIBE_QUEUE_NAME)
logging.info("queue mode:{}".format('redis'))
else:
queue = self.mns_account.get_queue(settings.SUBSCRIBE_QUEUE_NAME)
logging.info("queue mode:{}".format('mns'))
logging.info('queue: {}'.format(queue.get_attributes().queue_name))
# TODO: 改成LongPoll更好
while True:
handler_func = None
handle_success = False
#读取消息
try:
recv_msg = queue.receive_message(WAIT_SECONDS)
logging.info(
"Receive Message Succeed! ReceiptHandle:%s MessageBody:%s MessageID:%s"
% (recv_msg.receipt_handle, recv_msg.message_body,
recv_msg.message_id))
# 处理消息(consume)
data = json.loads(recv_msg.message_body)
message_name = data['name']
handler_func = handler_register.find_message_handler(
message_name)
if handler_func:
try:
response = handler_func(data['data'], recv_msg)
logging.info("service response: {}".format(response))
handle_success = True
#只有正常才能删除消息，否则消息仍然在队列中
try:
time.sleep(3)
queue.delete_message(recv_msg.receipt_handle)
logging.debug(
"Delete Message Succeed! ReceiptHandle:%s" %
recv_msg.receipt_handle)
except MNSExceptionBase, e:
logging.debug(
"Delete Message Fail! Exception:%s\n" % e)
except:
logging.info(u"Service Exception: {}".format(
unicode_full_stack()))
else:
#TODO: 这里是否需要删除消息？
try:
queue.delete_message(recv_msg.receipt_handle)
logging.debug(
"Delete Message Succeed! ReceiptHandle:%s" %
recv_msg.receipt_handle)
except MNSExceptionBase, e:
logging.debug("Delete Message Fail! Exception:%s\n" %
e)
logging.info(u"Error: no such service found : {}".format(
message_name))
except MNSExceptionBase as e:
if e.type == "QueueNotExist":
logging.debug(
"Queue not exist, please create queue before receive message."
)
break
elif e.type == "MessageNotExist":
logging.debug("Queue is empty! Waiting...")
else:
logging.debug("Receive Message Fail! Exception:%s\n" % e)
time.sleep(SLEEP_SECONDS)
continue
except Exception as e:
print u"Exception: {}".format(unicode_full_stack())
finally:
if handler_func:
message = {
'message_id': recv_msg.message_id,
'message_body_md5': '',
'data': args,
'topic_name': '',
'msg_name': message_name,
'handel_success': handle_success
}
watchdog.info(message, log_type='MNS_RECEIVE_LOG')
return
## AccessKeySecret 阿里云官网获取
## Endpoint 阿里云消息和通知服务官网获取, Example: http://$AccountId.mns.cn-hangzhou.aliyuncs.com
## WARNING： Please do not hard code your accessId and accesskey in next line.(more information: https://yq.aliyun.com/articles/55947)
accessKeyId = parser.get("Base", "AccessKeyId")
accessKeySecret = parser.get("Base", "AccessKeySecret")
endpoint = parser.get("Base", "Endpoint")
securityToken = ""
if parser.has_option("Optional", "SecurityToken") and parser.get(
"Optional", "SecurityToken") != "$SecurityToken":
securityToken = parser.get("Optional", "SecurityToken")
#初始化my_account
my_account = Account(endpoint, accessKeyId, accessKeySecret, securityToken)
##############Queue 相关操作#####################
my_queue = my_account.get_queue(
"MyQueue-%s" % time.strftime("%y%m%d-%H%M%S", time.localtime()))
#创建队列
## message被receive后，持续不可消费的时间 100秒
## message body的最大长度 10240Byte
## message最长存活时间 3600秒
## 新message可消费的默认延迟时间 10秒
## receive message时，长轮询时间 20秒
queue_meta = QueueMeta()
queue_meta.set_visibilitytimeout(100)
queue_meta.set_maximum_message_size(10240)
queue_meta.set_message_retention_period(3600)
queue_meta.set_delay_seconds(10)
queue_meta.set_polling_wait_seconds(20)
queue_meta.set_logging_enabled(True)
try:
self.__tmp_access_id = response_body.get("MessageTokenDTO").get(
"AccessKeyId")
self.__expire_time = response_body.get("MessageTokenDTO").get(
"ExpireTime")
self.__token = response_body.get("MessageTokenDTO").get(
"SecurityToken")
print "finsh refresh token..."
# 初始化 my_account, my_queue
token = Token()
token.refresh()
my_account = Account(endpoint, token.get_tmp_access_id(),
token.get_tmp_access_key(), token.get_token())
my_queue = my_account.get_queue(qname)
# my_queue.set_encoding(False)
# 循环读取删除消息直到队列空
# receive message请求使用long polling方式，通过wait_seconds指定长轮询时间为3秒
## long polling 解析:
### 当队列中有消息时，请求立即返回；
### 当队列中没有消息时，请求在MNS服务器端挂3秒钟，在这期间，有消息写入队列，请求会立即返回消息，3秒后，请求返回队列没有消息；
wait_seconds = 3
print "%sReceive And Delete Message From Queue%s\nQueueName:%s\nWaitSeconds:%s\n" % (
10 * "=", 10 * "=", qname, wait_seconds)
while True:
# 读取消息
try:
# token过期是否需要刷新
class Runner(object):
def __init__(self):
# 读取邮箱信息
self.db = Database(
'mysql+mysqldb://spider_admin:'
'[email protected]:'
'3306/spider?charset=utf8',
encoding='utf-8')
# 初始化数据库
self.db1 = Database(
'mysql+mysqldb://%s:%s@%s:%s/%s?charset=utf8' %
(MYSQL_USER, MYSQL_PASSWD, MYSQL_HOST, MYSQL_PORT, MYSQL_DB),
encoding='utf-8')
# 初始化MNS
self.mns_account = Account(MNS_ENDPOINT, MNS_ACCID, MNS_ACCKEY,
MNS_TOKEN)
self.mns_client = self.mns_account.get_queue(MNS_QUEUE)
self.logger = Logger.timed_rt_logger()
self.queue = RedisQueue('t_mail_by_date',
host=REDIS_HOST,
port=REDIS_PORT)
self.robot = DingDingRobot(
access_token="eb749abfe9080a69da6524b77f589b8f6ddbc"
"c182c7a41bf095b095336edb0a1")
def deliver(self, subject):
try:
if '51job.com' in subject:
source = "FIVE_ONE"
deliverJobName = re.search('(?<=申请贵公司).*(?=（)',
subject).group().strip()
subject_ = subject.replace('（', '(').replace('）', ')')
deliverCity = re.findall('(?<=\().*?(?=\))',
subject_)[-1].strip()
elif '58.com' in subject:
# (58.com)应聘贵公司供应链管理8k六险一金-北京 朝阳-刘春雨
source = "FIVE_EIGHT"
deliverJobName = re.search('(?<=应聘贵公司).*?(?=-)',
subject).group().strip()
deliverCity = re.findall('(?<=-).*?(?=\s+)',
subject)[-1].strip()
elif 'Zhaopin.com' in subject:
source = "ZHI_LIAN"
deliverJobName = re.search('(?<=应聘).*?(?=-)',
subject).group().strip()
deliverCity = re.findall('(?<=-).*?(?=-)', subject)[-1].strip()
else:
return None, None, None
return source, deliverJobName, deliverCity
except:
return None, None, None
def save2db(self, source, email, subject, content, emailJobType, emailCity,
deliverJobName, deliverJobCity, trackId):
try:
createTime = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
data = {
'source': source,
'email': email,
'subject': subject,
'content': content,
'emailJobType': emailJobType,
'emailCity': emailCity,
'createBy': email,
'updateBy': 'PYTHON-RECEIVER',
'deliverJobName': deliverJobName,
'deliverJobCity': deliverJobCity,
'createTime': createTime,
'trackId': trackId
}
try:
self.db1.session.execute(ResumeRaw.__table__.insert(), data)
self.db1.session.commit()
except Exception as e:
self.db1.session.rollback()
self.logger.exception("入库失败, 回滚： %s" % e)
self.robot.send_text("邮箱采集，入库失败： %s" % email)
time.sleep(600)
return
sql_id = self.db1.session.execute(
"select last_insert_id()").first()[0]
# print(sql_id)
self.logger.info("入库成功. %s" % str(sql_id))
return sql_id
except Exception as e:
self.logger.exception("入库失败： %s" % e)
return
def save2mns(self, sql_id, source, content, emailJobType, emailCity,
deliverJobName, deliverJobCity, trackId):
try:
# resume_uuid = uuid.uuid1()
kafka_data = {
"channelType": "WEB",
"content": {
"content": content,
"id": sql_id,
"createBy": "python",
"createTime": int(time.time() * 1000),
"ip": '',
'emailJobType': emailJobType,
'emailCity': emailCity,
'deliverJobName': deliverJobName,
'deliverJobCity': deliverJobCity,
"resumeSubmitTime": '',
"resumeUpdateTime": '',
"source": source,
"trackId": str(trackId),
"avatarUrl": '',
},
"interfaceType": "PARSE",
"resourceDataType": "RAW",
"resourceType": "RESUME_EMAIL",
"source": source,
"trackId": str(trackId),
"callSystemID": project_name,
"traceID": str(trackId)
}
dumps = json.dumps(kafka_data, ensure_ascii=False)
buf = StringIO()
f = gzip.GzipFile(mode='wb', fileobj=buf)
f.write(dumps)
f.close()
msg_body = base64.b64encode(buf.getvalue())
msg = Message(msg_body)
self.mns_client.send_message(msg)
self.logger.info("推送mns成功. %s" % str(sql_id))
except Exception as e:
self.logger.exception("推送mns失败: %s" % e)
def producer(self):
self.queue.clean()
# res = self.db.session.query(TEmail).filter_by(valid=1).all()
res = self.db.session.query(TEmail).filter(
TEmail.email.in_((
# '*****@*****.**',
# '*****@*****.**',
# '*****@*****.**',
# '*****@*****.**',
# '*****@*****.**',
# '*****@*****.**',
'*****@*****.**',
# '*****@*****.**',
# '*****@*****.**',
)))
if not self.queue.empty():
self.logger.info("剩余%s个企业邮箱未处理！" % str(self.queue.qsize()))
sys.exit()
# self.queue.clean()
for item in res:
self.queue.put(
(item.pop3_host.encode('utf-8'), item.username.encode('utf-8'),
item.password.encode('utf-8'),
item.position_type.encode('utf-8'),
item.preferred_city.encode('utf-8'),
item.source.encode('utf-8')))
self.logger.info("总计加载%s个企业邮箱！" % str(self.queue.qsize()))
def consumer(self):
while True:
try:
if not self.queue.empty():
host, username, password, position_type, preferred_city, \
source1 = eval(self.queue.get())
# print(host, username, password)
imap = MailReceiver(host, username, password)
try:
imap.server.login(username, password)
self.logger.info("%s: 登录成功 <%s>" %
(username, self.queue.qsize()))
except Exception as e:
if "LOGIN Login error" in e:
self.robot.send_text("邮箱登录异常： %s %s %s" %
(username, password, host))
self.logger.exception(
"%s: 登录失败 %s <%s>" %
(username, e, self.queue.qsize()))
continue
try:
folder_list = imap.folder_list()
except Exception as e:
self.logger.exception("%s: 邮件箱加载失败 %s" % (username, e))
continue
if not folder_list:
continue
# print(folder_list)
if "Deleted Messages" in folder_list:
delete_folder = u"Deleted Messages"
elif "已删除邮件" in folder_list:
delete_folder = u"已删除邮件"
else:
delete_folder = u"已删除"
self.logger.info("%s: 开始收取邮件." % username)
today = datetime.datetime.now()
days = 180
for day in xrange(1, days):
date = today - datetime.timedelta(days=day)
fmt_date = datetime.datetime.strftime(date, "%d-%b-%Y")
self.logger.info('%s开始收取：%s 邮件' % (username, fmt_date))
try:
message_list, msg_data = imap.message_list(
search_='ON %s' % fmt_date)
except Exception as e:
self.logger.exception("%s: 邮件list加载失败 %s" %
(username, e))
continue
if not msg_data:
self.logger.warning("%s: 没有收到新邮件." % username)
continue
self.logger.info("%s: 收到%s个新邮件" %
(username, str(len(message_list))))
# print(message_list)
for msg_id, message in msg_data.items():
res = imap.get_content(message)
if not res:
# message_list.append(message)
continue
try:
subject = res.get('subject').encode(
'utf-8', 'ignore')
except UnicodeDecodeError:
subject = res.get('subject')
content = res.get('body')
emoji_pattern = re.compile(
u"(\ud83d[\ude00-\ude4f])|(\ud83c[\udf00-\uffff])|(\ud83d[\u0000-\uddff])|(\ud83d[\ude80-\udeff])|(\ud83c[\udde0-\uddff])+",
flags=re.UNICODE)
content = emoji_pattern.sub(r'', content)
source, deliverJobName, deliverJobCity = self.deliver(
subject)
if not deliverJobName:
self.logger.warning(
"%s: 收到非期待邮件. subject: %s" %
(username, subject))
try:
imap.delete_message([msg_id],
delete_folder)
self.logger.info(
"%s: 非期待邮件<%s>成功转移到“已删除”" %
(username, msg_id))
except Exception as e:
self.logger.exception(
"%s: 非期待邮件<%s>转移到“已删除”失败 %s" %
(username, msg_id, e))
continue
self.logger.info(
"%s\t%s\t%s\t" %
(source, deliverJobName, deliverJobCity))
trackId = uuid.uuid4()
sql_id = self.save2db(source, username, subject,
content, position_type,
preferred_city,
deliverJobName,
deliverJobCity, trackId)
if not sql_id:
continue
self.save2mns(sql_id, source.encode('utf-8'),
content.encode('utf-8'),
position_type, preferred_city,
deliverJobName, deliverJobCity,
trackId)
try:
imap.delete_message([msg_id], delete_folder)
self.logger.info("%s: 邮件<%s>成功转移到“已删除”" %
(username, msg_id))
except Exception as e:
self.logger.exception(
"%s: 邮件<%s>转移到“已删除”失败 %s" %
(username, msg_id, e))
imap.close()
# else:
# self.logger.info("finished!")
# break
except Exception as e:
self.logger.exception(e)
#coding=utf8
import sys
import os
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)) + "/..")
import time
from sample_common import MNSSampleCommon
from mns.account import Account
from mns.queue import *
#从sample.cfg中读取基本配置信息
## WARNING： Please do not hard code your accessId and accesskey in next line.(more information: https://yq.aliyun.com/articles/55947)
accid,acckey,endpoint,token = MNSSampleCommon.LoadConfig()
#初始化 my_account, my_queue
my_account = Account(endpoint, accid, acckey, token)
queue_name = sys.argv[1] if len(sys.argv) > 1 else "MySampleQueue"
my_queue = my_account.get_queue(queue_name)
#创建队列, 具体属性请参考mns/queue.py中的QueueMeta结构
queue_meta = QueueMeta()
try:
queue_url = my_queue.create(queue_meta)
print "Create Queue Succeed! QueueName:%s\n" % queue_name
except MNSExceptionBase, e:
if e.type == "QueueAlreadyExist":
print "Queue already exist, please delete it before creating or use it directly."
sys.exit(0)
print "Create Queue Fail! Exception:%s\n" % e
## AccessKeySecret 阿里云官网获取
## Endpoint 阿里云消息和通知服务官网获取, Example: http://$AccountId.mns.cn-hangzhou.aliyuncs.com
## WARNING： Please do not hard code your accessId and accesskey in next line.(more information: https://yq.aliyun.com/articles/55947)
accessKeyId = parser.get("Base", "AccessKeyId")
accessKeySecret = parser.get("Base", "AccessKeySecret")
endpoint = parser.get("Base", "Endpoint")
securityToken = ""
if parser.has_option("Optional", "SecurityToken") and parser.get("Optional", "SecurityToken") != "$SecurityToken":
securityToken = parser.get("Optional", "SecurityToken")
#初始化my_account
my_account = Account(endpoint, accessKeyId, accessKeySecret, securityToken)
##############Queue 相关操作#####################
my_queue = my_account.get_queue("MyQueue-%s" % time.strftime("%y%m%d-%H%M%S", time.localtime()))
#创建队列
## message被receive后，持续不可消费的时间 100秒
## message body的最大长度 10240Byte
## message最长存活时间 3600秒
## 新message可消费的默认延迟时间 10秒
## receive message时，长轮询时间 20秒
queue_meta = QueueMeta()
queue_meta.set_visibilitytimeout(100)
queue_meta.set_maximum_message_size(10240)
queue_meta.set_message_retention_period(3600)
queue_meta.set_delay_seconds(10)
queue_meta.set_polling_wait_seconds(20)
queue_meta.set_logging_enabled(True)
try:
class MNSBaseReceiptSvc(ABC):
""" 回执订阅 """
wait_seconds = 3 # 长轮询时间为3秒
sleep_seconds = 15 # 3秒后，请求返回队列没有消息，sleep间隔
# ("InvalidArgument" "The value of numofmessages should between 1 and 16")
batch_count = 10 # 每次接收数据量
def __init__(self, q_cfg):
self.queue_cfg = q_cfg
self.token = DYToken(q_cfg)
self.account, self.queue = None, None
@property
def svc_name(self):
name = f'{self.__class__.__doc__} [{self.__class__.__name__}]'
return name
def token_and_queue_refresh(self):
""" Token过期刷新 """
if not self.token.need_refresh:
return
self.token.refresh()
if self.account:
self.account.mns_client.close_connection()
self.account = Account(endpoint, self.token.tmp_access_id, self.token.tmp_access_key, self.token.token)
self.queue = self.account.get_queue(self.queue_cfg.queue_name)
logger.info(f'{self.token.tk_desc} account and queue ready')
def handle(self, dic):
""" 业务处理 """
raise NotImplementedError
def handle_msg(self, recv_msg):
""" 单个消息处理 """
assert isinstance(recv_msg, Message)
body = json.loads(recv_msg.message_body)
receipt_handle = recv_msg.receipt_handle
message_id, dequeue_count = recv_msg.message_id, recv_msg.dequeue_count
logger.info(f'{self.svc_name} Receive Message {message_id} {dequeue_count} {receipt_handle}\n{body}')
is_ok = self.handle(body)
return is_ok
def start_logger(self):
logger.info(
f'\n{self.svc_name}\n'
f'{16 * "="} Receive And Delete Message From Queue {16 * "="}\n'
f'QueueName: {self.queue_cfg.queue_name}\n'
f'MessageType: {self.token.tk_desc}\n'
f'WaitSeconds: {self.wait_seconds}\n'
f'SleepSeconds: {self.sleep_seconds}\n'
)
def run(self):
self.start_logger()
while True:
receipt_handles = []
try:
self.token_and_queue_refresh()
recv_msgs = self.queue.batch_receive_message(self.batch_count, self.wait_seconds)
for recv_msg in recv_msgs:
is_ok = self.handle_msg(recv_msg)
if is_ok: # 处理成功才删除消息
receipt_handles.append(recv_msg.receipt_handle)
logger.info(f'{self.svc_name} Handle MessageID: {recv_msg.message_id} done {is_ok}')
except MNSExceptionBase as mns_exc:
if mns_exc.type == 'QueueNotExist':
logger.warning(f'{self.svc_name} Queue not exist, please create queue.')
break
elif mns_exc.type == 'MessageNotExist':
logger.info(f'{self.svc_name} Queue is empty! sleep {self.sleep_seconds}s')
time.sleep(self.sleep_seconds)
continue
logger.exception(f'{self.svc_name} Receive Message Fail! Exception: {mns_exc}')
break
except Exception as exc:
logger.warning(f'{self.svc_name} error {exc}')
raise
logger.info(f'{self.svc_name} Receive Message Count {len(receipt_handles)}')
if len(receipt_handles) == 0:
continue
try: # 删除消息
assert isinstance(self.queue, Queue)
self.queue.batch_delete_message(receipt_handles)
logger.info(f'{self.svc_name} Delete Message Succeed! ReceiptHandles: {receipt_handles}')
except MNSExceptionBase as exc:
logger.warning(f'{self.svc_name} Delete Message Fail! Exception: {exc}')
raise RuntimeError(f'{self.svc_name} break!')
class AliAPI:
def __init__(self):
self.client = AcsClient(app.config.get("ALI_ACCESS_KEY"),
app.config.get("ALI_ACCESS_SECRET"),
app.config.get("BASE_REGION"))
self.account = Account(app.config.get("MNS_ENDPOINT"),
app.config.get("ALI_ACCESS_KEY"),
app.config.get("ALI_ACCESS_SECRET"), "")
self.wait_seconds = 10
self.kube = Kubernetes()
def reboot_ecs(self, ecs_id):
"""
reboot ecs
:param ecs_id:
:return:
"""
try:
request = RebootInstanceRequest()
request.set_accept_format('json')
request.set_InstanceId(ecs_id)
response = self.client.do_action_with_exception(request)
logs.info("reboot ecs {} successed result is {}".format(
ecs_id, response))
return True
except ServerException as exc:
logs.info("reboot ecs failed msg is {}".format(exc))
def set_ecs(self, ecs_id, ecs_hostname, ecs_instancename):
"""
set ecs hostname
:param ecs_id:
:param ecs_hostname:
:param ecs_instancename:
:return:
"""
try:
request = ModifyInstanceAttributeRequest()
request.set_accept_format('json')
request.set_InstanceId(ecs_id)
request.set_InstanceName(ecs_instancename)
response = self.client.do_action_with_exception(request)
logs.info(
"set ecs ecs_id is {} ecs_hostname is {} ecs_instancename is {} response is {}"
.format(ecs_id, ecs_hostname, ecs_instancename, response))
return True
except ServerException as exc:
logs.error("set ecs failed msg is {}".format(exc))
def get_ecs_info(self, ecs_id):
"""
get ecs ip and hostname
:param ecs_id:
:return:
"""
ip, hostname = None, None
try:
request = DescribeInstancesRequest()
request.set_accept_format('json')
request.set_InstanceIds([
ecs_id,
])
response = json.loads(
self.client.do_action_with_exception(request))
# 私有网络IP
ip = response.get("Instances").get("Instance")[0].get(
"VpcAttributes").get("PrivateIpAddress").get("IpAddress")[0]
hostname = response.get("Instances").get("Instance")[0].get(
"HostName")
except ServerException as exc:
logs.error("get ecs info failed msg is {}".format(exc))
finally:
return ip, hostname
def handlers(self, transiton, ecs_id, hostname, ip, scaling_groupname):
"""
响应伸缩组活动
:param transiton:
:param ecs_id:
:param hostname:
:param ip:
:param scaling_groupname:
:return:
"""
assert self.set_ecs(ecs_id, hostname,
hostname), AssertionError('set ecs failed.')
if ip and hostname:
if hasattr(self.kube, transiton.lower()):
result = getattr(self.kube, transiton.lower())(ip, hostname)
if not result:
logs.info("Reboot Ecs id is {} hostname is {}".format(
ecs_id, hostname))
self.reboot_ecs(ecs_id)
# 发送钉钉通知
self.kube.notice.send_mardown(
title='伸缩组活动',
text='#### 伸缩组活动\n'
'> \n\n'
'> - 伸缩组名称 {}\n'
'> - 动作类型 **{}**\n'
'> - ECS服务器名称 {}\n'
'> - ECS服务器IP {}\n'
'> ###### {} \n'.format(
scaling_groupname,
"缩容机器" if transiton == "SCALE_IN" else "扩容机器", hostname,
ip, time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
def receive_msg(self, queue_name=app.config.get("MNS_QUEUENAME")):
queue = self.account.get_queue(queue_name)
while 1:
try:
recv_msg = queue.receive_message(self.wait_seconds)
message_body = bytes.decode(recv_msg.message_body)
change_msg_vis = queue.change_message_visibility(
recv_msg.receipt_handle, 35)
self.delete_msg(change_msg_vis)
try:
message_body = json.loads(message_body)
except json.decoder.JSONDecodeError:
logs.debug("json decoder failes.")
# 只处理dict类型数据
if type(message_body) is dict:
logs.info("recv msg is {}".format(message_body))
transiton = message_body.get("content").get(
"lifecycleTransition")
if transiton not in storage.cache:
storage.cache[transiton] = []
scaling_groupname = message_body.get("content").get(
"scalingGroupName")
for ecs_id in message_body.get("content").get(
"instanceIds"):
if ecs_id not in storage.cache[transiton]:
storage.cache[transiton].append(ecs_id)
ip, _ = self.get_ecs_info(ecs_id)
if ip in storage.host_map:
hostname = storage.host_map[ip]
else:
# 拼接主机名
hostname = "{}{}".format(
app.config.get("HOSTNAME_PREFIX"),
("%03d" % get_counter()))
storage.host_map[ip] = hostname
# 线程处理
threading.Thread(target=self.handlers,
args=(transiton, ecs_id, hostname,
ip,
scaling_groupname)).start()
except MNSExceptionBase as e:
logs.error("recv msg failed {}".format(e))
def delete_msg(self,
change_msg_vis,
queue_name=app.config.get("MNS_QUEUENAME")):
queue = self.account.get_queue(queue_name)
queue.delete_message(change_msg_vis.receipt_handle)
logs.info("del msg {}".format(change_msg_vis.receipt_handle))
#! -*- coding: utf-8 -*-
import json
from mns.account import Account
from mns.mns_exception import MNSExceptionBase
from utils import post_slack
from config import QUEUE_NAME, END_POINT, ACCESS_KEY_ID, ACCESS_KEY_SECRET
account = Account(END_POINT, ACCESS_KEY_ID, ACCESS_KEY_SECRET)
queue = account.get_queue(QUEUE_NAME)
def run():
while True:
try:
recv_msg = queue.receive_message(wait_seconds=10)
message = json.loads(recv_msg.message_body)['message']
print message
if message['levelDescription'] == u'恢复正常':
color = 'good'
else:
color = 'danger'
post_slack(message['dimensions'], message['metricName'],
message['expression'], color)
queue.delete_message(recv_msg.receipt_handle)
except MNSExceptionBase, e:
print e.message
if __name__ == '__main__':
class ResumeExchange(Spider):
def __init__(self):
super(ResumeExchange, self).__init__()
self.mns_account = Account(MNS_ENDPOINT, MNS_ACCID, MNS_ACCKEY,
MNS_TOKEN)
self.mns_client = self.mns_account.get_queue(MNS_QUEUE)
def get_targetJobType(self, emailJobType):
for item in job_type_mapping:
if item.upper() in emailJobType:
return job_type_mapping[item]
return
def save2db(self, sql, data):
# conn = pymysql.Connect(**MYSQL_CONF_TEST)
conn = pymysql.Connect(**MYSQL_CONF)
cur = conn.cursor()
try:
cur.execute(sql, data)
conn.commit()
cur.execute("select last_insert_id()")
sql_id = cur.fetchone()[0]
self.logger.info("入库成功. %s" % sql_id)
return sql_id
except Exception as e:
self.logger.exception(e)
return
finally:
cur.close()
conn.close()
def save2mns(self, sql_id, data_for_mns, trackId):
try:
kafka_data = {
"content": data_for_mns,
"channelType": "WEB",
"interfaceType": "NORMAL",
"resourceDataType": "PARSED",
"resourceType": "RESUME_INBOX",
"source": data_for_mns['source'],
"traceID": trackId,
"callSystemID": PROJECT_NAME
}
dumps = json.dumps(kafka_data, ensure_ascii=False)
buf = StringIO()
f = gzip.GzipFile(mode='wb', fileobj=buf)
f.write(dumps)
f.close()
msg_body = base64.b64encode(buf.getvalue())
msg = Message(msg_body)
self.mns_client.send_message(msg)
self.logger.info("推送mns成功. %s" % str(sql_id))
except Exception as e:
self.logger.exception("推送mns失败: %s" % e)
def run(self, file_path):
try:
with open(file_path, 'r') as file:
self.logger.info("开始上载简历.")
for index, line in enumerate(file):
# self.process1(index=index,line=line)
self.process2(index=index, line=line)
except Exception as e:
self.logger.exception(e)
def process2(self, index, line):
try:
self.logger.info('当前处理的内如：%s %s' % (index, line))
name, mobile, gender, degree, age, work_exp, target_job_type, register_location = line.strip(
'\n').split('\t')
except ValueError:
self.logger.warning("该行读取错误: " + str(index))
return
now_year = datetime.datetime.now().year
birthday = ''
if age:
birthday = str(now_year - int(age)) + '-05-15'
uuid_ = str(uuid.uuid1())
data = (name, mobile, gender, degree, birthday, work_exp,
target_job_type, register_location,
datetime.datetime.strftime(datetime.datetime.now(),
'%Y-%m-%d'),
datetime.datetime.strftime(datetime.datetime.now(),
'%Y-%m-%d %H:%M:%S'),
datetime.datetime.strftime(datetime.datetime.now(),
'%Y-%m-%d %H:%M:%S'), uuid_)
sql = '''
INSERT INTO spider.resume_parsed
(name,mobile,gender,degree,birthday,workExperience,targetJobType,registerLocation,targetWorkLocation,workType,status,source,targetSalary,resumeUpdateTime,createTime,updateTime,trackId)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s,'西安','全职','正在找工作','RESUME_EXCHANGE','面议',%s,%s,%s,%s)
'''
sql_id = self.save2db(sql=sql, data=data)
self.logger.info('解析入库成功 %s' % sql_id)
data_for_mns = {
'id':
sql_id,
'name':
name,
'mobile':
mobile,
'gender':
gender,
'degree':
degree,
'birthday':
birthday,
'workExperience':
work_exp,
'targetJobType':
target_job_type,
'registerLocation':
register_location,
'targetWorkLocation':
'西安',
'workType':
'全职',
'status':
'正在找工作',
'source':
'RESUME_EXCHANGE',
'targetSalary':
'面议',
'resumeUpdateTime':
datetime.datetime.strftime(datetime.datetime.now(),
'%Y-%m-%d %H:%M:%S'),
'createTime':
datetime.datetime.strftime(datetime.datetime.now(),
'%Y-%m-%d %H:%M:%S'),
'updateTime':
datetime.datetime.strftime(datetime.datetime.now(),
'%Y-%m-%d %H:%M:%S'),
'trackId':
uuid_,
}
self.save2mns(sql_id, data_for_mns, uuid_)
self.logger.info('处理完毕 %s' % sql_id)
def process1(self, index, line):
try:
name, emailJobType, targetJobType, gender, birthday, mobile, \
workExperience, email, currentLocation, registerLocation, \
career, education, degree, targetSalary = line.strip(
'\n').split('\t')
except ValueError:
self.logger.warning("该行读取错误: " + str(index))
return
try:
emailJobType = emailJobType.replace('北京', '').replace('求职',
'').replace(
'/', ',')
targetJobType = self.get_targetJobType(emailJobType)
if not targetJobType:
targetJobType = emailJobType
birthday = birthday.replace('/', '-')
career_lst = []
if career:
for item in career.split('|$|'):
career_lst.append({"companyName": item})
career = json.dumps(career_lst, ensure_ascii=False)
education_lst = []
if education:
for item in education.split('|$|'):
education_lst.append({"schoolName": item})
education = json.dumps(education_lst, ensure_ascii=False)
now = datetime.datetime.now()
trackId = str(uuid.uuid1())
data = (name, emailJobType, targetJobType, gender, birthday,
mobile, workExperience, email, currentLocation,
registerLocation, career, education, degree, targetSalary,
now, '2017-11-01 00:00:00', trackId)
sql = """
INSERT INTO spider.resume_parsed(name, emailJobType,
targetJobType, gender, birthday, mobile, \
workExperience, email, currentLocation, registerLocation, \
career, education, degree, targetSalary, createTime,
updateTime, trackId,
createBy, source, workType, status) VALUES (%s, %s, %s,
%s, %s, %s, %s, %s, %s,
%s, %s, %s, %s, %s, %s, %s, %s, 'PYTHON',
'RESUME_EXCHANGE',
'全职', '正在找工作')
"""
sql_id = self.save2db(sql=sql, data=data)
if not sql_id:
return
data_for_mns = {
'id': sql_id,
'name': name,
'emailJobType': emailJobType,
'targetJobType': targetJobType,
'gender': gender,
'birthday': birthday,
'mobile': mobile,
'workExperience': workExperience,
'email': email,
'currentLocation': currentLocation,
'registerLocation': registerLocation,
'career': career_lst,
'education': education_lst,
'degree': degree,
'targetSalary': targetSalary,
'createTime':
datetime.datetime.strftime(now, '%Y-%m-%d %H:%M:%S'),
'updateTime': '2017-11-01 00:00:00',
'trackId': trackId,
'source': 'RESUME_EXCHANGE',
'workType': '全职',
'status': '正在找工作'
}
self.save2mns(sql_id, data_for_mns, trackId)
self.logger.info(
'%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s' %
(index, name, emailJobType, targetJobType, gender, birthday,
mobile, workExperience, email, currentLocation,
registerLocation, career, education, degree, targetSalary))
except Exception as e:
self.logger.exception(e)
return
def __init__(self):
# initialize my_account, my_queue
my_account = Account(settings.ALIYUN_MNS_END_POINT,
settings.ALIYUN_MNS_ACCESS_KEY_ID,
settings.ALIYUN_MNS_ACCESS_KEY_SECRET, "")
self.my_queue = my_account.get_queue(settings.ALIYUN_MNS_QUEUE_NAME)
DB_DATABASE = "greengrass"
# Read the basic configurations from sample.cfg
# WARNING: Please do not hard code your accessId and accesskey in next line.
# (more information: https://yq.aliyun.com/articles/55947)
accessKeyId, accesskeySecret, endpoint, token = MNSSampleCommon.LoadConfig()
# create the connect for IoT Kit
clt = client.AcsClient(accessKeyId, accesskeySecret, 'cn-shanghai')
# Init my_account, my_queue
my_account = Account(endpoint, accessKeyId, accesskeySecret, token)
queue_name = sys.argv[1] if len(sys.argv) > 1 else queueName
boolbase64 = False if len(
sys.argv) > 2 and sys.argv[2].lower() == "false" else True
my_queue = my_account.get_queue(queue_name)
my_queue.set_encoding(boolbase64)
# get the connection to Amazon Greengrass RDS
db = MySQLdb.connect(DB_HOST, DB_USERNAME, DB_PASSWORD, DB_DATABASE)
# Read and delete the message from queue until the queue is empty
# receive message uses long polling mode, specify the long loop time to 3 second through wait_seconds
# long polling parsing
# Return the message when there are messages
# When queue is empty, request will wait for 3 second on server side.
# Request will return the message when mesages are wrote to this queue during this time.
# After 3 second, there is not yet message in queue, request will return 'MessageNotExist'
wait_seconds = 5
print "%sReceive And Delete Message From Queue%s\nQueueName:%s\nWaitSeconds:%s\n" % (
class RtmpProxy:
def __init__(self):
self.set_logger()
#加载阿里云相关配置
self.mns_endpoint = get_config("MNS", "mns_endpoint")
self.mns_access_id = get_config("MNS", "mns_access_id")
self.mns_access_key = get_config("MNS", "mns_access_key")
self.mns_queue_name = get_config("MNS", "mns_queue_name")
self.oss_endpoint = get_config("OSS", "oss_endpoint")
self.oss_access_id = get_config("OSS", "oss_access_id")
self.oss_access_key = get_config("OSS", "oss_access_key")
self.oss_bucket_name = get_config("OSS", "oss_bucket_name")
#加载rtmp相关配置
self.max_stream_count = int(get_config("Base", "max_stream_count"))
self.max_retries = 3
#初始化 msn account, queue
self.mns_account = Account(self.mns_endpoint, self.mns_access_id,
self.mns_access_key)
self.mns_queue = self.mns_account.get_queue(self.mns_queue_name)
self.mns_queue.set_encoding(False)
self.publish_count = 0
self.publish_mutex = threading.Lock()
self.bucket = oss2.Bucket(
oss2.Auth(self.oss_access_id, self.oss_access_key),
self.oss_endpoint, self.oss_bucket_name)
def set_logger(self):
if not os.path.isdir('logs'):
os.mkdir('logs')
self.access_logger = init_logger('access',
'logs/rtmp_access.log',
fmt='%(asctime)-15s|%(message)s')
self.logger = init_logger('', 'logs/rtmp_proxy.log',
self.get_loglevel())
def get_loglevel(self):
log_level = get_config("Base", "log_level")
if log_level == "debug":
return logging.DEBUG
elif log_level == "info":
return logging.INFO
elif log_level == "warning":
return logging.WARNING
elif log_level == "error":
return logging.ERROR
return logging.INFO
def main_loop(self):
while True:
try:
while self.publish_count >= self.max_stream_count:
logging.error("exceed the max stream count, count: %d/%d" %
(self.publish_count, self.max_stream_count))
self.print_access_log('-', 'fail',
'exceed the max stream count')
time.sleep(5)
logging.info("waiting for receive message")
recv_msg = self.mns_queue.receive_message(3)
try:
self.process_msg(recv_msg)
except Exception, e:
logging.error("Process Message Fail! Exception:%s" % e)
except MNSExceptionBase, e:
if e.type == "QueueNotExist":
logging.error(
"Queue not exist, please create queue before receive message."
)
sys.exit(0)
elif e.type == "MessageNotExist":
logging.debug("Queue is empty!")
continue
else:
logging.warning("Receive Message Fail! Exception:%s" % e)
continue
except Exception, e:
logging.error("Receive Message Fail! Exception: %s" % (e))
continue
try:
self.mns_queue.delete_message(recv_msg.receipt_handle)
logging.debug("Delete Message Succeed! ReceiptHandle:%s" %
recv_msg.receipt_handle)
except MNSExceptionBase, e:
logging.error("Delete Message Fail! Exception:%s" % e)
class Runner(object):
def __init__(self):
# 读取邮箱信息
self.db = Database(
'mysql+mysqldb://spider_admin:'
'[email protected]:'
'3306/spider?charset=utf8',
encoding='utf-8')
# 初始化数据库
self.db1 = Database(
'mysql+mysqldb://%s:%s@%s:%s/%s?charset=utf8' %
(MYSQL_USER, MYSQL_PASSWD, MYSQL_HOST, MYSQL_PORT, MYSQL_DB),
encoding='utf-8')
# 初始化MNS
self.mns_account = Account(MNS_ENDPOINT, MNS_ACCID, MNS_ACCKEY,
MNS_TOKEN)
self.mns_client = self.mns_account.get_queue(MNS_QUEUE)
self.logger = Logger.timed_rt_logger()
self.queue = RedisQueue('t_mail', host=REDIS_HOST, port=REDIS_PORT)
# self.h = RedisHash('t_mail_run_status', host=REDIS_HOST,
# port=REDIS_PORT)
self.robot = DingDingRobot(
access_token="eb749abfe9080a69da6524b77f589b8f6ddbc"
"c182c7a41bf095b095336edb0a1")
def deliver(self, subject):
try:
if '51job.com' in subject:
source = "FIVE_ONE"
deliverJobName = re.search('(?<=申请贵公司).*(?=（)',
subject).group().strip()
subject_ = subject.replace('（', '(').replace('）', ')')
deliverCity = re.findall('(?<=\().*?(?=\))',
subject_)[-1].strip()
elif '58.com' in subject:
# (58.com)应聘贵公司供应链管理8k六险一金-北京 朝阳-刘春雨
source = "FIVE_EIGHT"
deliverJobName = re.search('(?<=应聘贵公司).*?(?=-)',
subject).group().strip()
deliverCity = re.findall('(?<=-).*?(?=\s+)',
subject)[-1].strip()
elif 'Zhaopin.com' in subject:
source = "ZHI_LIAN"
deliverJobName = re.search('(?<=应聘).*?(?=-)',
subject).group().strip()
deliverCity = re.findall('(?<=-).*?(?=-)', subject)[-1].strip()
elif '来自猎聘网' in subject:
print(subject)
source = "LIE_PIN"
deliverJobName = re.search('(?<=【).*?(?=_)',
subject).group().strip()
deliverCity = re.search('(?<=_).*(?=】)',
subject).group().strip()
else:
return None, None, None
return source, deliverJobName, deliverCity
except:
return None, None, None
def save2db(self, source, email, subject, content, emailJobType, emailCity,
deliverJobName, deliverJobCity, trackId):
try:
createTime = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
data = {
'source': source,
'email': email,
'subject': subject,
'content': content,
'emailJobType': emailJobType,
'emailCity': emailCity,
'createBy': email,
'updateBy': 'PYTHON-RECEIVER',
'deliverJobName': deliverJobName,
'deliverJobCity': deliverJobCity,
'createTime': createTime,
'trackId': trackId
}
try:
self.db1.session.execute(ResumeRaw.__table__.insert(), data)
self.db1.session.commit()
except Exception as e:
self.db1.session.rollback()
self.logger.exception("入库失败, 回滚： %s" % e)
self.robot.send_text("邮箱采集，入库失败： %s" % email)
time.sleep(600)
return
sql_id = self.db1.session.execute(
"select last_insert_id()").first()[0]
# print(sql_id)
self.logger.info("入库成功. %s" % str(sql_id))
return sql_id
except Exception as e:
self.logger.exception("入库失败： %s" % e)
return
def save2mns(self, sql_id, source, content, emailJobType, emailCity,
deliverJobName, deliverJobCity, trackId):
try:
resume_uuid = uuid.uuid1()
kafka_data = {
"channelType": "WEB",
"content": {
"content": content,
"id": sql_id,
"createBy": "python",
"createTime": int(time.time() * 1000),
"ip": '',
'emailJobType': emailJobType,
'emailCity': emailCity,
'deliverJobName': deliverJobName,
'deliverJobCity': deliverJobCity,
"resumeSubmitTime": '',
"resumeUpdateTime": '',
"source": source,
"trackId": str(resume_uuid),
"avatarUrl": '',
'trackId': trackId
},
"interfaceType": "PARSE",
"resourceDataType": "RAW",
"resourceType": "RESUME_EMAIL",
"source": source,
"trackId": trackId,
"traceID": trackId,
"callSystemID": project_name
}
dumps = json.dumps(kafka_data, ensure_ascii=False)
buf = StringIO()
f = gzip.GzipFile(mode='wb', fileobj=buf)
f.write(dumps)
f.close()
msg_body = base64.b64encode(buf.getvalue())
msg = Message(msg_body)
self.mns_client.send_message(msg)
self.logger.info("推送mns成功. %s" % str(sql_id))
except Exception as e:
self.logger.exception("推送mns失败: %s" % e)
def producer(self):
# self.queue.clean()
res = self.db.session.query(TEmail).filter_by(valid=1).all()
# res = self.db.session.query(TEmail).filter(TEmail.email.in_((
# '*****@*****.**',
# )))
if not self.queue.empty():
self.logger.info("剩余%s个企业邮箱未处理！" % str(self.queue.qsize()))
sys.exit()
# self.queue.clean()
for item in res:
self.queue.put(
(item.pop3_host.encode('utf-8'), item.username.encode('utf-8'),
item.password.encode('utf-8'),
item.position_type.encode('utf-8'),
item.preferred_city.encode('utf-8'),
item.source.encode('utf-8')))
self.logger.info("总计加载%s个企业邮箱！" % str(self.queue.qsize()))
@retry(stop_max_attempt_number=3)
@cls_catch_exception
def deal_with_messages(self, imap, message_list, username, delete_folder,
position_type, preferred_city):
message_list, msg_data = imap.download_message_list(message_list)
if not msg_data:
self.logger.warning("%s: 没有收到新邮件." % username)
return
self.logger.info("%s: 收到%s个新邮件" % (username, str(len(message_list))))
# print(message_list)
for msg_id, message in msg_data.items():
res = imap.get_content(message)
if not res:
# message_list.append(message)
continue
try:
subject = res.get('subject').encode('utf-8', 'ignore')
except UnicodeDecodeError:
subject = res.get('subject')
content = res.get('body')
content = remove_emoji(content)
source, deliverJobName, deliverJobCity = self.deliver(subject)
if not deliverJobName:
self.logger.warning("%s: 收到非期待邮件. subject: %s" %
(username, subject))
try:
imap.delete_message([msg_id], delete_folder)
self.logger.info("%s: 非期待邮件<%s>成功转移到“已删除”" %
(username, msg_id))
except Exception as e:
self.logger.exception("%s: 非期待邮件<%s>转移到“已删除”失败 %s" %
(username, msg_id, e))
continue
self.logger.info("%s\t%s\t%s\t" %
(source, deliverJobName, deliverJobCity))
trackId = str(uuid.uuid4())
sql_id = self.save2db(source,
username,
subject,
content,
position_type,
preferred_city,
deliverJobName,
deliverJobCity,
trackId=trackId)
if not sql_id:
continue
self.save2mns(sql_id, source.encode('utf-8'),
content.encode('utf-8'), position_type,
preferred_city, deliverJobName, deliverJobCity,
trackId)
try:
imap.delete_message([msg_id], delete_folder)
self.logger.info("%s: 邮件<%s>成功转移到“已删除”" % (username, msg_id))
except Exception as e:
self.logger.exception("%s: 邮件<%s>转移到“已删除”失败 %s" %
(username, msg_id, e))
def inactive_account(self, username):
global RUN_STATUS
RUN_STATUS[username] = ''
self.logger.info('该帐号[%s]运行完毕.' % username)
def consumer(self):
while True:
try:
if not self.queue.empty():
host, username, password, position_type, preferred_city, \
source1 = eval(self.queue.get())
# print(host, username, password)
try:
if RUN_STATUS.get(username, ''):
self.logger.warning('该帐号[%s]当前正在运行.' % username)
continue
RUN_STATUS[username] = 1
self.logger.info('将该帐号[%s]标识为正在运行.' % username)
imap = MailReceiver(host, username, password)
try:
imap.server.login(username, password)
self.logger.info("%s: 登录成功 <%s>" %
(username, self.queue.qsize()))
except Exception as e:
if "LOGIN Login error" in e:
self.robot.send_text(
"邮箱登录异常： %s %s %s" %
(username, password, host))
self.logger.exception(
"%s: 登录失败 %s <%s>" %
(username, e, self.queue.qsize()))
self.inactive_account(username)
continue
try:
folder_list = imap.folder_list()
except Exception as e:
self.logger.exception("%s: 邮件箱加载失败 %s" %
(username, e))
self.inactive_account(username)
continue
if not folder_list:
self.inactive_account(username)
continue
# print(folder_list)
if "Deleted Messages" in folder_list:
delete_folder = u"Deleted Messages"
elif "已删除邮件" in folder_list:
delete_folder = u"已删除邮件"
else:
delete_folder = u"已删除"
self.logger.info("%s: 开始收取邮件." % username)
try:
message_list = imap.message_list()
except Exception as e:
self.logger.exception("%s: 邮件list加载失败 %s" %
(username, e))
self.inactive_account(username)
continue
if len(message_list) > WARNING_NUM:
self.robot.send_text("%s邮件数量超过告警阈值：%s" %
(username, WARNING_NUM))
if len(message_list) > RECEIVER_NUM:
tmp_lst = []
for item in xrange(0, len(message_list),
RECEIVER_NUM):
tmp_lst.append(message_list[item:item +
RECEIVER_NUM])
self.logger.info(
"%s: 共收到邮件%s封，开始进行分组收取，总计%s组" %
(username, len(message_list), len(tmp_lst)))
for index, tmp in enumerate(tmp_lst):
self.logger.info("%s: 当前开始收取第%s组邮件." %
(username, index + 1))
self.deal_with_messages(
imap, tmp, username, delete_folder,
position_type, preferred_city)
else:
self.logger.info("%s: 共收到邮件%s封。" %
(username, len(message_list)))
self.deal_with_messages(imap, message_list,
username, delete_folder,
position_type,
preferred_city)
imap.close()
self.inactive_account(username)
except Exception as e:
self.inactive_account(username)
self.logger.exception(e)
# else:
# self.logger.info("finished!")
# break
except Exception as e:
global RUN_STATUS
RUN_STATUS = {}
self.logger.exception(e)
def get_mns_client():
mns_account = Account(common_settings.ENDPOINT, common_settings.ACCID,
common_settings.ACCKEY, common_settings.TOKEN)
mns_client = mns_account.get_queue(common_settings.MNS_QUEUE)
return mns_client
class MnsClient():
"""mns client
"""
def __init__(self, cfg_file):
self.cfg_file = cfg_file
self.init_from_cfg()
self.last_recv_msg = None
def init_from_cfg(self):
parser = ConfigParser.ConfigParser()
parser.read(self.cfg_file)
for sec,op in required_ops:
if not parser.has_option(sec, op):
raise Exception("mnsclient load config file failed")
self.mns_account = Account(parser.get(BASE, END_POINT), parser.get(BASE, ACCESS_ID), parser.get(BASE, ACCESS_KEY_SECRET))
self.mns_queue = self.mns_account.get_queue(parser.get(BASE, QUEUE_NAME))
self.wait_seconds = int(parser.get(BASE, WAIT_SECOND))
def recv_msg(self, visible_timeout=60):
"""
@return: None means recv msg failed, otherwise, return the msg_body(means the msg that you send)
"""
ret_flag = None
try:
_msg = self.mns_queue.receive_message(self.wait_seconds)
try:
change_msg_vis = self.mns_queue.change_message_visibility(_msg.receipt_handle, visible_timeout)
logger.debug("Change Message Visibility Succeed!\nReceiptHandle:%s\nNextVisibleTime:%s\n\n" %
(change_msg_vis.receipt_handle, change_msg_vis.next_visible_time))
ret_flag = _msg.message_body
self.last_recv_msg = change_msg_vis
except MNSExceptionBase, e:
logger.warn("Change Message Visibility Fail!\nException:%s\n\n" % e)
except MNSExceptionBase as e:
if e.type == 'MessageNotExist':
logger.debug("no message left in queue [%s]" %self.mns_queue.queue_name)
else:
logger.warn("Receive Message Fail!\nException:%s\n\n" % e)
return ret_flag
def del_msg(self, retry=3):
"""
@return:
"""
num = retry
ret_flag = False
if self.last_recv_msg is None:
logger.debug("last recv msg is None, nothing to do")
return True
while num > 0:
try:
self.mns_queue.delete_message(self.last_recv_msg.receipt_handle)
logger.debug("Delete Message Succeed.\n\n")
ret_flag = True
self.last_recv_msg = None
return ret_flag
except MNSExceptionBase, e:
logger.warn("Del Message from %s Fail!\nException:%s\n\n, retry again" % (self.mns_queue.queue_name,e))
time.sleep(1.0)
logger.warn("try to delete Message %s time still Failed, message will be push back to queue [%s]" %(retry, self.mns_queue.queue_name))
return ret_flag
