def my_send_msg(num): # 从sample.cfg中读取基本配置信息 ## WARNING: Please do not hard code your accessId and accesskey in next line.(more information: https://yq.aliyun.com/articles/55947) accid, acckey, endpoint, token = MNSSampleCommon.LoadConfig() # 初始化 my_account, my_queue my_account = Account(endpoint, accid, acckey, token) queue_name = "MyQueue1" my_queue = my_account.get_queue(queue_name) try: # msg_body = "I am test message %s." % i msg_body = num msg = Message(msg_body) re_msg = my_queue.send_message(msg) # print # "Send Message Succeed! MessageBody:%s MessageID:%s" % (msg_body, re_msg.message_id) except MNSExceptionBase, e: if e.type == "QueueNotExist": print "Queue not exist, please create queue before send message." sys.exit(0) print "Send Message Fail! Exception:%s\n" % e
def run_produce(): account = Account(MNS_ENDPOINT, ALI_ACCESSKEYID, ALI_ACCESSKEYSECRET) q = account.get_queue(QUEUE_NAME) queue_meta = QueueMeta() queue_meta.set_visibilitytimeout(100) queue_meta.set_maximum_message_size(10240) queue_meta.set_message_retention_period(3600) queue_meta.set_delay_seconds(10) queue_meta.set_polling_wait_seconds(20) queue_meta.set_logging_enabled(True) try: q_url = q.create(queue_meta) sys.stdout.write("Create Queue Succeed!\nQueueURL:%s\n\n" % q_url) except MNSExceptionBase as e: sys.stderr.write("Create Queue Fail!\nException:%s\n\n" % e) sys.exit(1) for i in range(1, 10): msg_body = "Test Message n. {} at {}".format( i, datetime.now().isoformat()) message = Message(msg_body) message.set_delayseconds(0) message.set_priority(10) try: send_msg = q.send_message(message) sys.stdout.write( "Send Message Succeed.\nMessageBody:%s\nMessageId:%s\nMessageBodyMd5:%s\n\n" % (msg_body, send_msg.message_id, send_msg.message_body_md5)) except MNSExceptionBase as e: sys.stderr.write("Send Message Fail!\nException:%s\n\n" % e) sys.exit(1)
def run_consume(): account = Account(MNS_ENDPOINT, ALI_ACCESSKEYID, ALI_ACCESSKEYSECRET) q = account.get_queue(QUEUE_NAME) while True: try: wait_seconds = 10 recv_msg = q.receive_message(wait_seconds) sys.stdout.write( "Receive Message Succeed! \ \nMessageId: %s\nMessageBodyMD5: %s \ \nMessageBody: %s\nDequeueCount: %s \ \nEnqueueTime: %s\nFirstDequeueTime: %s \ \nPriority: %s\nNextVisibleTime: %s \ \nReceiptHandle: %s\n\n" % (recv_msg.message_id, recv_msg.message_body_md5, recv_msg.message_body, recv_msg.dequeue_count, recv_msg.enqueue_time, recv_msg.first_dequeue_time, recv_msg.priority, recv_msg.next_visible_time, recv_msg.receipt_handle)) q.delete_message(recv_msg.receipt_handle) sys.stdout.write("Delete Message Succeed.\n\n") except MNSExceptionBase as e: sys.stderr.write("Receive Message Fail!\nException:%s\n\n" % e) break try: q.delete() sys.stdout.write("Delete Queue Succeed!\n\n") except MNSExceptionBase as e: sys.stderr.write("Delete Queue Fail!\nException:%s\n\n" % e) sys.exit(1)
class aliyun_mns_api(object): def __init__(self): """客户端初始化""" cf = ConfigParser.ConfigParser() cf.read("aliyun.conf") host = cf.get('AccessKey_mns',"Host") accessid = cf.get('AccessKey_mns',"Access_ID") accesskey = cf.get('AccessKey_mns',"Access_Key") self.cli = Account(host,accessid,accesskey) def sendmessagerequest(self,queue_name): """向指定mns发送message""" with open('SendMessage.txt','r') as f: for line in f: message = Message(line) re_msg = self.cli.get_queue(queue_name).send_message(message) print "Send Message Succeed! MessageBody:%s MessageID:%s" % (line,re_msg.message_id) def getlistqueuenames(self): """查询消息列队名字""" result = self.cli.list_queue() name_list = [] for name in result[0]: real_name = str(name).split('/')[-1] name_list.append(real_name) #print name_list return name_list
def __init__(self): # 读取邮箱信息 self.db = Database( 'mysql+mysqldb://spider_admin:' '[email protected]:' '3306/spider?charset=utf8', encoding='utf-8') # 初始化数据库 self.db1 = Database( 'mysql+mysqldb://%s:%s@%s:%s/%s?charset=utf8' % (MYSQL_USER, MYSQL_PASSWD, MYSQL_HOST, MYSQL_PORT, MYSQL_DB), encoding='utf-8') # 初始化MNS self.mns_account = Account(MNS_ENDPOINT, MNS_ACCID, MNS_ACCKEY, MNS_TOKEN) self.mns_client = self.mns_account.get_queue(MNS_QUEUE) self.logger = Logger.timed_rt_logger() self.queue = RedisQueue('t_mail_by_date', host=REDIS_HOST, port=REDIS_PORT) self.robot = DingDingRobot( access_token="eb749abfe9080a69da6524b77f589b8f6ddbc" "c182c7a41bf095b095336edb0a1")
def __init__(self): self.id = conf.access_key_id self.secret = conf.access_key_secret self.point = conf.queue_endpoint self.queue_name = conf.queue_name self.auth = oss2.Auth(conf.access_key_id, conf.access_key_secret) self.bucket = oss2.Bucket(self.auth, conf.oss_endpoint, conf.bucket_name) self.oss_img_path = conf.oss_img_path client = MongoClient(conf.mongo_host) client['admin'].authenticate('root', 'qwer1234') self.userdb = client['qp_account'] for index, subject in enumerate(_ALL_SUPPORT_SUBJECTS): db_name = 'qp_' + subject self.db[index] = client[db_name] logging.critical('connect mongodb success:{}'.format(conf.mongo_host)) if self.id is None or self.secret is None or self.point is None or self.queue_name is None: raise Exception('lose conf') try: self.account = Account(self.point, self.id, self.secret) self.queue = self.account.get_queue(self.queue_name) logging.critical('init_MNS success:') logging.critical(u'id:{}, point:{}, queue_name'.format( self.id, self.point, self.queue_name)) except: traceback.print_exc() raise Exception('connect MNS fail')
def rece_del_msg(queue_name, wait_seconds=3): """ 从指定的消息队列中获取消息,默认等待3s,每次获取1个 :param queue_name: 消息队列的名称 :param wait_seconds: 等待时长 :return: (None, ""): 队列不存在;(False, ""): 获取失败;(True, "***"): 获取成功 """ access_key_id, access_key_secret, end_point, security_token = config_parse() my_account = Account(end_point, access_key_id, access_key_secret, security_token) my_queue = my_account.get_queue(queue_name) try: receive_message = my_queue.receive_message(wait_seconds) except MNSExceptionBase as e: if e.type == "QueueNotExist": result = (None, "") elif e.type == "MessageNotExist": result = (True, "") else: result = (False, "") else: result = (True, receive_message.message_body) try: my_queue.delete_message(receive_message.receipt_handle) except Exception as e: result = (False, "") return result
def __init__(self): self.set_logger() #加载阿里云相关配置 self.mns_endpoint = get_config("MNS", "mns_endpoint") self.mns_access_id = get_config("MNS", "mns_access_id") self.mns_access_key = get_config("MNS", "mns_access_key") self.mns_queue_name = get_config("MNS", "mns_queue_name") self.oss_endpoint = get_config("OSS", "oss_endpoint") self.oss_access_id = get_config("OSS", "oss_access_id") self.oss_access_key = get_config("OSS", "oss_access_key") self.oss_bucket_name = get_config("OSS", "oss_bucket_name") #加载rtmp相关配置 self.max_stream_count = int(get_config("Base", "max_stream_count")) self.max_retries = 3 #初始化 msn account, queue self.mns_account = Account(self.mns_endpoint, self.mns_access_id, self.mns_access_key) self.mns_queue = self.mns_account.get_queue(self.mns_queue_name) self.mns_queue.set_encoding(False) self.publish_count = 0 self.publish_mutex = threading.Lock() self.bucket = oss2.Bucket( oss2.Auth(self.oss_access_id, self.oss_access_key), self.oss_endpoint, self.oss_bucket_name)
def insert_db(self, item): try: self.sqlalchemy_handle.session.execute( spider_search.ResumeRaw.__table__.insert(), item ) self.sqlalchemy_handle.session.commit() except Exception as e: self.sqlalchemy_handle.session.rollback() self.logger.exception("入库失败, 回滚: %s" % e) raise e sql_id = self.sqlalchemy_handle.session.execute( "select last_insert_id()").first()[0] self.logger.info("入库成功. %s" % str(sql_id)) try: resume_uuid = uuid.uuid1() mns_data = { "channelType": "WEB", "content": { "content": item['content'], "id": sql_id, "createBy": "python", "createTime": int(time.time() * 1000), "ip": '', 'emailJobType': item['emailJobType'], 'emailCity': item['emailCity'], 'deliverJobName': '', 'deliverJobCity': '', "resumeSubmitTime": '', "resumeUpdateTime": '', "source": item['source'], "trackId": str(resume_uuid), "avatarUrl": '', }, "interfaceType": "PARSE", "resourceDataType": "RAW", "resourceType": "RESUME_SEARCH", "source": item['source'], "trackId": resume_uuid, "traceID": resume_uuid, "callSystemID": 'five_one_python3_scrapy' } dumps = json.dumps(mns_data, ensure_ascii=False) buf = StringIO() f = gzip.GzipFile(mode='wb', fileobj=buf) f.write(dumps) f.close() msg_body = base64.b64encode(buf.getvalue()) msg = Message(msg_body) # 初始化MNS mns_account = Account(MNS_ENDPOINT, MNS_ACCID, MNS_ACCKEY, MNS_TOKEN) mns_client = mns_account.get_queue(MNS_QUEUE) mns_client.send_message(msg) self.logger.info("推送mns成功. %s" % str(sql_id)) except Exception as e: self.logger.exception("推送mns失败: %s" % e)
def __init__(self): """客户端初始化""" cf = ConfigParser.ConfigParser() cf.read("aliyun.conf") host = cf.get('AccessKey_mns',"Host") accessid = cf.get('AccessKey_mns',"Access_ID") accesskey = cf.get('AccessKey_mns',"Access_Key") self.cli = Account(host,accessid,accesskey)
def __init__(self): self.client = AcsClient(app.config.get("ALI_ACCESS_KEY"), app.config.get("ALI_ACCESS_SECRET"), app.config.get("BASE_REGION")) self.account = Account(app.config.get("MNS_ENDPOINT"), app.config.get("ALI_ACCESS_KEY"), app.config.get("ALI_ACCESS_SECRET"), "") self.wait_seconds = 10 self.kube = Kubernetes()
def token_and_queue_refresh(self): """ Token过期刷新 """ if not self.token.need_refresh: return self.token.refresh() if self.account: self.account.mns_client.close_connection() self.account = Account(endpoint, self.token.tmp_access_id, self.token.tmp_access_key, self.token.token) self.queue = self.account.get_queue(self.queue_cfg.queue_name) logger.info(f'{self.token.tk_desc} account and queue ready')
def get_producer(self, settings): if self._mns_producer: return self._mns_producer account = Account( host=settings['MNS_ACCOUNT_ENDPOINT'], access_id=settings['MNS_ACCESSKEY_ID'], access_key=settings['MNS_ACCESSKEY_SECRET'], logger=logging.getLogger('evascrapy.pipelines.AliyunMnsPipeline'), debug=False, ) self._mns_producer = account.get_queue(settings['MNS_QUEUE_NAME']) return self._mns_producer
def send_message(tel="18611846371", module="hadoop", message="error"): # 从https://account.console.aliyun.com/#/secure获取$YourAccountid # 从https://ak-console.aliyun.com/#/accesskey获取$YourAccessId和$YourAccessKey # 从http://$YourAccountId.mns.cn-hangzhou.aliyuncs.com获取$YourMNSEndpoint, eg. http://1234567890123456.mns.cn-hangzhou.aliyuncs.com YourMNSEndpoint = "http://1111641897067585.mns.cn-hangzhou-internal.aliyuncs.com/" YourAccessId = "LTAIyhZeIPt8ivJd" YourAccessKey = "myvAcplJMk32PHIt2DSmWhRqcz6CHv" my_account = Account(YourMNSEndpoint, YourAccessId, YourAccessKey) YourTopicName = "sms.topic-cn-hangzhou" my_topic = my_account.get_topic(YourTopicName) #print my_topic ''' Step 2. 设置SMS消息体(必须) 注:目前暂时不支持消息内容为空,需要指定消息内容,不为空即可。 ''' msg_body1 = "报警短信消息" ''' Step 3. 生成SMS消息属性,single=False表示每个接收者参数不一样, single=True表示每个接收者参数一样 ''' # 3.1 设置SMSSignName和SMSTempateCode YourSignName = "app运营助手" YourSMSTemplateCode = "SMS_32785106" direct_sms_attr1 = DirectSMSInfo(free_sign_name=YourSignName, template_code=YourSMSTemplateCode, single=True) # 3.2 指定接收短信的手机号并指定发送给该接收人的短信中的参数值(在短信模板中定义的) cur_time = time.strftime('%H:%M:%S', time.localtime()) #变量内容不能太长 direct_sms_attr1.add_receiver(receiver=tel) direct_sms_attr1.set_params({ "machine": "短视频", "module": module, "time": cur_time, "message": message }) #Step 5. 生成SMS消息对象 msg1 = TopicMessage(msg_body1, direct_sms=direct_sms_attr1) re_msg = my_topic.publish_message(msg1) #print "Publish Message Succeed. MessageBody:%s MessageID:%s" % (msg_body1, re_msg.message_id) return 0
def init_from_cfg(self): parser = ConfigParser.ConfigParser() parser.read(self.cfg_file) for sec,op in required_ops: if not parser.has_option(sec, op): raise Exception("mnsclient load config file failed") self.mns_account = Account(parser.get(BASE, END_POINT), parser.get(BASE, ACCESS_ID), parser.get(BASE, ACCESS_KEY_SECRET)) self.mns_queue = self.mns_account.get_queue(parser.get(BASE, QUEUE_NAME)) self.wait_seconds = int(parser.get(BASE, WAIT_SECOND))
def create_queue(self, broker): """ 根据broker类型,创建相应的queue对象 """ if broker == 'redis': #创建redis queue from util import redis_queue queue = redis_queue.get_queue(settings.SUBSCRIBE_QUEUE_NAME) elif broker == 'mns': #创建aliyun mns queue self.mns_account = Account(\ settings.MNS_ENDPOINT, \ settings.MNS_ACCESS_KEY_ID, \ settings.MNS_ACCESS_KEY_SECRET, \ settings.MNS_SECURITY_TOKEN) queue = self.mns_account.get_queue(settings.SUBSCRIBE_QUEUE_NAME) else: queue = None return queue
def aliyunMsg(user, *args): code, product = args my_account = Account(Config.MNS_ENDPOINT, Config.MNS_ACCESSID, Config.MNS_ACCESSKEY) my_topic = my_account.get_topic(Config.MNS_TOPIC) msg_body1 = "sms-message1." direct_sms_attr1 = DirectSMSInfo(free_sign_name=Config.MNS_SIGN, template_code=Config.MNS_TEMPLATE_CODE, single=True) direct_sms_attr1.add_receiver(receiver=user) direct_sms_attr1.set_params({"code": code, "product": product}) msg1 = TopicMessage(msg_body1, direct_sms=direct_sms_attr1) try: re_msg = my_topic.publish_message(msg1) print "Publish Message Succeed. MessageBody:%s MessageID:%s" % ( msg_body1, re_msg.message_id) except MNSExceptionBase, e: if e.type == "TopicNotExist": print "Topic not exist, please create it." sys.exit(1) print "Publish Message Fail. Exception:%s" % e
def send_msg(queue_name, message_body): """ 向指定消息队列发送消息 :param queue_name: 消息队列的名称 :param message_body: 需要发送的消息 :return: True:发送成功;False:发送失败;None:该消息队列不存在 """ access_key_id, access_key_secret, end_point, security_token = config_parse() my_account = Account(end_point, access_key_id, access_key_secret, security_token) my_queue = my_account.get_queue(queue_name) try: msg = Message(str(message_body)) re_msg = my_queue.send_message(msg) # print re_msg.message_id except MNSExceptionBase as e: if e.type == "QueueNotExist": return None else: return False else: return True
def my_recv_msg(): #从sample.cfg中读取基本配置信息 ## WARNING: Please do not hard code your accessId and accesskey in next line.(more information: https://yq.aliyun.com/articles/55947) accid,acckey,endpoint,token = MNSSampleCommon.LoadConfig() #初始化 my_account, my_queue my_account = Account(endpoint, accid, acckey, token) queue_name = "MyQueue1" base64 = False if len(sys.argv) > 2 and sys.argv[2].lower() == "false" else True my_queue = my_account.get_queue(queue_name) my_queue.set_encoding(base64) #循环读取删除消息直到队列空 #receive message请求使用long polling方式,通过wait_seconds指定长轮询时间为3秒 ## long polling 解析: ### 当队列中有消息时,请求立即返回; ### 当队列中没有消息时,请求在MNS服务器端挂3秒钟,在这期间,有消息写入队列,请求会立即返回消息,3秒后,请求返回队列没有消息; wait_seconds = 3 # print "%sReceive And Delete Message From Queue%s\nQueueName:%s\nWaitSeconds:%s\n" % (10*"=", 10*"=", queue_name, wait_seconds) #读取消息 try: recv_msg = my_queue.receive_message(wait_seconds) # print "Receive Message Succeed! ReceiptHandle:%s MessageBody:%s MessageID:%s" % (recv_msg.receipt_handle, recv_msg.message_body, recv_msg.message_id) except MNSExceptionBase, e: if e.type == "QueueNotExist": print "Queue not exist, please create queue before receive message." sys.exit(0) elif e.type == "MessageNotExist": print "Queue is empty!" sys.exit(0) print "Receive Message Fail! Exception:%s\n" % e continue
class MNSReader(object): id = None secret = None point = None message = None queue_name = None db = dict() def __init__(self): self.id = conf.access_key_id self.secret = conf.access_key_secret self.point = conf.queue_endpoint self.queue_name = conf.queue_name self.auth = oss2.Auth(conf.access_key_id, conf.access_key_secret) self.bucket = oss2.Bucket(self.auth, conf.oss_endpoint, conf.bucket_name) self.oss_img_path = conf.oss_img_path client = MongoClient(conf.mongo_host) client['admin'].authenticate('root', 'qwer1234') self.userdb = client['qp_account'] for index, subject in enumerate(_ALL_SUPPORT_SUBJECTS): db_name = 'qp_' + subject self.db[index] = client[db_name] logging.critical('connect mongodb success:{}'.format(conf.mongo_host)) if self.id is None or self.secret is None or self.point is None or self.queue_name is None: raise Exception('lose conf') try: self.account = Account(self.point, self.id, self.secret) self.queue = self.account.get_queue(self.queue_name) logging.critical('init_MNS success:') logging.critical(u'id:{}, point:{}, queue_name'.format( self.id, self.point, self.queue_name)) except: traceback.print_exc() raise Exception('connect MNS fail') def pop(self): retry = 2 while retry > 0: try: self.queue.delete_message(self.message.receipt_handle) except MNSExceptionBase, e: logging.fatal('release message fail: ' + str(e)) retry -= 1 time.sleep(1) else: logging.info('release message success') break self.message = None
class MnsClient(): """mns client """ def __init__(self, cfg_file): self.cfg_file = cfg_file self.init_from_cfg() self.last_recv_msg = None def init_from_cfg(self): parser = ConfigParser.ConfigParser() parser.read(self.cfg_file) for sec,op in required_ops: if not parser.has_option(sec, op): raise Exception("mnsclient load config file failed") self.mns_account = Account(parser.get(BASE, END_POINT), parser.get(BASE, ACCESS_ID), parser.get(BASE, ACCESS_KEY_SECRET)) self.mns_queue = self.mns_account.get_queue(parser.get(BASE, QUEUE_NAME)) self.wait_seconds = int(parser.get(BASE, WAIT_SECOND)) def recv_msg(self, visible_timeout=60): """ @return: None means recv msg failed, otherwise, return the msg_body(means the msg that you send) """ ret_flag = None try: _msg = self.mns_queue.receive_message(self.wait_seconds) try: change_msg_vis = self.mns_queue.change_message_visibility(_msg.receipt_handle, visible_timeout) logger.debug("Change Message Visibility Succeed!\nReceiptHandle:%s\nNextVisibleTime:%s\n\n" % (change_msg_vis.receipt_handle, change_msg_vis.next_visible_time)) ret_flag = _msg.message_body self.last_recv_msg = change_msg_vis except MNSExceptionBase, e: logger.warn("Change Message Visibility Fail!\nException:%s\n\n" % e) except MNSExceptionBase as e: if e.type == 'MessageNotExist': logger.debug("no message left in queue [%s]" %self.mns_queue.queue_name) else: logger.warn("Receive Message Fail!\nException:%s\n\n" % e) return ret_flag def del_msg(self, retry=3): """ @return: """ num = retry ret_flag = False if self.last_recv_msg is None: logger.debug("last recv msg is None, nothing to do") return True while num > 0: try: self.mns_queue.delete_message(self.last_recv_msg.receipt_handle) logger.debug("Delete Message Succeed.\n\n") ret_flag = True self.last_recv_msg = None return ret_flag except MNSExceptionBase, e: logger.warn("Del Message from %s Fail!\nException:%s\n\n, retry again" % (self.mns_queue.queue_name,e)) time.sleep(1.0) logger.warn("try to delete Message %s time still Failed, message will be push back to queue [%s]" %(retry, self.mns_queue.queue_name)) return ret_flag
self.__tmp_access_key = response_body.get("MessageTokenDTO").get( "AccessKeySecret") self.__tmp_access_id = response_body.get("MessageTokenDTO").get( "AccessKeyId") self.__expire_time = response_body.get("MessageTokenDTO").get( "ExpireTime") self.__token = response_body.get("MessageTokenDTO").get( "SecurityToken") print "finsh refresh token..." # 初始化 my_account, my_queue token = Token() token.refresh() my_account = Account(endpoint, token.get_tmp_access_id(), token.get_tmp_access_key(), token.get_token()) my_queue = my_account.get_queue(qname) # my_queue.set_encoding(False) # 循环读取删除消息直到队列空 # receive message请求使用long polling方式,通过wait_seconds指定长轮询时间为3秒 ## long polling 解析: ### 当队列中有消息时,请求立即返回; ### 当队列中没有消息时,请求在MNS服务器端挂3秒钟,在这期间,有消息写入队列,请求会立即返回消息,3秒后,请求返回队列没有消息; wait_seconds = 3 print "%sReceive And Delete Message From Queue%s\nQueueName:%s\nWaitSeconds:%s\n" % ( 10 * "=", 10 * "=", qname, wait_seconds) while True: # 读取消息 try:
import sys import os sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)) + "/..") import time from sample.sample_common import MNSSampleCommon from mns.account import Account from mns.queue import * # 从sample.cfg中读取基本配置信息 # WARNING: Please do not hard code your accessId and accesskey in next line.(more information: # https://yq.aliyun.com/articles/55947) accid, acckey, endpoint, token = MNSSampleCommon.LoadConfig() #初始化 my_account, my_queue my_account = Account(endpoint, accid, acckey, token) queue_name = sys.argv[1] if len(sys.argv) > 1 else "MySampleQueue" base64 = False if len( sys.argv) > 2 and sys.argv[2].lower() == "false" else True my_queue = my_account.get_queue(queue_name) my_queue.set_encoding(base64) #循环读取删除消息直到队列空 #receive message请求使用long polling方式,通过wait_seconds指定长轮询时间为3秒 ## long polling 解析: ### 当队列中有消息时,请求立即返回; ### 当队列中没有消息时,请求在MNS服务器端挂3秒钟,在这期间,有消息写入队列,请求会立即返回消息,3秒后,请求返回队列没有消息; wait_seconds = 3 print(
#获取配置信息 ## AccessKeyId 阿里云官网获取 ## AccessKeySecret 阿里云官网获取 ## Endpoint 阿里云消息和通知服务官网获取, Example: http://$AccountId.mns.cn-hangzhou.aliyuncs.com ## WARNING: Please do not hard code your accessId and accesskey in next line.(more information: https://yq.aliyun.com/articles/55947) accessKeyId = parser.get("Base", "AccessKeyId") accessKeySecret = parser.get("Base", "AccessKeySecret") endpoint = parser.get("Base", "Endpoint") securityToken = "" if parser.has_option("Optional", "SecurityToken") and parser.get( "Optional", "SecurityToken") != "$SecurityToken": securityToken = parser.get("Optional", "SecurityToken") #初始化my_account my_account = Account(endpoint, accessKeyId, accessKeySecret, securityToken) ##############Queue 相关操作##################### my_queue = my_account.get_queue( "MyQueue-%s" % time.strftime("%y%m%d-%H%M%S", time.localtime())) #创建队列 ## message被receive后,持续不可消费的时间 100秒 ## message body的最大长度 10240Byte ## message最长存活时间 3600秒 ## 新message可消费的默认延迟时间 10秒 ## receive message时,长轮询时间 20秒 queue_meta = QueueMeta() queue_meta.set_visibilitytimeout(100) queue_meta.set_maximum_message_size(10240) queue_meta.set_message_retention_period(3600)
#coding=utf8 import sys import os sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)) + "/..") import time from sample_common import MNSSampleCommon from mns.account import Account from mns.queue import * #从sample.cfg中读取基本配置信息 ## WARNING: Please do not hard code your accessId and accesskey in next line.(more information: https://yq.aliyun.com/articles/55947) accid,acckey,endpoint,token = MNSSampleCommon.LoadConfig() #初始化 my_account, my_queue my_account = Account(endpoint, accid, acckey, token) queue_name = sys.argv[1] if len(sys.argv) > 1 else "MySampleQueue" my_queue = my_account.get_queue(queue_name) #创建队列, 具体属性请参考mns/queue.py中的QueueMeta结构 queue_meta = QueueMeta() try: queue_url = my_queue.create(queue_meta) print "Create Queue Succeed! QueueName:%s\n" % queue_name except MNSExceptionBase, e: if e.type == "QueueAlreadyExist": print "Queue already exist, please delete it before creating or use it directly." sys.exit(0) print "Create Queue Fail! Exception:%s\n" % e
class ResumeExchange(Spider): def __init__(self): super(ResumeExchange, self).__init__() self.mns_account = Account(MNS_ENDPOINT, MNS_ACCID, MNS_ACCKEY, MNS_TOKEN) self.mns_client = self.mns_account.get_queue(MNS_QUEUE) def get_targetJobType(self, emailJobType): for item in job_type_mapping: if item.upper() in emailJobType: return job_type_mapping[item] return def save2db(self, sql, data): # conn = pymysql.Connect(**MYSQL_CONF_TEST) conn = pymysql.Connect(**MYSQL_CONF) cur = conn.cursor() try: cur.execute(sql, data) conn.commit() cur.execute("select last_insert_id()") sql_id = cur.fetchone()[0] self.logger.info("入库成功. %s" % sql_id) return sql_id except Exception as e: self.logger.exception(e) return finally: cur.close() conn.close() def save2mns(self, sql_id, data_for_mns, trackId): try: kafka_data = { "content": data_for_mns, "channelType": "WEB", "interfaceType": "NORMAL", "resourceDataType": "PARSED", "resourceType": "RESUME_INBOX", "source": data_for_mns['source'], "traceID": trackId, "callSystemID": PROJECT_NAME } dumps = json.dumps(kafka_data, ensure_ascii=False) buf = StringIO() f = gzip.GzipFile(mode='wb', fileobj=buf) f.write(dumps) f.close() msg_body = base64.b64encode(buf.getvalue()) msg = Message(msg_body) self.mns_client.send_message(msg) self.logger.info("推送mns成功. %s" % str(sql_id)) except Exception as e: self.logger.exception("推送mns失败: %s" % e) def run(self, file_path): try: with open(file_path, 'r') as file: self.logger.info("开始上载简历.") for index, line in enumerate(file): # self.process1(index=index,line=line) self.process2(index=index, line=line) except Exception as e: self.logger.exception(e) def process2(self, index, line): try: self.logger.info('当前处理的内如:%s %s' % (index, line)) name, mobile, gender, degree, age, work_exp, target_job_type, register_location = line.strip( '\n').split('\t') except ValueError: self.logger.warning("该行读取错误: " + str(index)) return now_year = datetime.datetime.now().year birthday = '' if age: birthday = str(now_year - int(age)) + '-05-15' uuid_ = str(uuid.uuid1()) data = (name, mobile, gender, degree, birthday, work_exp, target_job_type, register_location, datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d'), datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d %H:%M:%S'), datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d %H:%M:%S'), uuid_) sql = ''' INSERT INTO spider.resume_parsed (name,mobile,gender,degree,birthday,workExperience,targetJobType,registerLocation,targetWorkLocation,workType,status,source,targetSalary,resumeUpdateTime,createTime,updateTime,trackId) VALUES (%s, %s, %s, %s, %s, %s, %s, %s,'西安','全职','正在找工作','RESUME_EXCHANGE','面议',%s,%s,%s,%s) ''' sql_id = self.save2db(sql=sql, data=data) self.logger.info('解析入库成功 %s' % sql_id) data_for_mns = { 'id': sql_id, 'name': name, 'mobile': mobile, 'gender': gender, 'degree': degree, 'birthday': birthday, 'workExperience': work_exp, 'targetJobType': target_job_type, 'registerLocation': register_location, 'targetWorkLocation': '西安', 'workType': '全职', 'status': '正在找工作', 'source': 'RESUME_EXCHANGE', 'targetSalary': '面议', 'resumeUpdateTime': datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d %H:%M:%S'), 'createTime': datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d %H:%M:%S'), 'updateTime': datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d %H:%M:%S'), 'trackId': uuid_, } self.save2mns(sql_id, data_for_mns, uuid_) self.logger.info('处理完毕 %s' % sql_id) def process1(self, index, line): try: name, emailJobType, targetJobType, gender, birthday, mobile, \ workExperience, email, currentLocation, registerLocation, \ career, education, degree, targetSalary = line.strip( '\n').split('\t') except ValueError: self.logger.warning("该行读取错误: " + str(index)) return try: emailJobType = emailJobType.replace('北京', '').replace('求职', '').replace( '/', ',') targetJobType = self.get_targetJobType(emailJobType) if not targetJobType: targetJobType = emailJobType birthday = birthday.replace('/', '-') career_lst = [] if career: for item in career.split('|$|'): career_lst.append({"companyName": item}) career = json.dumps(career_lst, ensure_ascii=False) education_lst = [] if education: for item in education.split('|$|'): education_lst.append({"schoolName": item}) education = json.dumps(education_lst, ensure_ascii=False) now = datetime.datetime.now() trackId = str(uuid.uuid1()) data = (name, emailJobType, targetJobType, gender, birthday, mobile, workExperience, email, currentLocation, registerLocation, career, education, degree, targetSalary, now, '2017-11-01 00:00:00', trackId) sql = """ INSERT INTO spider.resume_parsed(name, emailJobType, targetJobType, gender, birthday, mobile, \ workExperience, email, currentLocation, registerLocation, \ career, education, degree, targetSalary, createTime, updateTime, trackId, createBy, source, workType, status) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'PYTHON', 'RESUME_EXCHANGE', '全职', '正在找工作') """ sql_id = self.save2db(sql=sql, data=data) if not sql_id: return data_for_mns = { 'id': sql_id, 'name': name, 'emailJobType': emailJobType, 'targetJobType': targetJobType, 'gender': gender, 'birthday': birthday, 'mobile': mobile, 'workExperience': workExperience, 'email': email, 'currentLocation': currentLocation, 'registerLocation': registerLocation, 'career': career_lst, 'education': education_lst, 'degree': degree, 'targetSalary': targetSalary, 'createTime': datetime.datetime.strftime(now, '%Y-%m-%d %H:%M:%S'), 'updateTime': '2017-11-01 00:00:00', 'trackId': trackId, 'source': 'RESUME_EXCHANGE', 'workType': '全职', 'status': '正在找工作' } self.save2mns(sql_id, data_for_mns, trackId) self.logger.info( '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s' % (index, name, emailJobType, targetJobType, gender, birthday, mobile, workExperience, email, currentLocation, registerLocation, career, education, degree, targetSalary)) except Exception as e: self.logger.exception(e) return
class AliAPI: def __init__(self): self.client = AcsClient(app.config.get("ALI_ACCESS_KEY"), app.config.get("ALI_ACCESS_SECRET"), app.config.get("BASE_REGION")) self.account = Account(app.config.get("MNS_ENDPOINT"), app.config.get("ALI_ACCESS_KEY"), app.config.get("ALI_ACCESS_SECRET"), "") self.wait_seconds = 10 self.kube = Kubernetes() def reboot_ecs(self, ecs_id): """ reboot ecs :param ecs_id: :return: """ try: request = RebootInstanceRequest() request.set_accept_format('json') request.set_InstanceId(ecs_id) response = self.client.do_action_with_exception(request) logs.info("reboot ecs {} successed result is {}".format( ecs_id, response)) return True except ServerException as exc: logs.info("reboot ecs failed msg is {}".format(exc)) def set_ecs(self, ecs_id, ecs_hostname, ecs_instancename): """ set ecs hostname :param ecs_id: :param ecs_hostname: :param ecs_instancename: :return: """ try: request = ModifyInstanceAttributeRequest() request.set_accept_format('json') request.set_InstanceId(ecs_id) request.set_InstanceName(ecs_instancename) response = self.client.do_action_with_exception(request) logs.info( "set ecs ecs_id is {} ecs_hostname is {} ecs_instancename is {} response is {}" .format(ecs_id, ecs_hostname, ecs_instancename, response)) return True except ServerException as exc: logs.error("set ecs failed msg is {}".format(exc)) def get_ecs_info(self, ecs_id): """ get ecs ip and hostname :param ecs_id: :return: """ ip, hostname = None, None try: request = DescribeInstancesRequest() request.set_accept_format('json') request.set_InstanceIds([ ecs_id, ]) response = json.loads( self.client.do_action_with_exception(request)) # 私有网络IP ip = response.get("Instances").get("Instance")[0].get( "VpcAttributes").get("PrivateIpAddress").get("IpAddress")[0] hostname = response.get("Instances").get("Instance")[0].get( "HostName") except ServerException as exc: logs.error("get ecs info failed msg is {}".format(exc)) finally: return ip, hostname def handlers(self, transiton, ecs_id, hostname, ip, scaling_groupname): """ 响应伸缩组活动 :param transiton: :param ecs_id: :param hostname: :param ip: :param scaling_groupname: :return: """ assert self.set_ecs(ecs_id, hostname, hostname), AssertionError('set ecs failed.') if ip and hostname: if hasattr(self.kube, transiton.lower()): result = getattr(self.kube, transiton.lower())(ip, hostname) if not result: logs.info("Reboot Ecs id is {} hostname is {}".format( ecs_id, hostname)) self.reboot_ecs(ecs_id) # 发送钉钉通知 self.kube.notice.send_mardown( title='伸缩组活动', text='#### 伸缩组活动\n' '> \n\n' '> - 伸缩组名称 {}\n' '> - 动作类型 **{}**\n' '> - ECS服务器名称 {}\n' '> - ECS服务器IP {}\n' '> ###### {} \n'.format( scaling_groupname, "缩容机器" if transiton == "SCALE_IN" else "扩容机器", hostname, ip, time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))) def receive_msg(self, queue_name=app.config.get("MNS_QUEUENAME")): queue = self.account.get_queue(queue_name) while 1: try: recv_msg = queue.receive_message(self.wait_seconds) message_body = bytes.decode(recv_msg.message_body) change_msg_vis = queue.change_message_visibility( recv_msg.receipt_handle, 35) self.delete_msg(change_msg_vis) try: message_body = json.loads(message_body) except json.decoder.JSONDecodeError: logs.debug("json decoder failes.") # 只处理dict类型数据 if type(message_body) is dict: logs.info("recv msg is {}".format(message_body)) transiton = message_body.get("content").get( "lifecycleTransition") if transiton not in storage.cache: storage.cache[transiton] = [] scaling_groupname = message_body.get("content").get( "scalingGroupName") for ecs_id in message_body.get("content").get( "instanceIds"): if ecs_id not in storage.cache[transiton]: storage.cache[transiton].append(ecs_id) ip, _ = self.get_ecs_info(ecs_id) if ip in storage.host_map: hostname = storage.host_map[ip] else: # 拼接主机名 hostname = "{}{}".format( app.config.get("HOSTNAME_PREFIX"), ("%03d" % get_counter())) storage.host_map[ip] = hostname # 线程处理 threading.Thread(target=self.handlers, args=(transiton, ecs_id, hostname, ip, scaling_groupname)).start() except MNSExceptionBase as e: logs.error("recv msg failed {}".format(e)) def delete_msg(self, change_msg_vis, queue_name=app.config.get("MNS_QUEUENAME")): queue = self.account.get_queue(queue_name) queue.delete_message(change_msg_vis.receipt_handle) logs.info("del msg {}".format(change_msg_vis.receipt_handle))
#DB_PASSWORD = "******" DB_HOST = "47.95.232.88" DB_USERNAME = "******" DB_PASSWORD = "******" DB_DATABASE = "greengrass" # Read the basic configurations from sample.cfg # WARNING: Please do not hard code your accessId and accesskey in next line. # (more information: https://yq.aliyun.com/articles/55947) accessKeyId, accesskeySecret, endpoint, token = MNSSampleCommon.LoadConfig() # create the connect for IoT Kit clt = client.AcsClient(accessKeyId, accesskeySecret, 'cn-shanghai') # Init my_account, my_queue my_account = Account(endpoint, accessKeyId, accesskeySecret, token) queue_name = sys.argv[1] if len(sys.argv) > 1 else queueName boolbase64 = False if len( sys.argv) > 2 and sys.argv[2].lower() == "false" else True my_queue = my_account.get_queue(queue_name) my_queue.set_encoding(boolbase64) # get the connection to Amazon Greengrass RDS db = MySQLdb.connect(DB_HOST, DB_USERNAME, DB_PASSWORD, DB_DATABASE) # Read and delete the message from queue until the queue is empty # receive message uses long polling mode, specify the long loop time to 3 second through wait_seconds # long polling parsing # Return the message when there are messages # When queue is empty, request will wait for 3 second on server side.
#获取配置信息 ## AccessKeyId 阿里云官网获取 ## AccessKeySecret 阿里云官网获取 ## Endpoint 阿里云消息和通知服务官网获取, Example: http://$AccountId.mns.cn-hangzhou.aliyuncs.com ## WARNING: Please do not hard code your accessId and accesskey in next line.(more information: https://yq.aliyun.com/articles/55947) accessKeyId = parser.get("Base", "AccessKeyId") accessKeySecret = parser.get("Base", "AccessKeySecret") endpoint = parser.get("Base", "Endpoint") securityToken = "" if parser.has_option("Optional", "SecurityToken") and parser.get("Optional", "SecurityToken") != "$SecurityToken": securityToken = parser.get("Optional", "SecurityToken") #初始化my_account my_account = Account(endpoint, accessKeyId, accessKeySecret, securityToken) ##############Queue 相关操作##################### my_queue = my_account.get_queue("MyQueue-%s" % time.strftime("%y%m%d-%H%M%S", time.localtime())) #创建队列 ## message被receive后,持续不可消费的时间 100秒 ## message body的最大长度 10240Byte ## message最长存活时间 3600秒 ## 新message可消费的默认延迟时间 10秒 ## receive message时,长轮询时间 20秒 queue_meta = QueueMeta() queue_meta.set_visibilitytimeout(100) queue_meta.set_maximum_message_size(10240) queue_meta.set_message_retention_period(3600) queue_meta.set_delay_seconds(10)
#! -*- coding: utf-8 -*- import json from mns.account import Account from mns.mns_exception import MNSExceptionBase from utils import post_slack from config import QUEUE_NAME, END_POINT, ACCESS_KEY_ID, ACCESS_KEY_SECRET account = Account(END_POINT, ACCESS_KEY_ID, ACCESS_KEY_SECRET) queue = account.get_queue(QUEUE_NAME) def run(): while True: try: recv_msg = queue.receive_message(wait_seconds=10) message = json.loads(recv_msg.message_body)['message'] print message if message['levelDescription'] == u'恢复正常': color = 'good' else: color = 'danger' post_slack(message['dimensions'], message['metricName'], message['expression'], color) queue.delete_message(recv_msg.receipt_handle) except MNSExceptionBase, e: print e.message if __name__ == '__main__':
import sys import os sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)) + "/..") import time from sample_common import MNSSampleCommon from mns.account import Account from mns.topic import * #从sample.cfg中读取基本配置信息 ## WARNING: Please do not hard code your accessId and accesskey in next line.(more information: https://yq.aliyun.com/articles/55947) accid,acckey,endpoint,token = MNSSampleCommon.LoadConfig() #初始化 my_account, my_topic my_account = Account(endpoint, accid, acckey, token) topic_name = sys.argv[1] if len(sys.argv) > 1 else "MySampleTopic" my_topic = my_account.get_topic(topic_name) #循环发布多条消息 msg_count = 3 print "%sPublish Message To Topic%s\nTopicName:%s\nMessageCount:%s\n" % (10*"=", 10*"=", topic_name, msg_count) for i in range(msg_count): try: msg_body = "I am test message %s." % i msg = TopicMessage(msg_body) re_msg = my_topic.publish_message(msg) print "Publish Message Succeed. MessageBody:%s MessageID:%s" % (msg_body, re_msg.message_id) except MNSExceptionBase,e: if e.type == "TopicNotExist":
GPIO_clean() if __name__ == '__main__': print "start deamon!" #init my_account, my_topic endpoint = "http://1402010103453031.mns.cn-hangzhou.aliyuncs.com/" access_id = "LTAIfpx57LeEmMKs" access_key = "nUqBzUFc48cTVIZh3mkpcnmosxvGTz" token = "" quit = Queue() dr = DryingRack() GPIO_init() #init my_account, my_topic my_account = Account(endpoint, access_id, access_key, token) # queue_name = "control-queue" # my_queue = my_account.get_queue(queue_name) #Thread(target=control_thread,args=(my_account,)).start() #Thread(target=ioread_thread,args=(my_queue,)).start() Process(target=ioread_thread, args=(my_account, quit)).start() Process(target=control_thread, args=(my_account, quit)).start() #ioread_thread(my_account) while True: s = raw_input("type quit to quit:") if s == "quit": quiting = True quit.put("quit")
#coding:utf-8 import os from flask import Flask, jsonify, request from mns.account import Account from mns.topic import DirectSMSInfo, TopicMessage, MNSExceptionBase app = Flask(__name__) account = Account(os.getenv('END_POINT'), os.getenv('ACCESS_ID'), os.getenv('ACCESS_KEY')) topic = account.get_topic(os.getenv('TOPIC')) accessToken = os.getenv('ACCESS_TOKEN') class InvalidUsage(Exception): status_code = 400 def __init__(self, message, status_code=None, payload=None): Exception.__init__(self) self.message = message if status_code is not None: self.status_code = status_code self.payload = payload def to_dict(self): rv = dict(self.payload or ()) rv['message'] = self.message return rv @app.errorhandler(InvalidUsage) def handle_invalid_usage(error):
# coding: utf-8 import sys from mns.account import Account from mns.topic import DirectSMSInfo, TopicMessage from mns.mns_exception import MNSExceptionBase accid = 'LTAIf6atnIup14AK' acckey = 'zfoeCi2Au0le7vUbsFOmEVtPrtlfZf' endpoint = 'http://1771999519197750.mns.cn-hangzhou.aliyuncs.com/' topic_name = 'sms.topic-cn-hangzhou' my_account = Account(endpoint, accid, acckey) my_topic = my_account.get_topic(topic_name) ''' Step 2. 设置SMS消息体(必须) 注:目前暂时不支持消息内容为空,需要指定消息内容,不为空即可。 ''' msg_body1 = "sms-message1." ''' Step 3. 生成SMS消息属性,single=False表示每个接收者参数不一样, ''' # 3.1 设置SMSSignName和SMSTemplateCode direct_sms_attr1 = DirectSMSInfo(free_sign_name='宁师生活网', template_code='SMS_69460049', single=False) # 3.2 指定接收短信的手机号并指定发送给该接收人的短信中的参数值(在短信模板中定义的) direct_sms_attr1.add_receiver(receiver='18150850025', params={'code': '123456'}) ''' #Step 5. 生成SMS消息对象 ''' msg1 = TopicMessage(msg_body1, direct_sms=direct_sms_attr1) print(direct_sms_attr1.sms_params) try: ''' Step 6. 发布SMS消息
class Runner(object): def __init__(self): # 读取邮箱信息 self.db = Database( 'mysql+mysqldb://spider_admin:' '[email protected]:' '3306/spider?charset=utf8', encoding='utf-8') # 初始化数据库 self.db1 = Database( 'mysql+mysqldb://%s:%s@%s:%s/%s?charset=utf8' % (MYSQL_USER, MYSQL_PASSWD, MYSQL_HOST, MYSQL_PORT, MYSQL_DB), encoding='utf-8') # 初始化MNS self.mns_account = Account(MNS_ENDPOINT, MNS_ACCID, MNS_ACCKEY, MNS_TOKEN) self.mns_client = self.mns_account.get_queue(MNS_QUEUE) self.logger = Logger.timed_rt_logger() self.queue = RedisQueue('t_mail_by_date', host=REDIS_HOST, port=REDIS_PORT) self.robot = DingDingRobot( access_token="eb749abfe9080a69da6524b77f589b8f6ddbc" "c182c7a41bf095b095336edb0a1") def deliver(self, subject): try: if '51job.com' in subject: source = "FIVE_ONE" deliverJobName = re.search('(?<=申请贵公司).*(?=()', subject).group().strip() subject_ = subject.replace('(', '(').replace(')', ')') deliverCity = re.findall('(?<=\().*?(?=\))', subject_)[-1].strip() elif '58.com' in subject: # (58.com)应聘贵公司供应链管理8k六险一金-北京 朝阳-刘春雨 source = "FIVE_EIGHT" deliverJobName = re.search('(?<=应聘贵公司).*?(?=-)', subject).group().strip() deliverCity = re.findall('(?<=-).*?(?=\s+)', subject)[-1].strip() elif 'Zhaopin.com' in subject: source = "ZHI_LIAN" deliverJobName = re.search('(?<=应聘).*?(?=-)', subject).group().strip() deliverCity = re.findall('(?<=-).*?(?=-)', subject)[-1].strip() else: return None, None, None return source, deliverJobName, deliverCity except: return None, None, None def save2db(self, source, email, subject, content, emailJobType, emailCity, deliverJobName, deliverJobCity, trackId): try: createTime = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") data = { 'source': source, 'email': email, 'subject': subject, 'content': content, 'emailJobType': emailJobType, 'emailCity': emailCity, 'createBy': email, 'updateBy': 'PYTHON-RECEIVER', 'deliverJobName': deliverJobName, 'deliverJobCity': deliverJobCity, 'createTime': createTime, 'trackId': trackId } try: self.db1.session.execute(ResumeRaw.__table__.insert(), data) self.db1.session.commit() except Exception as e: self.db1.session.rollback() self.logger.exception("入库失败, 回滚: %s" % e) self.robot.send_text("邮箱采集,入库失败: %s" % email) time.sleep(600) return sql_id = self.db1.session.execute( "select last_insert_id()").first()[0] # print(sql_id) self.logger.info("入库成功. %s" % str(sql_id)) return sql_id except Exception as e: self.logger.exception("入库失败: %s" % e) return def save2mns(self, sql_id, source, content, emailJobType, emailCity, deliverJobName, deliverJobCity, trackId): try: # resume_uuid = uuid.uuid1() kafka_data = { "channelType": "WEB", "content": { "content": content, "id": sql_id, "createBy": "python", "createTime": int(time.time() * 1000), "ip": '', 'emailJobType': emailJobType, 'emailCity': emailCity, 'deliverJobName': deliverJobName, 'deliverJobCity': deliverJobCity, "resumeSubmitTime": '', "resumeUpdateTime": '', "source": source, "trackId": str(trackId), "avatarUrl": '', }, "interfaceType": "PARSE", "resourceDataType": "RAW", "resourceType": "RESUME_EMAIL", "source": source, "trackId": str(trackId), "callSystemID": project_name, "traceID": str(trackId) } dumps = json.dumps(kafka_data, ensure_ascii=False) buf = StringIO() f = gzip.GzipFile(mode='wb', fileobj=buf) f.write(dumps) f.close() msg_body = base64.b64encode(buf.getvalue()) msg = Message(msg_body) self.mns_client.send_message(msg) self.logger.info("推送mns成功. %s" % str(sql_id)) except Exception as e: self.logger.exception("推送mns失败: %s" % e) def producer(self): self.queue.clean() # res = self.db.session.query(TEmail).filter_by(valid=1).all() res = self.db.session.query(TEmail).filter( TEmail.email.in_(( # '*****@*****.**', # '*****@*****.**', # '*****@*****.**', # '*****@*****.**', # '*****@*****.**', # '*****@*****.**', '*****@*****.**', # '*****@*****.**', # '*****@*****.**', ))) if not self.queue.empty(): self.logger.info("剩余%s个企业邮箱未处理!" % str(self.queue.qsize())) sys.exit() # self.queue.clean() for item in res: self.queue.put( (item.pop3_host.encode('utf-8'), item.username.encode('utf-8'), item.password.encode('utf-8'), item.position_type.encode('utf-8'), item.preferred_city.encode('utf-8'), item.source.encode('utf-8'))) self.logger.info("总计加载%s个企业邮箱!" % str(self.queue.qsize())) def consumer(self): while True: try: if not self.queue.empty(): host, username, password, position_type, preferred_city, \ source1 = eval(self.queue.get()) # print(host, username, password) imap = MailReceiver(host, username, password) try: imap.server.login(username, password) self.logger.info("%s: 登录成功 <%s>" % (username, self.queue.qsize())) except Exception as e: if "LOGIN Login error" in e: self.robot.send_text("邮箱登录异常: %s %s %s" % (username, password, host)) self.logger.exception( "%s: 登录失败 %s <%s>" % (username, e, self.queue.qsize())) continue try: folder_list = imap.folder_list() except Exception as e: self.logger.exception("%s: 邮件箱加载失败 %s" % (username, e)) continue if not folder_list: continue # print(folder_list) if "Deleted Messages" in folder_list: delete_folder = u"Deleted Messages" elif "已删除邮件" in folder_list: delete_folder = u"已删除邮件" else: delete_folder = u"已删除" self.logger.info("%s: 开始收取邮件." % username) today = datetime.datetime.now() days = 180 for day in xrange(1, days): date = today - datetime.timedelta(days=day) fmt_date = datetime.datetime.strftime(date, "%d-%b-%Y") self.logger.info('%s开始收取:%s 邮件' % (username, fmt_date)) try: message_list, msg_data = imap.message_list( search_='ON %s' % fmt_date) except Exception as e: self.logger.exception("%s: 邮件list加载失败 %s" % (username, e)) continue if not msg_data: self.logger.warning("%s: 没有收到新邮件." % username) continue self.logger.info("%s: 收到%s个新邮件" % (username, str(len(message_list)))) # print(message_list) for msg_id, message in msg_data.items(): res = imap.get_content(message) if not res: # message_list.append(message) continue try: subject = res.get('subject').encode( 'utf-8', 'ignore') except UnicodeDecodeError: subject = res.get('subject') content = res.get('body') emoji_pattern = re.compile( u"(\ud83d[\ude00-\ude4f])|(\ud83c[\udf00-\uffff])|(\ud83d[\u0000-\uddff])|(\ud83d[\ude80-\udeff])|(\ud83c[\udde0-\uddff])+", flags=re.UNICODE) content = emoji_pattern.sub(r'', content) source, deliverJobName, deliverJobCity = self.deliver( subject) if not deliverJobName: self.logger.warning( "%s: 收到非期待邮件. subject: %s" % (username, subject)) try: imap.delete_message([msg_id], delete_folder) self.logger.info( "%s: 非期待邮件<%s>成功转移到“已删除”" % (username, msg_id)) except Exception as e: self.logger.exception( "%s: 非期待邮件<%s>转移到“已删除”失败 %s" % (username, msg_id, e)) continue self.logger.info( "%s\t%s\t%s\t" % (source, deliverJobName, deliverJobCity)) trackId = uuid.uuid4() sql_id = self.save2db(source, username, subject, content, position_type, preferred_city, deliverJobName, deliverJobCity, trackId) if not sql_id: continue self.save2mns(sql_id, source.encode('utf-8'), content.encode('utf-8'), position_type, preferred_city, deliverJobName, deliverJobCity, trackId) try: imap.delete_message([msg_id], delete_folder) self.logger.info("%s: 邮件<%s>成功转移到“已删除”" % (username, msg_id)) except Exception as e: self.logger.exception( "%s: 邮件<%s>转移到“已删除”失败 %s" % (username, msg_id, e)) imap.close() # else: # self.logger.info("finished!") # break except Exception as e: self.logger.exception(e)
def get_mns_client(): mns_account = Account(common_settings.ENDPOINT, common_settings.ACCID, common_settings.ACCKEY, common_settings.TOKEN) mns_client = mns_account.get_queue(common_settings.MNS_QUEUE) return mns_client
def __init__(self): super(ResumeExchange, self).__init__() self.mns_account = Account(MNS_ENDPOINT, MNS_ACCID, MNS_ACCKEY, MNS_TOKEN) self.mns_client = self.mns_account.get_queue(MNS_QUEUE)