def storeClick(rdd): if rdd.isEmpty() is False: resultList = rdd.collect() for item in resultList: mongodb = PymongoDateBase.instance().get_db() uuid = item["uuid"] colleciton = mongodb.push_record record = colleciton.find_one({"uuid": uuid}) if record: click = record["click"] type = record["type"] sendid = record["sendid"] if click == 0: sqldb = database.instance().get_session() if type == 1: #短信推送 supplier = sqldb.query( "select id,pushstatus from supplier where mobile=%s ", sendid) print supplier if supplier and supplier[0]["pushstatus"] != 2: sqldb.execute( "update supplier set pushscore=pushscore+1 where mobile=%s", sendid) else: sqldb.execute( "update users set pushscore=pushscore+1 where phone=%s", sendid) elif type == 2: #微信推送 sqldb.execute( "update users set pushscore=pushscore+1 where openid=%s", sendid) click = click + 1 colleciton.update({'uuid': uuid}, {'$set': {'click': click}})
def storeClick(rdd): if rdd.isEmpty() is False: resultList = rdd.collect() for item in resultList: mongodb = PymongoDateBase.instance().get_db() uuid = item["uuid"] colleciton = mongodb.push_record record=colleciton.find_one({"uuid":uuid}) if record: click=record["click"] type=record["type"] sendid=record["sendid"] if click==0: sqldb = database.instance().get_session() if type==1:#短信推送 supplier=sqldb.query("select id,pushstatus from supplier where mobile=%s ",sendid) print supplier if supplier and supplier[0]["pushstatus"]!=2: sqldb.execute("update supplier set pushscore=pushscore+1 where mobile=%s",sendid) else: sqldb.execute("update users set pushscore=pushscore+1 where phone=%s", sendid) elif type==2:#微信推送 sqldb.execute("update users set pushscore=pushscore+1 where openid=%s", sendid) click=click+1 colleciton.update({'uuid':uuid}, {'$set':{'click':click}})
def writeDB(rdd): if rdd.isEmpty() is False: resultList = rdd.collect() for item in resultList: mongodb = PymongoDateBase.instance().get_db() colleciton = mongodb.push_record uuid = item["uuid"] quoteid = item["quoteid"] sqldb = database.instance().get_session() ret = sqldb.get("select purchaseinfoid from quote where id=%s", quoteid) if ret: qinfoid = ret["purchaseinfoid"] record = colleciton.find_one({"uuid": uuid}) if record: pushid = record["pushid"] transform = mongodb.transform_rate.find_one( {"_id": pushid}) if transform: purchaseinfoid = transform["purchaseinfoid"] quote = [] if transform["quote"] != "": quote = transform["quote"].split(",") if str(qinfoid) == str( purchaseinfoid): #如果是推送的批次采购则进入统计 if str(quoteid) not in quote: quote.append(str(quoteid)) mongodb.transform_rate.update( {"_id": pushid}, {'$set': { 'quote': ','.join(quote) }})
def wrapper(self, *args, **kwargs): uuid = self.get_argument("uuid", None) # 如果是从push的链接访问进来 if uuid: self.session["uuid"] = uuid self.session.save() elif self.session.get("uuid") : uuid=self.session.get("uuid") if uuid!=None: try: producer_server = KafkaProduceServer(analysis_send_topic, kafka_server) userid=self.session.get("userid") if userid==""or userid==None: userid=-1 mongodb = PymongoDateBase.instance().get_db() colleciton = mongodb.push_record record=colleciton.find_one({"uuid":uuid}) if record: quoteid=self.session.get("quoteid") if quoteid:#报价之后取的quoteid: self.session["quoteid"] = -1 self.session.save() else: quoteid=-1 producer_server.sendJson("data", {'uuid': uuid, "url": self.request.uri, "monitor_type": record["type"], "method": self.request.method,"quoteid":quoteid, "userid": userid,"messagetype":1,"createtime":int(time.time())}) producer_server.close() except Exception,ex: logger = logging.getLogger() logger.error("kafkaProduce send error(%s)"%str(ex))
def writeDB(rdd): if rdd.isEmpty() is False: resultList=rdd.collect() for item in resultList: mongodb = PymongoDateBase.instance().get_db() colleciton = mongodb.push_record uuid=item["uuid"] quoteid=item["quoteid"] sqldb = database.instance().get_session() ret=sqldb.get("select purchaseinfoid from quote where id=%s",quoteid) if ret: qinfoid=ret["purchaseinfoid"] record = colleciton.find_one({"uuid": uuid}) if record: pushid=record["pushid"] transform=mongodb.transform_rate.find_one({"_id":pushid}) if transform: purchaseinfoid=transform["purchaseinfoid"] quote=[] if transform["quote"]!="": quote=transform["quote"].split(",") if str(qinfoid)==str(purchaseinfoid):#如果是推送的批次采购则进入统计 if str(quoteid) not in quote: quote.append(str(quoteid)) mongodb.transform_rate.update({"_id":pushid}, {'$set':{'quote':','.join(quote)}})
def writeAll(rdd): #纯记录用户数据 if rdd.isEmpty() is False: resultList = rdd.collect() for item in resultList: db = PymongoDateBase.instance().get_db() colleciton = db.user_view colleciton.insert(item)
def post(self): self.log.info(self.request.arguments) token = self.get_argument("token", "") timestamp = self.get_argument("timestamp", "") signature = self.get_argument("signature", "") if verify(sms_hook_app_key, token, timestamp, signature): statusCode = self.get_argument("statusCode", 0) templateId = self.get_argument("templateId", None) phone = self.get_argument("phone", "") if templateId and templateId == "870": mongodb = PymongoDateBase.instance().get_db() colleciton = mongodb.push_record eventType = self.get_argument("eventType", None) if eventType == "2": colleciton.update({'sendid': phone}, {'$set': { 'sendstatus': 1 }}) if eventType in ("4", "5"): colleciton.update({'sendid': phone}, {'$set': { 'sendstatus': 2 }}) if statusCode in ('500', '510', '590'): #关机停机的用户 self.db.execute( "update supplier set pushstatus=0 where mobile=%s", phone) self.api_response({'status': 'success', 'message': '成功接收消息'}) else: self.api_response({'status': 'fail', 'message': '参数错误'})
def monitor_click(task): #监控点击情况 mongodb = PymongoDateBase.instance().get_db() sqldb = database.instance().get_session() # items =mongodb.transform_rate.find()#检测发送超过一天的统计记录 条件通过createtime format = "%Y-%m-%d" todaystr = datetime.now().strftime("%Y-%m-%d") print todaystr today = datetime.strptime(todaystr, format) yesterday = today - timedelta(days=1) todayStamp = time.mktime(today.timetuple()) yesterdayStamp = time.mktime(yesterday.timetuple()) print todayStamp print yesterdayStamp ret = mongodb.push_record.find({ "createtime": { "$gt": int(yesterdayStamp), "$lt": int(todayStamp) }, "type": 1 }) for item in ret: if item["click"] == 0: sqldb.execute( "update supplier set maxpush=maxpush+1 where mobile=%s", item["sendid"])
def writeAll(rdd):#纯记录用户数据 if rdd.isEmpty() is False: resultList=rdd.collect() for item in resultList: db = PymongoDateBase.instance().get_db() colleciton = db.user_view colleciton.insert(item)
def analysis_record():#每天九点定时检测 print "start analysis_record start" mongodb = PymongoDateBase.instance().get_db() sqldb = database.instance().get_session() #items =mongodb.transform_rate.find()#检测发送超过一天的统计记录 条件通过createtime format="%Y-%m-%d" todaystr=datetime.now().strftime("%Y-%m-%d") today=datetime.strptime(todaystr, format) yesterday=today- timedelta(days=1) todayStamp = time.mktime(today.timetuple()) yesterdayStamp =time.mktime(yesterday.timetuple()) print todayStamp print yesterdayStamp func = ''' function(obj,prev){ if (obj.order>prev.latest_order){ prev.latest_order=obj.order prev.latest_id=obj._id } } ''' ret = mongodb.transform_rate.group(['purchaseinfoid','type'], {"createtime":{"$gt":int(yesterdayStamp),"$lt":int(todayStamp)}}, {"latest_order": 0, "latest_id": ""}, func) for item in ret : print item id=item["latest_id"] transform=mongodb.transform_rate.find_one({"_id":id}) type=transform["type"] order=transform["order"] if order > max_push_time: continue purchaseinfoid=transform["purchaseinfoid"] purchaseinfo=sqldb.get("select status from purchase_info where id=%s",purchaseinfoid) if purchaseinfo["status"]==0: print "purchaseinfo status==0" continue accept = sqldb.query("select id from quote where purchaseinfoid=%s and state=1" % purchaseinfoid) if len(accept)>0: print "purchaseinfo accept=%s"%len(accept) continue quote_count =0 if transform["quote"]!="": quote_count=len(transform["quote"].split(",")) push_count=mongodb.push_record.find({"pushid":id}).count() if push_count!=0: click_count=mongodb.push_record.find({"pushid":id,"click":{'$gt':0}}).count() reject_rate=1 if quote_count!=0: aject = sqldb.query("select id from quote where id in(%s) and state=2" % transform["quote"]) quote_aject_count = len(aject) reject_rate=quote_aject_count/(quote_count*1.0) if click_count/(push_count*1.0)<click_rate or quote_count/(push_count*1.0)<quote_rate or reject_rate>reject_quote_rate : task = {"purchaseinfoid": purchaseinfoid, "tasktype": 1,"channel":type} print "task_generate task=%s"%task task_generate.apply_async(args=[task])
def pushPurchaseWx(openids, purchase,uuidmap,sendtype=1): if 'Linux' not in platform.system(): return templateId = 'OxXsRhlyc17kt6ubwV7F0fD8ffRl12rGGS3mnpvpoU4' if sendtype == 2: templateId = "lRRAoLj5-udp8NvSY3IY-tuRQbJb53Ca_FbAU30SdGo" link = 'http://m.yaocai.pro/purchase/purchaseinfo/%s' % purchase["purchaseinfoid"] qtime = int(purchase["createtime"]) purchase["name"] = purchase["name"].encode('utf-8') if isinstance(purchase["name"], unicode) else purchase["name"] purchase["variety"] = purchase["variety"].encode('utf-8') if isinstance(purchase["variety"], unicode) else purchase["variety"] purchase["specification"] = purchase["specification"].encode('utf-8') if isinstance(purchase["specification"], unicode) else purchase["specification"] purchase["origin"] = purchase["origin"].encode('utf-8') if isinstance(purchase["origin"], unicode) else purchase["origin"] purchase["quality"] = purchase["quality"].encode('utf-8') if isinstance(purchase["quality"], unicode) else purchase["quality"] purchase["quantity"] = purchase["quantity"].encode('utf-8') if isinstance(purchase["quantity"], unicode) else purchase["quantity"] purchase["unit"] = purchase["unit"].encode('utf-8') if isinstance(purchase["unit"], unicode) else purchase["unit"] for openid in openids: openid = openid.encode('utf-8') if isinstance(openid, unicode) else openid data = { "first": { "value":"%s 邀请您报价" % purchase["name"], "color":"#173177" }, "keyword1": { "value":"%s(%s),产地:%s,%s" % (purchase["variety"],purchase["specification"],purchase["origin"],purchase["quality"]), "color":"#173177" }, "keyword2": { "value":"%s%s" % (purchase["quantity"],purchase["unit"]), "color":"#173177" }, "keyword3":{ "value":purchase["nickname"], "color":"#173177" }, "keyword4": { "value": time.strftime("%Y年%m月%d日 %H:%M", time.localtime(qtime)), "color": "#173177" }, "remark":{ "value":"点击“详情”,立即报价", "color":"#173177" } } uuid = uuidmap[openid] sendlink=link+"?uuid="+uuid reuslt=sendwx(templateId, openid, sendlink, data,sendtype) if reuslt: message = json.loads(reuslt.encode("utf-8")) db = PymongoDateBase.instance().get_db() colleciton = db.push_record if message["errcode"]==0: colleciton.update({'uuid': uuid}, {'$set': {'sendstatus': 1}}) else: colleciton.update({'uuid': uuid}, {'$set': {'sendstatus': 2}}) time.sleep(3)
def post(self): uuid = self.session.get("uuid") userid=self.session.get("userid") content=self.get_argument("content","") pid=self.get_argument("pid","") if userid==None: userid="" if uuid==None: uuid="" mongodb = PymongoDateBase.instance().get_db() item={"uuid":uuid,"userid":userid,"content":content,"purchaseinfoid":pid,"createtime":int(time.time())} mongodb.feedback.insert(item) self.api_response({'status': 'success', 'message': '反馈成功'})
def post(self): uuid = self.session.get("uuid") userid = self.session.get("userid") content = self.get_argument("content", "") pid = self.get_argument("pid", "") if userid == None: userid = "" if uuid == None: uuid = "" mongodb = PymongoDateBase.instance().get_db() item = { "uuid": uuid, "userid": userid, "content": content, "purchaseinfoid": pid, "createtime": int(time.time()) } mongodb.feedback.insert(item) self.api_response({'status': 'success', 'message': '反馈成功'})
def monitor_click(task): #监控点击情况 mongodb = PymongoDateBase.instance().get_db() sqldb = database.instance().get_session() # items =mongodb.transform_rate.find()#检测发送超过一天的统计记录 条件通过createtime format = "%Y-%m-%d" todaystr = datetime.now().strftime("%Y-%m-%d") print todaystr today = datetime.strptime(todaystr, format) yesterday = today - timedelta(days=1) todayStamp = time.mktime(today.timetuple()) yesterdayStamp = time.mktime(yesterday.timetuple()) print todayStamp print yesterdayStamp ret = mongodb.push_record.find({"createtime": {"$gt": int(yesterdayStamp), "$lt": int(todayStamp)}, "type": 1}) for item in ret: if item["click"] == 0: sqldb.execute("update supplier set maxpush=maxpush+1 where mobile=%s", item["sendid"])
def sendPush(rdd): if rdd.isEmpty() is False: resultList = rdd.collect() for item in resultList: taskid = item["taskid"] mongodb = PymongoDateBase.instance().get_db() sqldb = database.instance().get_session() taskid = ObjectId(taskid) task = mongodb.celery_task.find_one({"_id": taskid}) if task: mongodb.celery_task.update({"_id": taskid}, {'$set': { 'status': 1 }}) purchaseinfoid = task["purchaseinfoid"] count = task["order"] channel = task["channel"] tasktype = task["tasktype"] taskinfo = mongodb.celery_task_info.find_one( {"taskid": taskid}) if taskinfo: sendlist = taskinfo["sendlist"].split(",") if tasktype == 1: purchaseinfo = sqldb.get( "select pi.id purchaseinfoid,pi.varietyid,pi.name variety,pi.specification,pi.quantity,pi.unit,pi.quality,pi.origin,pi.pushcount,p.userid,p.createtime from purchase_info pi left join purchase p on pi.purchaseid = p.id where pi.id = %s", purchaseinfoid) u = sqldb.get( "select name,nickname from users where id = %s", purchaseinfo["userid"]) purchaseinfo["name"] = u["name"] purchaseinfo["nickname"] = u["nickname"] push_user_infos = [] uuidmap = {} sendstatus = 0 # 0,未发送,1:发送成功,2:失败 colleciton = mongodb.transform_rate createtime = int(time.time()) push_id = colleciton.insert({ "purchaseinfoid": purchaseinfoid, "varietyname": purchaseinfo["variety"], "order": count, "quote": "", "type": channel, "createtime": createtime }) for send in sendlist: uuid = md5(str(time.time()) + str(send))[8:-8] sendid = send createtime = int(time.time()) push_user = { "pushid": push_id, "uuid": uuid, "createtime": createtime, "click": 0, "sendid": sendid, "sendstatus": sendstatus, "type": channel } push_user_infos.append(push_user) uuidmap[sendid] = uuid colleciton = mongodb.push_record colleciton.insert_many(push_user_infos) if channel == 1: print sendlist, purchaseinfo, uuidmap #pushPurchase(sendlist, purchaseinfo, uuidmap) attentions = [] #关注用户 notattentions = [] #非关注用户 for phone in sendlist: userinfo = None userinfo = sqldb.get( "select id,maxpush,openid from users where phone=%s", phone) print userinfo if userinfo and userinfo["openid"] != "": attentions.append(phone) else: if userinfo: notattentions.append(phone) sqldb.execute( "update users set maxpush=maxpush+1 where phone=%s", phone) else: #supplierb表里面的 attentions.append(phone) if len(notattentions) != 0: print notattentions, purchaseinfo thread.start_new_thread( pushPurchase, (notattentions, purchaseinfo, uuidmap, 2)) if len(attentions) != 0: print attentions, purchaseinfo thread.start_new_thread( pushPurchase, (attentions, purchaseinfo, uuidmap)) else: #print sendlist, purchaseinfo,uuidmap #pushPurchaseWx(sendlist, purchaseinfo,uuidmap) thread.start_new_thread( pushPurchaseWx, (sendlist, purchaseinfo, uuidmap)) elif tasktype == 2: sendid = taskinfo["sendlist"] print purchaseinfoid ret = sqldb.query( "select id from quote where purchaseinfoid =%s and state=0", purchaseinfoid) #未回复的报价个数 num = len(ret) if num != 0: ret = sqldb.query( "select id from quote where purchaseinfoid =%s and state=0 order by price", purchaseinfoid) qid = ret[0]["id"] purchaseinfo = sqldb.get( "select pi.name,pi.purchaseid,pi.unit,q.price from quote q left join purchase_info pi on q.purchaseinfoid=pi.id where q.id=%s", qid) if channel == 1: if sendid != "": uuid = md5(str(time.time()) + str(sendid))[8:-8] createtime = int(time.time()) push_user = { "pushid": "", "uuid": uuid, "createtime": createtime, "click": 0, "sendid": sendid, "sendstatus": 0, "type": channel } colleciton = mongodb.push_record record_id = colleciton.insert(push_user) colleciton = mongodb.notify_record notify_user = { "createtime": int(time.time()), "sendid": sendid, "type": channel, "purchaseinfoid": purchaseinfoid, "recordid": record_id } colleciton.insert(notify_user) #print sendid, str(num),purchaseinfo["name"].encode("utf8"),purchaseinfo["price"].encode("utf8"),purchaseinfo["unit"].encode("utf8"),str(purchaseinfo["purchaseid"]) #reply_quote_notify(sendid, str(num), purchaseinfo["name"],purchaseinfo["price"], purchaseinfo["unit"], str(purchaseinfoid)) thread.start_new_thread( reply_quote_notify, (sendid, str(num), purchaseinfo["name"], purchaseinfo["price"], purchaseinfo["unit"], str(purchaseinfoid), uuid)) elif channel == 2: print sendid if sendid != "": user = sqldb.query( "select id from users where openid=%s", sendid) if user: sendtype = 1 else: sendtype = 2 uuid = md5(str(time.time()) + str(sendid))[8:-8] createtime = int(time.time()) push_user = { "pushid": "", "uuid": uuid, "createtime": createtime, "click": 0, "sendid": sendid, "sendstatus": 0, "type": channel } colleciton = mongodb.push_record record_id = colleciton.insert(push_user) colleciton = mongodb.notify_record notify_user = { "createtime": int(time.time()), "sendid": sendid, "type": channel, "purchaseinfoid": purchaseinfoid, "recordid": record_id } colleciton.insert(notify_user) #print sendid, str(num),purchaseinfo["name"].encode("utf8"),purchaseinfo["price"].encode("utf8"),purchaseinfo["unit"].encode("utf8"),str(purchaseinfo["purchaseid"]),sendtype #reply_wx_notify(sendid, str(num), purchaseinfo["name"],purchaseinfo["price"], purchaseinfo["unit"], str(purchaseinfoid),str(purchaseinfo["purchaseid"])) thread.start_new_thread( reply_wx_notify, (sendid, str(num), purchaseinfo["name"], purchaseinfo["price"], purchaseinfo["unit"], str(purchaseinfoid), str(purchaseinfo["purchaseid"]), uuid, sendtype)) pass
def analysis_record(): #每天九点定时检测 print "start analysis_record start" mongodb = PymongoDateBase.instance().get_db() sqldb = database.instance().get_session() #items =mongodb.transform_rate.find()#检测发送超过一天的统计记录 条件通过createtime format = "%Y-%m-%d" todaystr = datetime.now().strftime("%Y-%m-%d") today = datetime.strptime(todaystr, format) yesterday = today - timedelta(days=1) todayStamp = time.mktime(today.timetuple()) yesterdayStamp = time.mktime(yesterday.timetuple()) print todayStamp print yesterdayStamp func = ''' function(obj,prev){ if (obj.order>prev.latest_order){ prev.latest_order=obj.order prev.latest_id=obj._id } } ''' ret = mongodb.transform_rate.group( ['purchaseinfoid', 'type'], {"createtime": { "$gt": int(yesterdayStamp), "$lt": int(todayStamp) }}, { "latest_order": 0, "latest_id": "" }, func) for item in ret: print item id = item["latest_id"] transform = mongodb.transform_rate.find_one({"_id": id}) type = transform["type"] order = transform["order"] if order > max_push_time: continue purchaseinfoid = transform["purchaseinfoid"] purchaseinfo = sqldb.get( "select status from purchase_info where id=%s", purchaseinfoid) if purchaseinfo["status"] == 0: print "purchaseinfo status==0" continue accept = sqldb.query( "select id from quote where purchaseinfoid=%s and state=1" % purchaseinfoid) if len(accept) > 0: print "purchaseinfo accept=%s" % len(accept) continue quote_count = 0 if transform["quote"] != "": quote_count = len(transform["quote"].split(",")) push_count = mongodb.push_record.find({"pushid": id}).count() if push_count != 0: click_count = mongodb.push_record.find({ "pushid": id, "click": { '$gt': 0 } }).count() reject_rate = 1 if quote_count != 0: aject = sqldb.query( "select id from quote where id in(%s) and state=2" % transform["quote"]) quote_aject_count = len(aject) reject_rate = quote_aject_count / (quote_count * 1.0) if click_count / (push_count * 1.0) < click_rate or quote_count / ( push_count * 1.0) < quote_rate or reject_rate > reject_quote_rate: task = { "purchaseinfoid": purchaseinfoid, "tasktype": 1, "channel": type } print "task_generate task=%s" % task task_generate.apply_async(args=[task])
def task_generate(task): #生成发送任务 mongodb = PymongoDateBase.instance().get_db() sqldb = database.instance().get_session() tasktype = task["tasktype"] purchaseinfoid = str(task["purchaseinfoid"]) channel = task["channel"] record = mongodb.celery_task.find({ "purchaseinfoid": purchaseinfoid, "channel": channel, "tasktype": tasktype }).sort("order", pymongo.DESCENDING) if record.count() == 0: order = 1 else: order = record[0]["order"] + 1 print "start order %s" % order purchaseinfo = sqldb.get( "select pi.id purchaseinfoid,pi.varietyid,pi.status,p.userid uid from purchase_info pi left join purchase p on pi.purchaseid = p.id where pi.id = %s", purchaseinfoid) if purchaseinfo == None: return elif purchaseinfo["status"] == 0: return if tasktype == 1: sendids = set() filtersend = [] if order > max_push_time: return if order != 1: #不是第一次发送,过滤发送过的 records = mongodb.transform_rate.find({ "purchaseinfoid": purchaseinfoid, "type": channel }) for item in records: pushrecord = mongodb.push_record.find({"pushid": item["_id"]}) for item in pushrecord: filtersend.append(item["sendid"]) if channel == 1: #短信渠道 print "channel=%s" % channel phonecondition = "" filtersend = [str(i) for i in filtersend] if filtersend != []: phonecondition = " and phone not in(%s)" % ",".join(filtersend) task = { "purchaseinfoid": purchaseinfoid, "tasktype": tasktype, "channel": 1, "order": order, "status": 0, "createtime": int(time.time()) } userphones = sqldb.query( "select phone from users where find_in_set(%s,varietyids) and maxpush<3 " + phonecondition + " order by pushscore limit 0,%s", purchaseinfo["varietyid"], max_wx_num) if filtersend != []: phonecondition = " and mobile not in(%s)" % ",".join( filtersend) yt = sqldb.query( "select mobile from supplier where find_in_set(%s,variety) and mobile != '' and pushstatus=1 and maxpush<3 " + phonecondition + " order by pushscore limit 0, %s", purchaseinfo["varietyid"], max_phone_num) for i in userphones: sendids.add(str(i["phone"])) for j in yt: sendids.add(str(j["mobile"])) sendids = list(set(sendids)) print "sendids= %s" % sendids elif channel == 2: #微信渠道 print "channel=%s" % channel wxcondition = "" filtersend = ["'" + str(i) + "'" for i in filtersend] if filtersend != []: wxcondition = " and openid not in(%s)" % ",".join(filtersend) task = { "purchaseinfoid": purchaseinfoid, "tasktype": tasktype, "channel": 2, "order": order, "status": 0, "createtime": int(time.time()) } userwxs = sqldb.query( "select openid from users where find_in_set(%s,varietyids) and openid!=''" + wxcondition + " order by pushscore", purchaseinfo["varietyid"]) for i in userwxs: sendids.add(str(i["openid"])) sendids = list(set(sendids)) print "sendids=%s" % sendids if len(sendids) != 0: taskid = mongodb.celery_task.insert(task) taskinfo = {"taskid": taskid, "sendlist": ",".join(sendids)} collection = mongodb.celery_task_info collection.insert(taskinfo) sendkafka.apply_async(args=[taskid]) else: #提醒采购商 if order != 1: if order > max_notify_time or (int(time.time()) - int( record[0]["createtime"])) < notify_days * 24 * 60 * 60: return sendids = [] if channel == 1: userphone = sqldb.get("select id,phone from users where id=%s", purchaseinfo["uid"]) if userphone: sendids.append(userphone["phone"]) elif channel == 2: useropenid = sqldb.get("select id,openid from users where id=%s", purchaseinfo["uid"]) if useropenid: sendids.append(useropenid["openid"]) useropenid2 = sqldb.get("select id,openid2 from users where id=%s", purchaseinfo["uid"]) if useropenid: sendids.append(useropenid2["openid2"]) print "channel=%s,sendid=%s" % (channel, sendids) if sendids != []: for sendid in sendids: task = { "purchaseinfoid": purchaseinfoid, "tasktype": tasktype, "channel": channel, "order": order, "status": 0, "createtime": int(time.time()) } taskid = mongodb.celery_task.insert(task) taskinfo = {"taskid": taskid, "sendlist": sendid} collection = mongodb.celery_task_info collection.insert(taskinfo) sendkafka.apply_async(args=[taskid])
def sendPush(rdd): if rdd.isEmpty() is False: resultList = rdd.collect() for item in resultList: taskid=item["taskid"] mongodb = PymongoDateBase.instance().get_db() sqldb = database.instance().get_session() taskid=ObjectId(taskid) task= mongodb.celery_task.find_one({"_id":taskid}) if task: mongodb.celery_task.update({"_id":taskid}, {'$set':{'status':1}}) purchaseinfoid=task["purchaseinfoid"] count=task["order"] channel=task["channel"] tasktype=task["tasktype"] taskinfo=mongodb.celery_task_info.find_one({"taskid":taskid}) if taskinfo: sendlist=taskinfo["sendlist"].split(",") if tasktype==1: purchaseinfo = sqldb.get("select pi.id purchaseinfoid,pi.varietyid,pi.name variety,pi.specification,pi.quantity,pi.unit,pi.quality,pi.origin,pi.pushcount,p.userid,p.createtime from purchase_info pi left join purchase p on pi.purchaseid = p.id where pi.id = %s", purchaseinfoid) u = sqldb.get("select name,nickname from users where id = %s", purchaseinfo["userid"]) purchaseinfo["name"] = u["name"] purchaseinfo["nickname"] = u["nickname"] push_user_infos = [] uuidmap={} sendstatus = 0 # 0,未发送,1:发送成功,2:失败 colleciton = mongodb.transform_rate createtime = int(time.time()) push_id=colleciton.insert({"purchaseinfoid":purchaseinfoid ,"varietyname":purchaseinfo["variety"],"order":count,"quote":"","type":channel,"createtime":createtime}) for send in sendlist: uuid = md5(str(time.time())+ str(send))[8:-8] sendid = send createtime = int(time.time()) push_user = {"pushid":push_id ,"uuid":uuid,"createtime":createtime,"click":0,"sendid":sendid,"sendstatus":sendstatus,"type":channel} push_user_infos.append(push_user) uuidmap[sendid]=uuid colleciton = mongodb.push_record colleciton.insert_many(push_user_infos) if channel==1: print sendlist, purchaseinfo, uuidmap #pushPurchase(sendlist, purchaseinfo, uuidmap) attentions=[]#关注用户 notattentions=[]#非关注用户 for phone in sendlist: userinfo = None userinfo=sqldb.get("select id,maxpush,openid from users where phone=%s",phone) print userinfo if userinfo and userinfo["openid"]!="": attentions.append(phone) else: if userinfo: notattentions.append(phone) sqldb.execute("update users set maxpush=maxpush+1 where phone=%s", phone) else: #supplierb表里面的 attentions.append(phone) if len(notattentions)!=0: print notattentions, purchaseinfo thread.start_new_thread(pushPurchase, (notattentions, purchaseinfo, uuidmap,2)) if len(attentions) != 0: print attentions, purchaseinfo thread.start_new_thread(pushPurchase, (attentions, purchaseinfo, uuidmap)) else: #print sendlist, purchaseinfo,uuidmap #pushPurchaseWx(sendlist, purchaseinfo,uuidmap) thread.start_new_thread(pushPurchaseWx, (sendlist, purchaseinfo, uuidmap)) elif tasktype==2: sendid=taskinfo["sendlist"] print purchaseinfoid ret = sqldb.query("select id from quote where purchaseinfoid =%s and state=0", purchaseinfoid)#未回复的报价个数 num=len(ret) if num!=0: ret= sqldb.query("select id from quote where purchaseinfoid =%s and state=0 order by price", purchaseinfoid) qid=ret[0]["id"] purchaseinfo = sqldb.get("select pi.name,pi.purchaseid,pi.unit,q.price from quote q left join purchase_info pi on q.purchaseinfoid=pi.id where q.id=%s",qid) if channel==1: if sendid!="": uuid = md5(str(time.time()) + str(sendid))[8:-8] createtime = int(time.time()) push_user = {"pushid": "", "uuid": uuid, "createtime": createtime, "click": 0, "sendid": sendid, "sendstatus": 0, "type": channel} colleciton = mongodb.push_record record_id=colleciton.insert(push_user) colleciton = mongodb.notify_record notify_user = {"createtime": int(time.time()), "sendid": sendid, "type": channel,"purchaseinfoid":purchaseinfoid,"recordid":record_id} colleciton.insert(notify_user) #print sendid, str(num),purchaseinfo["name"].encode("utf8"),purchaseinfo["price"].encode("utf8"),purchaseinfo["unit"].encode("utf8"),str(purchaseinfo["purchaseid"]) #reply_quote_notify(sendid, str(num), purchaseinfo["name"],purchaseinfo["price"], purchaseinfo["unit"], str(purchaseinfoid)) thread.start_new_thread(reply_quote_notify,(sendid, str(num), purchaseinfo["name"],purchaseinfo["price"], purchaseinfo["unit"], str(purchaseinfoid),uuid)) elif channel==2: print sendid if sendid!="": user=sqldb.query("select id from users where openid=%s", sendid) if user: sendtype=1 else: sendtype=2 uuid = md5(str(time.time()) + str(sendid))[8:-8] createtime = int(time.time()) push_user = {"pushid": "", "uuid": uuid, "createtime": createtime, "click": 0, "sendid": sendid, "sendstatus": 0, "type": channel} colleciton = mongodb.push_record record_id = colleciton.insert(push_user) colleciton = mongodb.notify_record notify_user = {"createtime": int(time.time()), "sendid": sendid, "type": channel,"purchaseinfoid":purchaseinfoid,"recordid":record_id} colleciton.insert(notify_user) #print sendid, str(num),purchaseinfo["name"].encode("utf8"),purchaseinfo["price"].encode("utf8"),purchaseinfo["unit"].encode("utf8"),str(purchaseinfo["purchaseid"]),sendtype #reply_wx_notify(sendid, str(num), purchaseinfo["name"],purchaseinfo["price"], purchaseinfo["unit"], str(purchaseinfoid),str(purchaseinfo["purchaseid"])) thread.start_new_thread(reply_wx_notify, (sendid, str(num), purchaseinfo["name"],purchaseinfo["price"], purchaseinfo["unit"], str(purchaseinfoid),str(purchaseinfo["purchaseid"]),uuid,sendtype)) pass
for item in alluser: user={"phone":item,"notclick":0} notclick.append(user) ret = mongodb.push_record.find({"createtime": {"$gt": int(yesterdayStamp), "$lt": int(todayStamp)},"type":1}) for item in ret: if item["click"]==0: update(item["sendid"],notclick) notnum=0 for item in notclick: if item["notclick"]>3: notnum+=1 print totalcount print notnum ''' mongodb = PymongoDateBase.instance().get_db() sqldb = database.instance().get_session() # items =mongodb.transform_rate.find()#检测发送超过一天的统计记录 条件通过createtime format = "%Y-%m-%d" todaystr = datetime.now().strftime("%Y-%m-%d") print todaystr todaystr="2016-09-28" today = datetime.strptime(todaystr, format) yesterday = today - timedelta(days=4) todayStamp = time.mktime(today.timetuple()) yesterdayStamp = time.mktime(yesterday.timetuple()) print todayStamp print yesterdayStamp ret = mongodb.push_record.find({"createtime": {"$gt": int(yesterdayStamp), "$lt": int(todayStamp)}, "type": 1})
def pushPurchaseWx(openids, purchase, uuidmap, sendtype=1): if 'Linux' not in platform.system(): return templateId = 'FsfroTOrMI_yvbByrKQikIYzxaDvY-p0VMfw6ehpa7M' #if sendtype == 2: # templateId = "lRRAoLj5-udp8NvSY3IY-tuRQbJb53Ca_FbAU30SdGo" link = 'http://m.yaobest.com/purchase/purchaseinfo/%s' % purchase[ "purchaseinfoid"] qtime = int(purchase["createtime"]) purchase["name"] = purchase["name"].encode('utf-8') if isinstance( purchase["name"], unicode) else purchase["name"] purchase["variety"] = purchase["variety"].encode('utf-8') if isinstance( purchase["variety"], unicode) else purchase["variety"] purchase["specification"] = purchase["specification"].encode( 'utf-8') if isinstance(purchase["specification"], unicode) else purchase["specification"] purchase["origin"] = purchase["origin"].encode('utf-8') if isinstance( purchase["origin"], unicode) else purchase["origin"] purchase["quality"] = purchase["quality"].encode('utf-8') if isinstance( purchase["quality"], unicode) else purchase["quality"] purchase["quantity"] = purchase["quantity"].encode('utf-8') if isinstance( purchase["quantity"], unicode) else purchase["quantity"] purchase["unit"] = purchase["unit"].encode('utf-8') if isinstance( purchase["unit"], unicode) else purchase["unit"] for openid in openids: openid = openid.encode('utf-8') if isinstance(openid, unicode) else openid data = { "first": { "value": "%s 邀请您报价" % purchase["name"], "color": "#173177" }, "keyword1": { "value": "%s(%s),产地:%s,%s" % (purchase["variety"], purchase["specification"], purchase["origin"], purchase["quality"]), "color": "#173177" }, "keyword2": { "value": "%s%s" % (purchase["quantity"], purchase["unit"]), "color": "#173177" }, "keyword3": { "value": purchase["nickname"], "color": "#173177" }, "keyword4": { "value": time.strftime("%Y年%m月%d日 %H:%M", time.localtime(qtime)), "color": "#173177" }, "remark": { "value": "点击“详情”,立即报价", "color": "#173177" } } uuid = uuidmap[openid] sendlink = link + "?uuid=" + uuid reuslt = sendwx(templateId, openid, sendlink, data, sendtype) if reuslt: message = json.loads(reuslt.encode("utf-8")) db = PymongoDateBase.instance().get_db() colleciton = db.push_record if message["errcode"] == 0: colleciton.update({'uuid': uuid}, {'$set': {'sendstatus': 1}}) else: colleciton.update({'uuid': uuid}, {'$set': {'sendstatus': 2}}) time.sleep(3)
def task_generate(task):#生成发送任务 mongodb = PymongoDateBase.instance().get_db() sqldb = database.instance().get_session() tasktype=task["tasktype"] purchaseinfoid = str(task["purchaseinfoid"]) channel = task["channel"] record = mongodb.celery_task.find( {"purchaseinfoid": purchaseinfoid, "channel": channel, "tasktype": tasktype}).sort("order", pymongo.DESCENDING) if record.count() == 0: order = 1 else: order = record[0]["order"] + 1 print "start order %s"%order purchaseinfo = sqldb.get( "select pi.id purchaseinfoid,pi.varietyid,pi.status,p.userid uid from purchase_info pi left join purchase p on pi.purchaseid = p.id where pi.id = %s", purchaseinfoid) if purchaseinfo==None: return elif purchaseinfo["status"]==0: return if tasktype==1: sendids = set() filtersend = [] if order>max_push_time: return if order!=1:#不是第一次发送,过滤发送过的 records = mongodb.transform_rate.find({"purchaseinfoid": purchaseinfoid, "type": channel}) for item in records: pushrecord= mongodb.push_record.find({"pushid": item["_id"]}) for item in pushrecord: filtersend.append(item["sendid"]) if channel == 1: #短信渠道 print "channel=%s"%channel phonecondition = "" filtersend = [str(i) for i in filtersend] if filtersend != []: phonecondition = " and phone not in(%s)" % ",".join(filtersend) task = {"purchaseinfoid": purchaseinfoid, "tasktype": tasktype, "channel": 1, "order": order, "status": 0,"createtime":int(time.time())} userphones = sqldb.query( "select phone from users where find_in_set(%s,varietyids) and maxpush<3 " + phonecondition + " order by pushscore limit 0,%s", purchaseinfo["varietyid"], max_wx_num) if filtersend != []: phonecondition = " and mobile not in(%s)" % ",".join(filtersend) yt = sqldb.query( "select mobile from supplier where find_in_set(%s,variety) and mobile != '' and pushstatus=1 and maxpush<3 " + phonecondition + " order by pushscore limit 0, %s", purchaseinfo["varietyid"], max_phone_num) for i in userphones: sendids.add(str(i["phone"])) for j in yt: sendids.add(str(j["mobile"])) sendids = list(set(sendids)) print "sendids= %s"%sendids elif channel==2: #微信渠道 print "channel=%s"%channel wxcondition="" filtersend = ["'"+str(i)+"'" for i in filtersend] if filtersend!=[]: wxcondition = " and openid not in(%s)" % ",".join(filtersend) task = {"purchaseinfoid": purchaseinfoid, "tasktype": tasktype, "channel": 2, "order": order, "status": 0,"createtime":int(time.time())} userwxs = sqldb.query( "select openid from users where find_in_set(%s,varietyids) and openid!=''" + wxcondition + " order by pushscore", purchaseinfo["varietyid"]) for i in userwxs: sendids.add(str(i["openid"])) sendids = list(set(sendids)) print "sendids=%s"%sendids if len(sendids) != 0: taskid = mongodb.celery_task.insert(task) taskinfo = {"taskid": taskid, "sendlist": ",".join(sendids)} collection = mongodb.celery_task_info collection.insert(taskinfo) sendkafka.apply_async(args=[taskid]) else: #提醒采购商 if order!=1: if order>max_notify_time or (int(time.time())-int(record[0]["createtime"]))<notify_days*24*60*60: return sendids=[] if channel==1: userphone=sqldb.get("select id,phone from users where id=%s",purchaseinfo["uid"]) if userphone: sendids.append(userphone["phone"]) elif channel==2: useropenid=sqldb.get("select id,openid from users where id=%s",purchaseinfo["uid"]) if useropenid: sendids.append(useropenid["openid"]) useropenid2 = sqldb.get("select id,openid2 from users where id=%s", purchaseinfo["uid"]) if useropenid: sendids.append(useropenid2["openid2"]) print "channel=%s,sendid=%s"%(channel,sendids) if sendids!=[]: for sendid in sendids: task = {"purchaseinfoid": purchaseinfoid, "tasktype": tasktype, "channel": channel, "order": order, "status": 0, "createtime": int(time.time())} taskid = mongodb.celery_task.insert(task) taskinfo = {"taskid": taskid, "sendlist": sendid} collection = mongodb.celery_task_info collection.insert(taskinfo) sendkafka.apply_async(args=[taskid])
def get(self): pushid=self.get_argument("id", "") mongodb = PymongoDateBase.instance().get_db() record={} detail=[] page = self.get_argument("page", 0) page = (int(page) - 1) if page > 0 else 0 limit_num = config.conf['POST_NUM'] skip_num = page * config.conf['POST_NUM'] num = 0 if pushid!="": item=mongodb.transform_rate.find_one(ObjectId(pushid)) purchaseinfoid = item["purchaseinfoid"] ret = self.db.get("select varietyid,name from purchase_info where id=%s", purchaseinfoid) if ret: item["varietyname"] = ret["name"] else: item["varietyname"] = u"不存在" if item["quote"] == "": item["quotetime"] = 0 else: item["quotetime"] = len(item["quote"].split(",")) item["type"] = monitor_type[str(item["type"])] push_count = mongodb.push_record.find({"pushid": item["_id"]}).count() item["pushcount"] = push_count if push_count != 0: item["clickcount"] = mongodb.push_record.find({"pushid": item["_id"], "click": {'$gt': 0}}).count() else: item["clickcount"] = 0 timeArray = time.localtime(float(item["createtime"])) item["time"] = time.strftime("%Y-%m-%d %H:%M:%S", timeArray) record = {"purchaseinfoid": purchaseinfoid, "order": item["order"], "varietyname": item["varietyname"], "quotetime": item["quotetime"], "type": item["type"], "pushcount": item["pushcount"], "clickcount": item["clickcount"], "time": item["time"], "id": item["_id"] } pushrecords=mongodb.push_record.find({"pushid":item["_id"]}).skip(skip_num).limit(limit_num) num=mongodb.push_record.find({"pushid":item["_id"]}).count() for pushrecord in pushrecords: uuid=pushrecord["uuid"] sendstatus=pushrecord["sendstatus"] monitortype=pushrecord["type"] sendid=pushrecord["sendid"] item={} click=0 register=0#0未注册,1注册,2已经注册过 quote=0 accept=0 reject=0 phone="" login=0 timeArray = time.localtime(float(pushrecord["createtime"])) item["createtime"] = time.strftime("%Y-%m-%d %H:%M:%S", timeArray) if pushrecord["click"]!=0: click=1 if monitortype==1: phone=sendid user=self.db.get("select id from users where phone=%s",sendid) if user: registerurl=mongodb.user_view.find({"uuid":uuid,"url":{"$regex":"^/regsuccess"}}).count() if registerurl!=0: register= 1 else: register = 2 else: users = self.db.query("select id,phone from users where openid=%s", sendid) user2 = self.db.query("select id,phone from users where openid2=%s", sendid) if users: phone=users[0]["phone"] elif user2: phone=user2[0]["phone"] register = 2 loginurl=mongodb.user_view.find({"uuid":uuid,"userid":{"$ne":-1}}).count() if loginurl!=0: login=1 quoteurl=mongodb.user_view.find_one({"uuid":uuid,"url":{"$regex":"^/quotesuccess"}}) if quoteurl: quote=1 qid=quoteurl["quoteid"] quoteinfo=self.db.get("select id state from quote where id=%s",qid) if quoteinfo: if quoteinfo["state"]==1: accept=1 elif quoteinfo["state"]==2: reject=1 item["uuid"]=uuid item["sendstatus"]=sendstatus item["click"]=click item["register"]=register item["quote"]=quote item["accept"]=accept item["reject"]=reject item["phone"]=phone supplier=self.db.query("select id from supplier where mobile=%s",phone) if supplier: item["supplierid"]=supplier[0]["id"] else: item["supplierid"]=-1 item["login"]=login detail.append(item) query_str = {} query_str["id"] = pushid nav = { 'model': 'stat/pushrecord/detail', 'cur': page + 1, 'num': num, 'query': "%s" % urlencode(query_str) } self.render("record_detail.html",record=record,detail=detail,nav=nav)
def get(self): starttime= self.get_argument("starttime", "") endtime=self.get_argument("endtime", "") channel=self.get_argument("channel", -1) pid = self.get_argument("pid", "") push_type = self.get_argument("type", 1) mongodb = PymongoDateBase.instance().get_db() records=[] start_time = 0 end_time = 0 page = self.get_argument("page", 0) page = (int(page) - 1) if page > 0 else 0 limit_num=config.conf['POST_NUM'] skip_num=page * config.conf['POST_NUM'] num=0 if starttime != "" and endtime != "": start_time = int(time.mktime(datetime.strptime(str(starttime), "%Y-%m-%d %H:%M:%S").timetuple())) end_time = int(time.mktime(datetime.strptime(str(endtime), "%Y-%m-%d %H:%M:%S").timetuple())) condition={} if start_time != 0 and end_time != 0: condition["createtime"]={"$gt":int(start_time),"$lt":int(end_time)} if int(channel)!=-1: condition["type"]=int(channel) if pid!="": condition["purchaseinfoid"]=pid if int(push_type)==1: items = mongodb.transform_rate.find(condition).sort([("createtime",-1)]).skip(skip_num).limit(limit_num) num =mongodb.transform_rate.find(condition).count() for item in items: purchaseinfoid=item["purchaseinfoid"] ret=self.db.get("select varietyid,name from purchase_info where id=%s",purchaseinfoid) if ret: item["varietyname"]=ret["name"] else: if item["varietyname"]=="": item["varietyname"]="不存在" else: item["varietyname"] =item["varietyname"].encode("utf8")+"(不存在)" if item["quote"]=="": item["quotetime"]=0 else: item["quotetime"]=len(item["quote"].split(",")) item["type"]=monitor_type[str(item["type"])] push_count = mongodb.push_record.find({"pushid": item["_id"]}).count() item["pushcount"]=push_count if push_count != 0: item["clickcount"] = mongodb.push_record.find({"pushid": item["_id"], "click": {'$gt': 0}}).count() else: item["clickcount"] =0 timeArray = time.localtime(float(item["createtime"])) item["time"] = time.strftime("%Y-%m-%d %H:%M:%S", timeArray) record={"purchaseinfoid":purchaseinfoid,"order":item["order"],"varietyname":item["varietyname"], "quotetime": item["quotetime"],"type":item["type"],"pushcount":item["pushcount"],"clickcount":item["clickcount"], "time":item["time"],"id":item["_id"] } records.append(record) elif int(push_type)==2: items = mongodb.notify_record.find(condition).sort([("createtime",-1)]).skip(skip_num).limit(limit_num) num=mongodb.notify_record.find(condition).count() for item in items: purchaseinfoid = item["purchaseinfoid"] ret=self.db.get("select varietyid,name from purchase_info where id=%s",purchaseinfoid) if ret: item["varietyname"]=ret["name"] else: item["varietyname"] =u"不存在" #push_count = mongodb.push_record.find({"pushid": item["_id"]}).count() item["pushcount"] = 1 item["type"] = monitor_type[str(item["type"])] timeArray = time.localtime(float(item["createtime"])) item["time"] = time.strftime("%Y-%m-%d %H:%M:%S", timeArray) item["click"]=0 #if item["recordid"]!="": # recordid=item["recordid"] # pushrecord=mongodb.push_record.find_one({"_id": recordid}) # if pushrecord and pushrecord["click"]!=0: # item["click"]=1 record = {"purchaseinfoid": purchaseinfoid, "varietyname": item["varietyname"], "type": item["type"], "pushcount": item["pushcount"],"time": item["time"], "id": item["_id"] } records.append(record) query_str={} query_str["type"]=push_type if starttime!="": query_str["starttime"]=starttime if endtime!="": query_str["endtime"] = endtime if int(channel)!=-1: query_str["channel"] = channel nav = { 'model': 'stat/pushrecord', 'cur': page + 1, 'num': num, 'query': "%s" % urlencode(query_str), } self.render("push_record.html", starttime=starttime,channel=channel,pid=pid, endtime=endtime,type=push_type,records=records,nav=nav)