def onReceiveMsg(self, msg): # @UnusedVariable """ :param msg: """ if msg.msgType == const_kuaishou.DATA_TYPE_TAG_NAME: device = const_kuaishou.randomDevice() target = self.url.format(did=device['did'], ud=device['ud']) urlPack = urlprovider.UrlPack(priority=20, url=target) if msg.getExtra('pcursor') == None: urlPack.setForm( self.form.format(tag=msg.msgData, token=device['token'])) else: urlPack.setForm( self.form_second.format(tag=msg.msgData, pcursor=msg.getExtra('pcursor'), token=device['token'])) urlPack.fillMsg(msg, self.pipe) urlPack.addKey('tag', msg.msgData) urlPack.addKey('topic_id', msg.getExtra('topic_id')) urlPack.addKey('ext', {"token_client_salt": device['token_client_salt']}) self.add(urlPack) return True return False
def onReceiveMsg(self, msg): # @UnusedVariable """ :param msg: """ if msg.msgType == const.DATA_TYPE_AUTHOR: user_id = msg.msgData if msg.getExtra("max_time") is None: target = self.url_first.format( user_id=user_id, iid=const_huoshan.get_iid(), device_id=const_huoshan.get_device_id(), uuid=const_huoshan.get_uuid(), openudid=const_huoshan.get_openudid()) else: target = self.url_next.format( max_time=msg.getExtra("max_time"), user_id=user_id, iid=const_huoshan.get_iid(), device_id=const_huoshan.get_device_id(), uuid=const_huoshan.get_uuid(), openudid=const_huoshan.get_openudid()) urlPack = urlprovider.UrlPack(priority=20, url=target) urlPack.fillMsg(msg, self.pipe) self.add(urlPack) return True return False
def onReceiveMsg(self, msg): #仅处理话题 if msg.msgType == const_douyin.DATA_TYPE_TOPIC: for offset in range(0, 5): #self.addUrl(self.concat(msg.msgData, offset)) target = self.concat(msg.msgData, offset) urlPack = urlprovider.UrlPack(priority=20, url=target) urlPack.fillMsg(msg, self.pipe) self.add(urlPack) return True return False
def onReceiveMsg(self, msg): """ :param msg: """ log.debug("KuaiShouAuthorDetailProvider receive {}".format(msg)) if msg.msgType == const_kuaishou.DATA_TYPE_AUTHOR: urlPack = urlprovider.UrlPack(priority=0, url=self.url) urlPack.setForm(self.form.format(user=msg.msgData)) urlPack.fillMsg(msg, self.pipe) self.add(urlPack) return True return False
def onReceiveMsg(self, msg): """ :param msg: """ log.debug("KuaiShouAuthorVideosProvider receive {}".format(msg)) if msg.msgType == const_kuaishou.DATA_TYPE_AUTHOR: urlPack = urlprovider.UrlPack(priority=0, url=self.url) if msg.getExtra("pcursor") is None: urlPack.setForm(self.form_first.format(user_id=msg.msgData)) else: urlPack.setForm(self.form_next.format(user_id=msg.msgData, pcursor=msg.getExtra("pcursor"))) urlPack.fillMsg(msg, self.pipe) self.add(urlPack) return True return False
def onReceiveMsg(self, msg): """ :param msg: """ log.debug("HuoshanMainFeedProvider receive {}".format(msg)) if msg.msgType == const_huoshan.DATA_TYPE_MAINFEED: target = self.url.format(iid=const_huoshan.get_iid(), device_id=const_huoshan.get_device_id(), openudid=const_huoshan.get_openudid(), uuid=const_huoshan.get_uuid()) urlPack = urlprovider.UrlPack(priority=0, url=target) urlPack.fillMsg(msg, self.pipe) self.add(urlPack) return True return False
def onReceiveMsg(self, msg): """ :param msg: """ log.debug("DouyinNearByProvider receive {}".format(msg)) if msg.msgType == const_douyin.DATA_TYPE_NEARBY: target = self.url.format(iid=const_douyin.get_iid(), device_id=const_douyin.get_device_id(), openudid=const_douyin.get_openudid(), uuid=const_douyin.get_uuid()) urlPack = urlprovider.UrlPack(priority=20, url=target) urlPack.fillMsg(msg, self.pipe) self.add(urlPack) return True return False
def onReceiveMsg(self, msg): # @UnusedVariable """ :param msg: """ if msg.msgType == const_douyin.DATA_TYPE_TOPIC: target = self.url.format(ch_id=msg.msgData, iid=const_douyin.get_iid(), device_id=const_douyin.get_device_id(), uuid=const_douyin.get_uuid(), openudid=const_douyin.get_openudid()) urlPack = urlprovider.UrlPack(priority=20, url=target) urlPack.fillMsg(msg, self.pipe) self.add(urlPack) return True return False
def onReceiveMsg(self, msg): """ :param msg: """ log.debug("KuaiShouVideoDetailProvider receive {}".format(msg)) if msg.msgType == const_kuaishou.DATA_TYPE_VIDEO: key = dbtools.gen_object_key(const.DATA_TYPE_VIDEO, 'kuaishou', msg.msgData) if self.db.getOne(const.getTable('VIDEO'), key, '_key_') is None: urlPack = urlprovider.UrlPack(priority=0, url=self.url) urlPack.setForm(self.form.format(vid=msg.msgData)) urlPack.fillMsg(msg, self.pipe) self.add(urlPack) return True else: log.debug("vid:{} has already inserted".format(msg.msgData)) return False
def onReceiveMsg(self, msg): # @UnusedVariable """ :param msg: """ if msg.msgType == const_kuaishou.DATA_TYPE_TAG_NAME: urlPack = urlprovider.UrlPack(priority=20, url=self.url) if msg.getExtra('pcursor') == None: form = json.dumps({"tag":msg.msgData, "count":20, "pcursor":""}) else: form = json.dumps({"tag":msg.msgData, "count":20, "pcursor":msg.getExtra('pcursor')}) urlPack.setForm(form) urlPack.fillMsg(msg, self.pipe) urlPack.addKey('tag', msg.msgData) #urlPack.addKey('topic_id', msg.getExtra('topic_id')) self.add(urlPack) return True return False
def onReceiveMsg(self, msg): # @UnusedVariable """ :param msg: """ if msg.msgType == const_kuaishou.DATA_TYPE_TAG_KEYWORD: device = const_kuaishou.randomDevice() msg.msgData = urllib.quote_plus(urllib.unquote(msg.msgData)) target = self.url.format(ud=device['ud'], did=device['did']) urlPack = urlprovider.UrlPack(priority=20, url=target) urlPack.setForm( self.form.format(keyword=msg.msgData, token=device['token'])) urlPack.fillMsg(msg, self.pipe) urlPack.addKey('keyword', msg.msgData) urlPack.addKey('ext', {"token_client_salt": device['token_client_salt']}) self.add(urlPack) return True return False
def onReceiveMsg(self, msg): # @UnusedVariable """ :param msg: """ if msg.msgType == const_douyin.DATA_TYPE_TOPIC_KEYWORD: keyword = msg.msgData target = self.url.format(keyword=keyword, iid=const_douyin.get_iid(), device_id=const_douyin.get_device_id(), openudid=const_douyin.get_openudid(), uuid=const_douyin.get_uuid(), cursor=msg.getKey('cursor', 0)) urlPack = urlprovider.UrlPack(1, url=target) urlPack.addKey('keyword', keyword) urlPack.addKey('cursor', msg.getKey('cursor', 0)) urlPack.fillMsg(msg, self.pipe) self.add(urlPack) return True return False
def onReceiveMsg(self, msg): """ :param msg: """ log.debug("KuaiShouMainFeedProvider receive {}".format(msg)) if msg.msgType == const_kuaishou.DATA_TYPE_KEYWORD: device = const_kuaishou.randomDevice() target = self.url.format(did=device['did'], ud=device['ud']) msg.msgData = urllib.quote_plus(urllib.unquote(msg.msgData)) pcursor = msg.getKey('pcursor', 1) urlPack = urlprovider.UrlPack(priority=0, url=target) urlPack.setForm( self.form.format(pcursor=pcursor, keyword=msg.msgData, token=device['token'])) urlPack.fillMsg(msg, self.pipe) urlPack.addKey('keyword', msg.msgData) urlPack.addKey('pcursor', pcursor) urlPack.addKey('ext', {"token_client_salt": device['token_client_salt']}) self.add(urlPack) return True return False
def onReceiveMsg(self, msg): """ :param msg: """ log.debug("DouyinAuthorVideoProvider receive {}".format(msg)) if msg.msgType == const_douyin.DATA_TYPE_AUTHOR: cursor = 0 if msg.getExtra("cursor") is None: cursor = 0 else: cursor = msg.getExtra("cursor") target = self.url.format(user_id=msg.msgData, cursor=cursor, iid=const_douyin.get_iid(), device_id=const_douyin.get_device_id(), openudid=const_douyin.get_openudid(), uuid=const_douyin.get_uuid()) urlPack = urlprovider.UrlPack(priority=20, url=target) urlPack.fillMsg(msg, self.pipe) self.add(urlPack) return True return False
def onReceiveMsg(self, msg): # @UnusedVariable """ :param msg: """ if msg.msgType == const_huoshan.DATA_TYPE_VIDEO: if msg.getExtra("offset") is None: offset = 0 else: offset = msg.getExtra("offset") target = self.url.format(vid=msg.msgData, offset=offset, iid=const_huoshan.get_iid(), device_id=const_huoshan.get_device_id(), openudid=const_huoshan.get_openudid(), uuid=const_huoshan.get_uuid()) urlPack = urlprovider.UrlPack(priority=20, url=target) urlPack.fillMsg(msg, self.pipe) urlPack.addKey('vid', msg.msgData) urlPack.addKey('offset', offset) self.add(urlPack) return True return False