def onReceiveMsg(self, msg): # @UnusedVariable
"""
:param msg:
"""
if msg.msgType == const_kuaishou.DATA_TYPE_TAG_NAME:
device = const_kuaishou.randomDevice()
target = self.url.format(did=device['did'], ud=device['ud'])
urlPack = urlprovider.UrlPack(priority=20, url=target)
if msg.getExtra('pcursor') == None:
urlPack.setForm(
self.form.format(tag=msg.msgData, token=device['token']))
else:
urlPack.setForm(
self.form_second.format(tag=msg.msgData,
pcursor=msg.getExtra('pcursor'),
token=device['token']))
urlPack.fillMsg(msg, self.pipe)
urlPack.addKey('tag', msg.msgData)
urlPack.addKey('topic_id', msg.getExtra('topic_id'))
urlPack.addKey('ext',
{"token_client_salt": device['token_client_salt']})
self.add(urlPack)
return True
return False
def onReceiveMsg(self, msg): # @UnusedVariable
"""
:param msg:
"""
if msg.msgType == const.DATA_TYPE_AUTHOR:
user_id = msg.msgData
if msg.getExtra("max_time") is None:
target = self.url_first.format(
user_id=user_id,
iid=const_huoshan.get_iid(),
device_id=const_huoshan.get_device_id(),
uuid=const_huoshan.get_uuid(),
openudid=const_huoshan.get_openudid())
else:
target = self.url_next.format(
max_time=msg.getExtra("max_time"),
user_id=user_id,
iid=const_huoshan.get_iid(),
device_id=const_huoshan.get_device_id(),
uuid=const_huoshan.get_uuid(),
openudid=const_huoshan.get_openudid())
urlPack = urlprovider.UrlPack(priority=20, url=target)
urlPack.fillMsg(msg, self.pipe)
self.add(urlPack)
return True
return False
def onReceiveMsg(self, msg):
#仅处理话题
if msg.msgType == const_douyin.DATA_TYPE_TOPIC:
for offset in range(0, 5):
#self.addUrl(self.concat(msg.msgData, offset))
target = self.concat(msg.msgData, offset)
urlPack = urlprovider.UrlPack(priority=20, url=target)
urlPack.fillMsg(msg, self.pipe)
self.add(urlPack)
return True
return False
def onReceiveMsg(self, msg):
"""
:param msg:
"""
log.debug("KuaiShouAuthorDetailProvider receive {}".format(msg))
if msg.msgType == const_kuaishou.DATA_TYPE_AUTHOR:
urlPack = urlprovider.UrlPack(priority=0, url=self.url)
urlPack.setForm(self.form.format(user=msg.msgData))
urlPack.fillMsg(msg, self.pipe)
self.add(urlPack)
return True
return False
def onReceiveMsg(self, msg):
"""
:param msg:
"""
log.debug("KuaiShouAuthorVideosProvider receive {}".format(msg))
if msg.msgType == const_kuaishou.DATA_TYPE_AUTHOR:
urlPack = urlprovider.UrlPack(priority=0, url=self.url)
if msg.getExtra("pcursor") is None:
urlPack.setForm(self.form_first.format(user_id=msg.msgData))
else:
urlPack.setForm(self.form_next.format(user_id=msg.msgData, pcursor=msg.getExtra("pcursor")))
urlPack.fillMsg(msg, self.pipe)
self.add(urlPack)
return True
return False
def onReceiveMsg(self, msg):
"""
:param msg:
"""
log.debug("HuoshanMainFeedProvider receive {}".format(msg))
if msg.msgType == const_huoshan.DATA_TYPE_MAINFEED:
target = self.url.format(iid=const_huoshan.get_iid(),
device_id=const_huoshan.get_device_id(),
openudid=const_huoshan.get_openudid(),
uuid=const_huoshan.get_uuid())
urlPack = urlprovider.UrlPack(priority=0, url=target)
urlPack.fillMsg(msg, self.pipe)
self.add(urlPack)
return True
return False
def onReceiveMsg(self, msg):
"""
:param msg:
"""
log.debug("DouyinNearByProvider receive {}".format(msg))
if msg.msgType == const_douyin.DATA_TYPE_NEARBY:
target = self.url.format(iid=const_douyin.get_iid(),
device_id=const_douyin.get_device_id(),
openudid=const_douyin.get_openudid(),
uuid=const_douyin.get_uuid())
urlPack = urlprovider.UrlPack(priority=20, url=target)
urlPack.fillMsg(msg, self.pipe)
self.add(urlPack)
return True
return False
def onReceiveMsg(self, msg): # @UnusedVariable
"""
:param msg:
"""
if msg.msgType == const_douyin.DATA_TYPE_TOPIC:
target = self.url.format(ch_id=msg.msgData,
iid=const_douyin.get_iid(),
device_id=const_douyin.get_device_id(),
uuid=const_douyin.get_uuid(),
openudid=const_douyin.get_openudid())
urlPack = urlprovider.UrlPack(priority=20, url=target)
urlPack.fillMsg(msg, self.pipe)
self.add(urlPack)
return True
return False
def onReceiveMsg(self, msg):
"""
:param msg:
"""
log.debug("KuaiShouVideoDetailProvider receive {}".format(msg))
if msg.msgType == const_kuaishou.DATA_TYPE_VIDEO:
key = dbtools.gen_object_key(const.DATA_TYPE_VIDEO, 'kuaishou', msg.msgData)
if self.db.getOne(const.getTable('VIDEO'), key, '_key_') is None:
urlPack = urlprovider.UrlPack(priority=0, url=self.url)
urlPack.setForm(self.form.format(vid=msg.msgData))
urlPack.fillMsg(msg, self.pipe)
self.add(urlPack)
return True
else:
log.debug("vid:{} has already inserted".format(msg.msgData))
return False
def onReceiveMsg(self, msg): # @UnusedVariable
"""
:param msg:
"""
if msg.msgType == const_kuaishou.DATA_TYPE_TAG_NAME:
urlPack = urlprovider.UrlPack(priority=20, url=self.url)
if msg.getExtra('pcursor') == None:
form = json.dumps({"tag":msg.msgData, "count":20, "pcursor":""})
else:
form = json.dumps({"tag":msg.msgData, "count":20, "pcursor":msg.getExtra('pcursor')})
urlPack.setForm(form)
urlPack.fillMsg(msg, self.pipe)
urlPack.addKey('tag', msg.msgData)
#urlPack.addKey('topic_id', msg.getExtra('topic_id'))
self.add(urlPack)
return True
return False
def onReceiveMsg(self, msg): # @UnusedVariable
"""
:param msg:
"""
if msg.msgType == const_kuaishou.DATA_TYPE_TAG_KEYWORD:
device = const_kuaishou.randomDevice()
msg.msgData = urllib.quote_plus(urllib.unquote(msg.msgData))
target = self.url.format(ud=device['ud'], did=device['did'])
urlPack = urlprovider.UrlPack(priority=20, url=target)
urlPack.setForm(
self.form.format(keyword=msg.msgData, token=device['token']))
urlPack.fillMsg(msg, self.pipe)
urlPack.addKey('keyword', msg.msgData)
urlPack.addKey('ext',
{"token_client_salt": device['token_client_salt']})
self.add(urlPack)
return True
return False
def onReceiveMsg(self, msg): # @UnusedVariable
"""
:param msg:
"""
if msg.msgType == const_douyin.DATA_TYPE_TOPIC_KEYWORD:
keyword = msg.msgData
target = self.url.format(keyword=keyword,
iid=const_douyin.get_iid(),
device_id=const_douyin.get_device_id(),
openudid=const_douyin.get_openudid(),
uuid=const_douyin.get_uuid(),
cursor=msg.getKey('cursor', 0))
urlPack = urlprovider.UrlPack(1, url=target)
urlPack.addKey('keyword', keyword)
urlPack.addKey('cursor', msg.getKey('cursor', 0))
urlPack.fillMsg(msg, self.pipe)
self.add(urlPack)
return True
return False
def onReceiveMsg(self, msg):
"""
:param msg:
"""
log.debug("KuaiShouMainFeedProvider receive {}".format(msg))
if msg.msgType == const_kuaishou.DATA_TYPE_KEYWORD:
device = const_kuaishou.randomDevice()
target = self.url.format(did=device['did'], ud=device['ud'])
msg.msgData = urllib.quote_plus(urllib.unquote(msg.msgData))
pcursor = msg.getKey('pcursor', 1)
urlPack = urlprovider.UrlPack(priority=0, url=target)
urlPack.setForm(
self.form.format(pcursor=pcursor,
keyword=msg.msgData,
token=device['token']))
urlPack.fillMsg(msg, self.pipe)
urlPack.addKey('keyword', msg.msgData)
urlPack.addKey('pcursor', pcursor)
urlPack.addKey('ext',
{"token_client_salt": device['token_client_salt']})
self.add(urlPack)
return True
return False
def onReceiveMsg(self, msg):
"""
:param msg:
"""
log.debug("DouyinAuthorVideoProvider receive {}".format(msg))
if msg.msgType == const_douyin.DATA_TYPE_AUTHOR:
cursor = 0
if msg.getExtra("cursor") is None:
cursor = 0
else:
cursor = msg.getExtra("cursor")
target = self.url.format(user_id=msg.msgData,
cursor=cursor,
iid=const_douyin.get_iid(),
device_id=const_douyin.get_device_id(),
openudid=const_douyin.get_openudid(),
uuid=const_douyin.get_uuid())
urlPack = urlprovider.UrlPack(priority=20, url=target)
urlPack.fillMsg(msg, self.pipe)
self.add(urlPack)
return True
return False
def onReceiveMsg(self, msg): # @UnusedVariable
"""
:param msg:
"""
if msg.msgType == const_huoshan.DATA_TYPE_VIDEO:
if msg.getExtra("offset") is None:
offset = 0
else:
offset = msg.getExtra("offset")
target = self.url.format(vid=msg.msgData,
offset=offset,
iid=const_huoshan.get_iid(),
device_id=const_huoshan.get_device_id(),
openudid=const_huoshan.get_openudid(),
uuid=const_huoshan.get_uuid())
urlPack = urlprovider.UrlPack(priority=20, url=target)
urlPack.fillMsg(msg, self.pipe)
urlPack.addKey('vid', msg.msgData)
urlPack.addKey('offset', offset)
self.add(urlPack)
return True
return False