コード例 #1
0
 def storeurl(self, url, originalurl, step, customized={}):
     urlparam = URLContext()
     urlparam.url = url
     urlparam.originalurl = originalurl
     urlparam.step = step
     urlparam.customized = customized
     URLManager.getinstance().storeurl(url, urlparam)
コード例 #2
0
 def storeurl(self, url, originalurl, step, others={}):
     urlparam = URLContext()
     urlparam.url = url
     urlparam.originalurl = originalurl
     urlparam.step = step
     urlparam.type = URLContext.S1_MAIN_BODY
     urlparam.customized = others
     URLManager.getinstance().storeurl(url, urlparam, constant.REQUEST_TYPE_IMG)
コード例 #3
0
 def storeposturl(self, url, originalurl, step, data, others={}):
     urlcontext = URLContext()
     urlcontext.url = json.dumps({'url': url, 'data': urllib.urlencode(data)})
     urlcontext.originalurl = originalurl
     urlcontext.step = step
     urlcontext.type = URLContext.S1_COMMENTS
     urlcontext.customized = others
     URLManager.getinstance().storeurl(urlcontext.url, urlcontext, constant.REQUEST_TYPE_POST)
コード例 #4
0
 def storeurl(self, url, originalurl, step, others={}):
     urlparam = URLContext()
     urlparam.url = url
     urlparam.originalurl = originalurl
     urlparam.step = step
     urlparam.type = URLContext.S1_COMMENTS
     urlparam.customized = others
     URLManager.getinstance().storeurl(url, urlparam)
コード例 #5
0
 def __storeqeuryurl__(self, url, step, data, customized={}):
     customized[constant.SPIDER_CHANNEL] = constant.SPIDER_CHANNEL_S2
     urlcontext = URLContext()
     urlcontext.url = json.dumps({'url': url, 'data': urllib.urlencode(data)})
     urlcontext.originalurl = self.fakeoriginalurl
     urlcontext.step = step
     urlcontext.type = URLContext.S2_QUERY
     urlcontext.customized = customized
     URLManager.getinstance().storeurl(urlcontext.url, urlcontext, constant.REQUEST_TYPE_POST)
コード例 #6
0
 def __storeqeuryurllist__(self, urllist, step, customized={}):
     for url in urllist:
         customized[constant.SPIDER_CHANNEL] = constant.SPIDER_CHANNEL_S2
         urlcontext = URLContext()
         urlcontext.url = url
         urlcontext.originalurl = self.fakeoriginalurl
         urlcontext.type = URLContext.S2_QUERY
         urlcontext.step = step
         urlcontext.customized = customized
         URLManager.getinstance().storeurl(url, urlcontext)