def storeurl(self, url, originalurl, step, customized={}):
urlparam = URLContext()
urlparam.url = url
urlparam.originalurl = originalurl
urlparam.step = step
urlparam.customized = customized
URLManager.getinstance().storeurl(url, urlparam)
def storeurl(self, url, originalurl, step, others={}):
urlparam = URLContext()
urlparam.url = url
urlparam.originalurl = originalurl
urlparam.step = step
urlparam.type = URLContext.S1_MAIN_BODY
urlparam.customized = others
URLManager.getinstance().storeurl(url, urlparam, constant.REQUEST_TYPE_IMG)
def storeposturl(self, url, originalurl, step, data, others={}):
urlcontext = URLContext()
urlcontext.url = json.dumps({'url': url, 'data': urllib.urlencode(data)})
urlcontext.originalurl = originalurl
urlcontext.step = step
urlcontext.type = URLContext.S1_COMMENTS
urlcontext.customized = others
URLManager.getinstance().storeurl(urlcontext.url, urlcontext, constant.REQUEST_TYPE_POST)
def storeurl(self, url, originalurl, step, others={}):
urlparam = URLContext()
urlparam.url = url
urlparam.originalurl = originalurl
urlparam.step = step
urlparam.type = URLContext.S1_COMMENTS
urlparam.customized = others
URLManager.getinstance().storeurl(url, urlparam)
def __storeqeuryurl__(self, url, step, data, customized={}):
customized[constant.SPIDER_CHANNEL] = constant.SPIDER_CHANNEL_S2
urlcontext = URLContext()
urlcontext.url = json.dumps({'url': url, 'data': urllib.urlencode(data)})
urlcontext.originalurl = self.fakeoriginalurl
urlcontext.step = step
urlcontext.type = URLContext.S2_QUERY
urlcontext.customized = customized
URLManager.getinstance().storeurl(urlcontext.url, urlcontext, constant.REQUEST_TYPE_POST)
def __storeqeuryurllist__(self, urllist, step, customized={}):
for url in urllist:
customized[constant.SPIDER_CHANNEL] = constant.SPIDER_CHANNEL_S2
urlcontext = URLContext()
urlcontext.url = url
urlcontext.originalurl = self.fakeoriginalurl
urlcontext.type = URLContext.S2_QUERY
urlcontext.step = step
urlcontext.customized = customized
URLManager.getinstance().storeurl(url, urlcontext)