def __init__(self):
self.salver = SALVER
self.r = redis_db.RedisQueue('new')
self.html = HtmlDownloader()
self.parser = HtmlParser()
if self.salver:
start_urls = [
'https://www.guazi.com/bj/buy/o{}/#bread'.format(i)
for i in range(1, 230)
]
self.r.put(start_urls)
def __init__(self):
self.salver = SALVER
self.r = redis_db.RedisQueue('new')
self.html = HtmlDownloader()
self.parser = HtmlParser()
if self.salver:
start_urls = [
'https://weibo.com/p/aj/v6/mblog/mbloglist?ajwvr=6&domain'
'=100505&from=page_100505_profile&wvr=6&mod=data&is_hot=1&pagebar=1&pl_name=Pl_Of'
'ficial_MyProfileFeed__20&id=1005051713926427&script_uri=/p/1005051713926427/home&feed_typ'
'e=0&page={}&pre_page=1&domain_op=100505&__rnd=1526295208644'.
format(i) for i in range(200)
]
self.r.put(start_urls)
def __init__(self):
self.salver = SALVER
self.r = redis_db.RedisQueue('new')
self.html = HtmlDownloader()
self.parser = HtmlParser()
if self.salver:
start_urls = 'https://s.taobao.com/search?q={}&app=detailproduct&through=1'.format(KEYS)
self.r.put(start_urls)
for i in range(1, PAGE_NUM):
url = 'https://s.taobao.com/search?data-key=s&data-value=88&ajax=true&_ksTS=1526525628728_733&callback=jsonp734&q={}&imgfile=&ie=utf8&app=detailproduct&through=1&bcoffset=4&p4ppushleft=6%2C48&s={}'.format(
KEYS, i * 44)
self.r.put(url)
def __init__(self):
self.r = redis_db.RedisQueue('new')