def process_listpage(self):
item_num = 0
hxs = HtmlXPathSelector(self.response)
products = hxs.select('//div[@id="proShow"]/ul/li')
for prod in products:
prod_name = \
extract_value(prod.select('div[@class="inforBg"]/span/a/@title'))
prod_url = \
extract_value(prod.select('div[@class="inforBg"]/span/a/@href'))
img_url = \
extract_value(prod.select('div[@class="inforBg"]/p/img/@src'))
request = self.make_request_from_response(\
img_url,
cur_idepth=self.basic_link_info.cur_idepth,
prod_url=self.urljoin(prod_url),
prod_name=prod_name,
cat = self.response.meta['cat']
)
self.crawl(request)
item_num += 1
self.next_page(hxs)
return item_num
def next_page(self, hxs):
url = None
last_page = int(extract_value(
hxs.select('//a[@id="pageLast"]/text()')))
current_page = int(
extract_value(
hxs.select(
'//div[@class="snPages"]/a[@class="current"]/text()')))
if current_page == 1:
#http://www.suning.com/emall/pcd_10052_10051_-7_N_20089_20002_.html
#['pcd', '10052', '10051', '-7', 'N', '20089', '20002', '.html']
fs = self.response.url.split('/')[-1].split('_')
url = "http://www.suning.com/emall/secondPointSearchNewCmd?"\
"storeId=%s&catalogId=%s&categoryId=%s&topBrandName="\
"&top=N&top_category=%s&sortIndex=5¤tPage=1&isList=0"\
% (fs[1], fs[2], fs[-2], fs[-3],)
elif current_page < last_page:
url = self.response.url.replace(
'currentPage=%s' % (current_page - 1),
'currentPage=%s' % current_page)
if url:
request = self.make_request_from_response(url=url)
self.crawl(request)
self.log('next page:%s' % request.url)
def process_listpage(self):
item_num = 0
hxs = HtmlXPathSelector(self.response)
goods = hxs.select(
'//div[@class="floorConn"]/div[@class="goodsList"]/ul/li')
gname_dict = {}
gid_list = []
for item in goods:
name = extract_value(
item.select('.//p[@class="productName"]/@title'))
#url = extract_value(item.select('div[@class="txt"]/a/@href'))
gid = extract_value(item.select('.//span[@name="price"]/@id'))
gid_list.append(gid)
gname_dict[gid] = name
item_num += 1
request = self.make_request_from_response(
url="%s%s" % (self.PRICE_REQUST_URL, ",".join(gid_list)),
cur_idepth=self.basic_link_info.cur_idepth,
gname_dict=gname_dict,
)
self.crawl(request)
self.next_page(hxs)
return item_num
def process_listpage(self):
item_num = 0
hxs = HtmlXPathSelector(self.response)
products = hxs.select('//div[@id="proShow"]/ul/li')
for prod in products:
prod_name = \
extract_value(prod.select('div[@class="inforBg"]/span/a/@title'))
prod_url = \
extract_value(prod.select('div[@class="inforBg"]/span/a/@href'))
img_url = \
extract_value(prod.select('div[@class="inforBg"]/p/img/@src'))
request = self.make_request_from_response(\
img_url,
cur_idepth=self.basic_link_info.cur_idepth,
prod_url=self.urljoin(prod_url),
prod_name=prod_name,
cat = self.response.meta['cat']
)
self.crawl(request)
item_num += 1
self.next_page(hxs)
return item_num
def next_page(self, hxs):
poa = hxs.select('//div[@id="bottom_pagenum"]/span/a')
for item in poa:
text = extract_value(item.select('text()'))
if text.find('下一页') != -1:
url = extract_value(item.select('@href'))
request = self.make_request_from_response(
url="%s%s" % (self.BASE_URL, url), )
self.crawl(request)
self.log('next page:%s' % request.url)
break
def next_page(self, hxs):
poa = hxs.select('//div[@id="bottom_pagenum"]/span/a')
for item in poa:
text = extract_value(item.select('text()'))
if text.find('下一页') != -1:
url = extract_value(item.select('@href'))
request = self.make_request_from_response(
url="%s%s" % (self.BASE_URL, url),
)
self.crawl(request)
self.log('next page:%s' % request.url)
break
def process(self):
item_num = 0
hxs = HtmlXPathSelector(self.response)
prolist = hxs.select('//div[@id="prodlist"]/li')
for item in prolist:
url = "%s%s" % (self.BASE_URL ,
extract_value(item.select('a/@href')))
sprice = extract_value(
item.select('p[@class="pimg"]/span[@class="pinfo"]/i[@class="ltprice"]/text()')
)
price = canonicalize_price(sprice)
name = extract_value(
item.select('p[@class="pimg"]/span[@class="pname"]/a/text()')
)
self.save(url, name, (), price)
item_num += 1
self.next_page(hxs)
return item_num
def process_entrypage(self):
item_num = 0
hxs = HtmlXPathSelector(self.response)
floors = hxs.select('//div[@class="sFloors l"]/div[@class="sFloor clearfix"]')
#we don't need floor[0], which is book
for i in range(1, len(floors)):
cat1 = extract_value(floors[i].select('h3/a/text()'))
subcats = floors[i].select('ul/li/dl/dt')
for subcat in subcats:
cat2 = extract_value(subcat.select('a/text()'))
url = extract_value(subcat.select('a/@href'))
request = self.make_request_from_response( \
self.urljoin(url),
cur_idepth=self.basic_link_info.cur_idepth,
cat=[cat1, cat2])
self.crawl(request)
item_num += 1
return item_num
def process_entrypage(self):
item_num = 0
hxs = HtmlXPathSelector(self.response)
floors = hxs.select(
'//div[@class="sFloors l"]/div[@class="sFloor clearfix"]')
#we don't need floor[0], which is book
for i in range(1, len(floors)):
cat1 = extract_value(floors[i].select('h3/a/text()'))
subcats = floors[i].select('ul/li/dl/dt')
for subcat in subcats:
cat2 = extract_value(subcat.select('a/text()'))
url = extract_value(subcat.select('a/@href'))
request = self.make_request_from_response( \
self.urljoin(url),
cur_idepth=self.basic_link_info.cur_idepth,
cat=[cat1, cat2])
self.crawl(request)
item_num += 1
return item_num
def next_page(self, hxs):
url = None
last_page = int(extract_value(hxs.select('//a[@id="pageLast"]/text()')))
current_page = int(extract_value(
hxs.select('//div[@class="snPages"]/a[@class="current"]/text()')))
if current_page == 1:
#http://www.suning.com/emall/pcd_10052_10051_-7_N_20089_20002_.html
#['pcd', '10052', '10051', '-7', 'N', '20089', '20002', '.html']
fs = self.response.url.split('/')[-1].split('_')
url = "http://www.suning.com/emall/secondPointSearchNewCmd?"\
"storeId=%s&catalogId=%s&categoryId=%s&topBrandName="\
"&top=N&top_category=%s&sortIndex=5¤tPage=1&isList=0"\
% (fs[1], fs[2], fs[-2], fs[-3],)
elif current_page < last_page:
url = self.response.url.replace(
'currentPage=%s' % (current_page-1), 'currentPage=%s' % current_page)
if url:
request = self.make_request_from_response(url=url)
self.crawl(request)
self.log('next page:%s' % request.url)
def process_entrypage(self):
item_num = 0
hxs = HtmlXPathSelector(self.response)
links = hxs.select('//div[contains(@id, "JDS_")]')
for index, link in enumerate(links):
cat1 = extract_value(link.select('div[@class="mt"]/h2/a/text()'))
dls = link.select('div[@class="mc"]/dl[@class="fore"]')
for dl in dls:
cat2 = extract_value(dl.select('dt/a/text()'))
dds = dl.select('dd/em')
for dd in dds:
cat3 = extract_value(dd.select('a/text()'))
catlist = (cat1, cat2, cat3)
url = dd.select('a/@href').extract()[0]
request = self.make_request_from_response( \
"http://www.360buy.com/%s"%url,
cur_idepth=self.basic_link_info.cur_idepth,
cat=catlist)
self.crawl(request)
item_num += 1
return item_num
def process_entrypage(self):
item_num = 0
hxs = HtmlXPathSelector(self.response)
links = hxs.select('//div[contains(@id, "JDS_")]')
for index, link in enumerate(links):
cat1 = extract_value(link.select('div[@class="mt"]/h2/a/text()'))
dls = link.select('div[@class="mc"]/dl[@class="fore"]')
for dl in dls:
cat2 = extract_value(dl.select('dt/a/text()'))
dds = dl.select('dd/em')
for dd in dds:
cat3 = extract_value(dd.select('a/text()'))
catlist = (cat1, cat2, cat3)
url = dd.select('a/@href').extract()[0]
request = self.make_request_from_response( \
"http://www.360buy.com/%s"%url,
cur_idepth=self.basic_link_info.cur_idepth,
cat=catlist)
self.crawl(request)
item_num += 1
return item_num
def process_listpage(self):
item_num = 0
hxs = HtmlXPathSelector(self.response)
goods = hxs.select('//div[@class="floorConn"]/div[@class="goodsList"]/ul/li')
gname_dict = {}
gid_list = []
for item in goods:
name = extract_value(item.select('.//p[@class="productName"]/@title'))
#url = extract_value(item.select('div[@class="txt"]/a/@href'))
gid = extract_value(item.select('.//span[@name="price"]/@id'))
gid_list.append(gid)
gname_dict[gid] = name
item_num += 1
request = self.make_request_from_response(
url= "%s%s" % (self.PRICE_REQUST_URL, ",".join(gid_list)),
cur_idepth=self.basic_link_info.cur_idepth,
gname_dict=gname_dict,
)
self.crawl(request)
self.next_page(hxs)
return item_num
def process_listpage(self):
item_num = 0
hxs = HtmlXPathSelector(self.response)
skus = hxs.select('//li[@sku]')
for sku in skus:
skuvalue = ''.join(sku.select('@sku').extract())
imgurl = sku.select('div[@class="p-price"]/img/@src').extract()[0]
url = sku.select('div[@class="p-name"]/a/@href').extract()[0]
name = extract_value(sku.select('div[@class="p-name"]/a/text()'))
request = self.make_request_from_response(\
imgurl,
cur_idepth=self.basic_link_info.cur_idepth,
gurl=url, name=name, sku=skuvalue
)
self.crawl(request)
item_num += 1
self.next_page(hxs)
return item_num
def process_listpage(self):
item_num = 0
hxs = HtmlXPathSelector(self.response)
skus = hxs.select('//li[@sku]')
for sku in skus:
skuvalue = ''.join(sku.select('@sku').extract())
imgurl = sku.select('div[@class="p-price"]/img/@src').extract()[0]
url = sku.select('div[@class="p-name"]/a/@href').extract()[0]
name = extract_value(sku.select('div[@class="p-name"]/a/text()'))
request = self.make_request_from_response(\
imgurl,
cur_idepth=self.basic_link_info.cur_idepth,
gurl=url, name=name, sku=skuvalue
)
self.crawl(request)
item_num += 1
self.next_page(hxs)
return item_num