def parse_list(self, response):
for item_sel in response.xpath('//div[@class="search-result-content"]/ul[@id="search-result-items"]/li[@class="grid-tile"]/div[@class="product-tile "]'):
values_from_list = self.extract_values_from_list(item_sel, response)
selector_list = [".BVRRWidget"]
script = make_lua_script(selector_list)
request = Request(values_from_list['url'], callback=self.parse_item, meta={
'splash': {
'endpoint': 'execute',
'args': {'lua_source': script}
}
})
request.meta['values_from_list'] = values_from_list
yield request
def parse(self, response):
selector_list = ["li[rel=shop-all-men]"]
script = make_lua_script(selector_list)
request = Request(response.url,
callback=self.parse_redirect_url,
meta={
'splash': {
'endpoint': 'execute',
'args': {
'lua_source': script
}
}
})
yield request
def parse_ajax(self, response):
data = json.loads(response.body)
total_page = int(data['productCategoryFacetedSearch']['productCategory']['productCategoryPaginator']['pageNumberTotal'])
page = 0
selector_list = [".productPlaceholder"]
script = make_lua_script(selector_list)
while page < total_page:
list_url = self.get_url_for_list(response.meta.get('url'), page)
yield Request(list_url, callback=self.parse_list, meta={
'splash': {
'endpoint': 'execute',
'args': {'lua_source': script}
}
})
page += 1
def parse_list(self, response):
data = json.loads(response.body)
for item in data['items']:
if self.check_scrapable(item):
url = get_absolute_url(response, item['link'])
values_from_list = self.extract_values_from_list(item, response)
selector_list = [".BVRRWidget", "iframe#contentFrame"]
script = make_lua_script(selector_list, '&&')
request = Request(url, callback=self.parse_item, meta={
'splash': {
'endpoint': 'execute',
'args': {'lua_source': script}
}
})
request.meta['values_from_list'] = values_from_list
yield request
def parse_data(self, response):
values_from_list = response.meta.get('values_from_list', {})
pid = re.search('\??pid=(.*?)(&|$)', values_from_list['url']).group(1)
pattern = "StyleColor\(\"%s\".*?\.styleColorImagesMap = (.*?);" % pid
images_data = json.loads(re.search(pattern, response.body).group(1).replace("'", "\""))
selector_list = [".bv-content-list"]
script = make_lua_script(selector_list)
request = Request(values_from_list['url'], callback=self.parse_item, meta={
'splash': {
'endpoint': 'execute',
'args': {'lua_source': script}
}
})
request.meta['values_from_list'] = values_from_list
request.meta['images_data'] = images_data
yield request
def parse_list(self, response):
for item_sel in response.xpath(
'//div[@id="mainContent"]//ol[@id="products"]/li[@class="productListing"]'
):
values_from_list = self.extract_values_from_list(
item_sel, response)
selector_list = ["div#TurnToReviewsContent[style]"]
script = make_lua_script(selector_list)
request = Request(values_from_list['url'],
callback=self.parse_item,
meta={
'splash': {
'endpoint': 'execute',
'args': {
'lua_source': script
}
}
})
request.meta['values_from_list'] = values_from_list
yield request
def parse_list(self, response):
for item_sel in response.xpath(
'//ol[contains(@class,"products")]/li[contains(@id,"product")]'
):
values_from_list = self.extract_values_from_list(
item_sel, response)
if values_from_list:
selector_list = [".detail", ".current"]
script = make_lua_script(selector_list, "&&")
request = Request(values_from_list['url'],
callback=self.parse_item,
meta={
'splash': {
'endpoint': 'execute',
'args': {
'lua_source': script
}
}
})
request.meta['values_from_list'] = values_from_list
yield request