def parse(self, response):
hxs = HtmlXPathSelector(response)
# currently selecting bottom level categories, and their parents and parents of parents ("grandparents") in their fields
links = hxs.select("//div[@id='sitemap']//li[@class='bullet3']//a")
parent_links = hxs.select("//div[@id='sitemap']//li[@class='bullet2']//a")
grandparent_links = hxs.select("//div[@id='sitemap']//li[@class='bullet1']//a")
items = []
#TODO: mark special categories (if appropriate for any)
for link in links:
# extract immediate parent of this link (first preceding sibling (of the parent node) with class='bullet2')
parent = link.select("parent::node()/preceding-sibling::*[@class='bullet2'][1]/a")
# extract grandparent of this link (first preceding sibling of the parent's parent node witch class='bullet1')
grandparent = parent.select("parent::node()/preceding-sibling::*[@class='bullet1'][1]/a")
item = CategoryItem()
item['text'] = link.select('text()').extract()[0]
item['url'] = link.select('@href').extract()[0]
item['parent_text'] = parent.select('text()').extract()[0]
item['parent_url'] = parent.select('@href').extract()[0]
item['grandparent_text'] = grandparent.select('text()').extract()[0]
item['grandparent_url'] = grandparent.select('@href').extract()[0]
# this will be considered lower than the main level, because these categories are very detailed
item['level'] = -1
items.append(item)
for link in parent_links:
# extract immediate parent of this link (first preceding sibling (of the parent node) with class='bullet2')
parent = link.select("parent::node()/preceding-sibling::*[@class='bullet1'][1]/a")
item = CategoryItem()
item['text'] = link.select('text()').extract()[0]
item['url'] = link.select('@href').extract()[0]
item['parent_text'] = parent.select('text()').extract()[0]
item['parent_url'] = parent.select('@href').extract()[0]
# this will be considered the main level of the nested list (it's comparable with the main level of the other sitemaps)
item['level'] = 0
items.append(item)
for link in grandparent_links:
item = CategoryItem()
item['text'] = link.select('text()').extract()[0]
item['url'] = link.select('@href').extract()[0]
item['level'] = 1
items.append(item)
return items
def parseSubcats(self, response):
hxs = HtmlXPathSelector(response)
parent = response.meta['parent']
# extract subcategories
subcats_links = hxs.select(
"//div[@class='sideNav']/div[@class='innerWrap'][1]//ul/li/a")
for subcat_link in subcats_links:
item = CategoryItem()
item['url'] = Utils.add_domain(
subcat_link.select("@href").extract()[0],
"http://www.tigerdirect.com")
item['text'] = subcat_link.select("text()").extract()[0]
item['parent_text'] = parent['text']
item['parent_url'] = parent['url']
item['level'] = parent['level'] - 1
item['department_text'] = response.meta['department_text']
item['department_id'] = response.meta['department_id']
item['department_text'] = response.meta['department_text']
#print 'passing to parse category ', item
# there are some loops in their categories tree, so we need to check this to avoid infinite loops in crawling
if item['url'] not in self.parsed_urls:
yield Request(url = item['url'], callback = self.parseCategory,\
meta = {'item' : item,\
'department_text' : response.meta['department_text'], 'department_url' : response.meta['department_url'],\
'department_id' : response.meta['department_id']})
def _parse_category(self, response):
category = response.meta['category']
parent = response.meta.get('parent', {})
category['catid'] = self._get_catid()
category['url'] = response.url
category['parent_text'] = parent.get('text')
category['parent_url'] = parent.get('url')
category['parent_catid'] = parent.get('catid')
category['grandparent_text'] = parent.get('parent_text')
category['grandparent_url'] = parent.get('parent_url')
category['level'] = parent.get('level', 0) + 1
category['department_text'] = response.meta['department']['text']
category['department_url'] = response.meta['department']['url']
category['department_id'] = response.meta['department']['catid']
#category['description_text'] = self._description_text.first(response)
description_text = first(response.xpath(self._xpath_description_text).extract())
if description_text:
category['description_wc'] = len(Utils.normalize_text(description_text))
keywords = first(response.xpath(self._xpath_keywords).extract())
if description_text:
category['description_text'] = description_text
if description_text and keywords:
(category['keyword_count'], category['keyword_density']) = Utils.phrases_freq(keywords, description_text)
if category.get('nr_products') is None:
nr_products = re_find('\d+', first(response.css(self._css_product_numbers_text).extract()))
category['nr_products'] = int(nr_products) if nr_products is not None else None
subcategory_links = LinkExtractor(restrict_xpaths=self._xpath_category_links)
for link in subcategory_links.extract_links(response):
text, nr_products = re.search('(.+?) \((\d+)\) *', link.text).groups()
nr_products = int(nr_products)
child = CategoryItem(text=text, nr_products=nr_products)
meta = {'category': child, 'department': response.meta['department'], 'parent': category}
yield Request(link.url, callback=self._parse_category, meta=meta)
yield category
def parse(self, response):
hxs = HtmlXPathSelector(response)
#TODO: add departments with no subcategories!!
# get urls of pages for each category
urls = hxs.select(
"//div[@class='siteMapSubCell']//ul/li/a/@href").extract()
# add departments to items
departments = hxs.select("//div[@class='siteMapSubCell']//h4/a")
items = []
for department in departments:
item = CategoryItem()
item['text'] = department.select('text()').extract()[0]
item['url'] = department.select('@href').extract()[0]
item['level'] = 1
items.append(item)
# parse each page in urls list with parsePage
# build urls by adding the prefix of the main page url
first = True
root_url = "http://www.sears.com/shc/s"
for url in urls:
request = Request(root_url + "/" + url, callback=self.parsePage)
# send these only once (the first time)
if first:
request.meta['items'] = items
first = False
else:
request.meta['items'] = []
yield request
def parse(self, response):
hxs = HtmlXPathSelector(response)
links = hxs.select("//div[@id='sitemap_header']/a")
root_url = "http://www1.macys.com"
department_id = 0
for link in links:
item = CategoryItem()
text = link.select('text()').extract()[0]
item['text'] = text
# remove unnecessary suffix from URL
url = link.select('@href').extract()[0]
m = re.match("(.*\?id=[0-9]+)&?.*", url)
if m:
item['url'] = m.group(1)
else:
item['url'] = url
item['level'] = 1
# only yield this item after parsing its page and extracting additional info
#yield item
department_id += 1
# create request to extract subcategories for this category
yield Request(item['url'], callback = self.parseCategory, meta = {'parent' : item, 'level' : 1, \
'department_text' : item['text'], 'department_url' : item['url'], 'department_id' : department_id})
def parseList(self, response):
hxs = HtmlXPathSelector(response)
items = []
# add all department names
departments = hxs.select("//div[@id='showallprods']/ul/li/a")
root_url = "http://www.staples.com"
department_id = 0
for department in departments:
item = CategoryItem()
item['text'] = department.select("text()").extract()[0]
item['url'] = root_url + department.select("@href").extract()[0]
item['level'] = 1
#yield item
# # parse each department page for its categories, pass the department item too so that it's added to the list in parseDept
# yield Request(item['url'], callback = self.parseDept, meta = {"department": item})
department_id += 1
zipcode = "12345"
request = Request(item['url'], callback = self.parseDept, cookies = {"zipcode" : zipcode}, \
headers = {"Cookie" : "zipcode=" + zipcode}, meta = {"dont_redirect" : True, "dont_merge_cookies" : True, \
"parent": item, "level": 1, \
"department_text" : item["text"], "department_url" : item["url"], "department_id" : department_id})
yield request
def parse(self, response):
hxs = HtmlXPathSelector(response)
links = hxs.select(
"//div[@class='sr_siteMap_container']/div[position()>2 and position()<5]//a"
)
root_url = "http://www1.bloomingdales.com"
#TODO: add registry as special category?
department_id = 0
for link in links:
item = CategoryItem()
item['text'] = link.select('text()').extract()[0]
url = link.select('@href').extract()[0]
# if it's a relative URL complete it with the domain
if not url.startswith("http"):
url = root_url + url
item['url'] = url
item['level'] = 1
department_id += 1
item['department_text'] = item['text']
item['department_url'] = item['url']
item['department_id'] = department_id
#yield item
# create request to extract subcategories for this category
yield Request(item['url'], callback = self.parseCategory, meta = {'parent' : item, \
"department_text" : item['text'], "department_url" : item['url'], "department_id" : department_id, \
"dont_merge_cookies" : True}, \
cookies = {"shippingCountry" : "US"}, headers = {"Cookie" : "shippingCountry=" + "US"})
def parse(self, response):
hxs = HtmlXPathSelector(response)
category_links = hxs.select("//div[@class='itmNav']//a")
# unique ids for departments
department_id = 0
for category_link in category_links:
item = CategoryItem()
category_name = category_link.select("text()").extract()
if category_name:
item['text'] = category_name[0]
else:
sys.stderr.write("Error: no name for category in element " + category_link.extract())
continue
item['url'] = self.clean_url(category_link.select("@href").extract()[0])
# mark as department
item['level'] = 1
department_id += 1
# mark it as its own department, will be passed on to its subcategories
item['department_text'] = item['text']
item['department_url'] = item['url']
item['department_id'] = department_id
#items.append(item)
yield Request(url = item['url'], callback = self.parseCategory, meta = {"item" : item, \
'department_text' : item['department_text'], 'department_url' : item['department_url'], 'department_id' : item['department_id']})
def parseCategory(self, response):
hxs = HtmlXPathSelector(response)
item = response.meta['item']
# extract number of products if available
#TODO check
count_holder = hxs.select("//div[@class='recordCount']/span[@id='RecordCount_1']/text()")
if count_holder:
item['nr_products'] = int(count_holder.extract()[0])
#TODO
# try to change URL "Category" to "SubCategory", see if you find the product count there
# extract description if available
description_holders = hxs.select("//div[@id='bcaShopWindowSEO']")
# if the list is not empty and contains at least one non-whitespace item
if description_holders:
description_texts = description_holders.select(".//text()[not(ancestor::h2)]").extract()
# replace all whitespace with one space, strip, and remove empty texts; then join them
item['description_text'] = " ".join([re.sub("\s+"," ", description_text.strip()) for description_text in description_texts if description_text.strip()])
tokenized = Utils.normalize_text(item['description_text'])
item['description_wc'] = len(tokenized)
description_title = description_holders.select(".//h2/text()").extract()
if description_title:
item['description_title'] = description_title[0].strip()
(item['keyword_count'], item['keyword_density']) = Utils.phrases_freq(item['description_title'], item['description_text'])
else:
item['description_wc'] = 0
yield item
parent = item
#TODO
# extract and parse subcategories
subcats = hxs.select("//dl[@class='categoryList primaryNav']/dd/a")
for subcat in subcats:
item = CategoryItem()
item['text'] = subcat.select("text()").extract()[0].strip()
#TODO: check out some huge URLs
item['url'] = self.clean_url(subcat.select("@href").extract()[0])
item['parent_text'] = parent['text']
item['parent_url'] = parent['url']
item['level'] = parent['level'] - 1
item['department_text'] = response.meta['department_text']
item['department_url'] = response.meta['department_url']
item['department_id'] = response.meta['department_id']
yield Request(url = item['url'], callback = self.parseCategory, meta = {"item" : item, \
"department_text" : response.meta['department_text'], "department_url" : response.meta['department_url'], "department_id" : response.meta['department_id']})
def parseSubcategory(self, response):
hxs = HtmlXPathSelector(response)
subcategory = response.meta['item']
# yield this subcategory
yield subcategory
# if subcategory was special, we'll mark all subsubcategories as special
if 'special' in subcategory:
special = True
else:
special = False
# get its subcategories
subsubcategories = hxs.select(
"//div[@class='product-category-expanded']//h3[@class='title']")
for subsubcategory in subsubcategories:
item = CategoryItem()
item['text'] = subsubcategory.select("a/text()").extract()[0]
item['url'] = Utils.add_domain(
subsubcategory.select("a/@href").extract()[0], self.base_url)
if special:
item['special'] = 1
item['parent_text'] = subcategory['text']
item['parent_url'] = subcategory['url']
item['department_text'] = subcategory['department_text']
item['department_url'] = subcategory['department_url']
item['department_id'] = subcategory['department_id']
item['level'] = subcategory['level'] - 1
description_text_holder = subsubcategory.select(
"following-sibling::p[@class='description'][1]/text()"
).extract()
if description_text_holder:
item['description_text'] = description_text_holder[0]
item['description_title'] = item['text']
description_tokenized = Utils.normalize_text(
item['description_text'])
item['description_wc'] = len(description_tokenized)
(item['keyword_count'],
item['keyword_density']) = Utils.phrases_freq(
item['description_title'], item['description_text'])
else:
item['description_wc'] = 0
# parse subcategory page to get product count, or further subsubcategory
yield Request(item['url'],
callback=self.parseSubcategoryPage,
meta={'item': item})
def _parse_category(self, response):
category = response.meta['category']
self._populate_category(response)
classifications = self._scrape_classifications(response)
categories = classifications.pop("Shop By Category", [])
for url, text, nr_products in categories:
new_category = CategoryItem(text=text, nr_products=nr_products)
yield Request(url, self._parse_category, meta={"category": new_category, "parent": category})
if category.get('nr_products') is None:
category['nr_products'] = sum((item[2] for item in categories))
category['classification'] = {key: [{'name': itm[1], 'nr_products': itm[2]} for itm in value]
for key,value in classifications.iteritems()}
yield category
def parse(self, response):
hxs = HtmlXPathSelector(response)
# extract all bottom level categories
links = hxs.select("//div[@class='links']//a")
# extract parent categories
parent_links = hxs.select("//div[@class='header_no_icon']")
items = []
for link in links:
# extract parent category
parent = link.select(
"parent::node()/parent::node()/parent::node()/div/div[position()=1]/a"
)
item = CategoryItem()
item['text'] = link.select('text()').extract()[0]
item['url'] = link.select('@href').extract()[0]
parent_text = parent.select('h2/text()').extract()
parent_url = parent.select('@href').extract()
if parent_text:
item['parent_text'] = parent_text[0]
if parent_url:
item['parent_url'] = parent_url[0]
item['level'] = 0
items.append(item)
for parent in parent_links:
item = CategoryItem()
item['text'] = parent.select('a/h2/text()').extract()[0]
item['url'] = parent.select('a/@href').extract()[0]
item['level'] = 1
items.append(item)
return items
def parse(self, response):
hxs = HtmlXPathSelector(response)
links = hxs.select("//table//tr[1]/td//a[ancestor::h4]")
department_id = 0
for link in links:
item = CategoryItem()
item['text'] = link.select('text()').extract()[0]
item['url'] = link.select('@href').extract()[0]
item['level'] = 1
department_id += 1
item['department_text'] = item['text']
item['department_url'] = item['url']
item['department_id'] = department_id
yield Request(url = item['url'], callback = self.parseCategory, meta = {'item' : item,\
'department_url' : item['department_url'], 'department_text' : item['department_text'], 'department_id' : department_id})
def parseCategory(self, response):
hxs = HtmlXPathSelector(response)
parent = response.meta['parent']
# extract product count if any
product_count_holder = hxs.select(
"//span[@class='productCount'][1]/text()").extract()
if product_count_holder:
parent['nr_products'] = int(product_count_holder[0])
# extract description if any
# just assume no description (haven't found any page with descriptions for bloomingdales)
parent['description_wc'] = 0
# yield parent item (if it hasn't been output before)
if 'parent_url' not in parent or (parent['url'], parent['parent_url'],
parent['level']) not in self.crawled:
if 'parent_url' in parent:
self.crawled.append(
(parent['url'], parent['parent_url'], parent['level']))
yield parent
# extract subcategories
subcats = hxs.select("//div[@class='gn_left_nav2_standard']//a")
for subcat in subcats:
item = CategoryItem()
item['text'] = subcat.select('text()').extract()[0]
item['url'] = subcat.select('@href').extract()[0]
item['level'] = parent['level'] - 1
item['parent_text'] = response.meta['parent']['text']
item['parent_url'] = response.url
item['department_text'] = response.meta['department_text']
item['department_url'] = response.meta['department_url']
item['department_id'] = response.meta['department_id']
# create request to extract subcategories for this category
yield Request(item['url'], callback = self.parseCategory, meta = {'parent' : item, \
"department_text" : item['department_text'], "department_url" : item['department_url'], "department_id" : item['department_id'], \
"dont_merge_cookies" : True}, \
cookies = {"shippingCountry" : "US"}, headers = {"Cookie" : "shippingCountry=" + "US"})
def parse(self, response):
hxs = HtmlXPathSelector(response)
#links = hxs.select("//div[@class='MidContainer']/div[3]//a[@class='NavM']")
#parent_links = hxs.select("//div[@class='MidContainer']/div[3]//a[@class='NavXLBold']")
#parent_links = hxs.select("//div[@class='MidContainer']/div/div/div[not(@class)]//a[@class='NavXLBold']")
parent_links = hxs.select(
"//div[@class='linkGroup']/div[not (@class)]/a[@class='NavXLBold'][@href]"
)
# #TODO: check this
# item['nr_products'] = -1
# yield item
#yield Request(item['url'], callback = self.parseCategory, meta = {'item' : item})
department_id = 0
for link in parent_links:
item = CategoryItem()
#TO remove:
# # link to artificial parent category
# item['parent_catid'] = 0
item['text'] = link.select('text()').extract()[0]
item['url'] = link.select('@href').extract()[0]
# add domain if relative URL
item['url'] = Utils.add_domain(item['url'], self.root_url)
item['level'] = 1
department_id += 1
# send category page to parseCategory function to extract description and number of products and add them to the item
yield Request(item['url'], callback = self.parseCategory, meta = {'item' : item, \
'department_text' : item['text'], 'department_url' : item['url'], 'department_id' : department_id})
def parse(self, response):
hxs = HtmlXPathSelector(response)
links_level1 = hxs.select("//div[@id='siteDirectory']//table//a")
titles_level1 = hxs.select("//div//table//h2")
# add level 1 categories to items
# first one is a special category ("Unlimited Instant Videos"), add it separately
special_item = CategoryItem()
special_item['text'] = titles_level1[0].select('text()').extract()[0]
special_item['level'] = 2
special_item['special'] = 1
special_item['department_text'] = special_item['text']
special_item['department_id'] = self.department_count
self.departments_ids[special_item['text']] = special_item['department_id']
self.department_count += 1
#items.append(special_item)
yield special_item
# the rest of the titles are not special
for title in titles_level1[1:]:
item = CategoryItem()
item['text'] = title.select('text()').extract()[0]
item['level'] = 2
item['department_text'] = item['text']
item['department_id'] = self.department_count
self.departments_ids[item['text']] = item['department_id']
self.department_count += 1
# if item is found among extra_toplevel_categories_urls, add info from that url
extra_category = self.find_matching_key(item['text'], self.extra_toplevel_categories_urls)
if extra_category:
item['url'] = self.extra_toplevel_categories_urls[extra_category]
item['department_url'] = item['url']
self.department_urls[item['text']] = item['url']
# collect number of products from this alternate URL
yield Request(item['url'], callback = self.extract_nrprods_and_subcats, meta = {'item' : item})
else:
yield item
# add level 1 categories to items
for link in links_level1:
item = CategoryItem()
item['text'] = link.select('text()').extract()[0]
root_url = "http://www.amazon.com"
item['url'] = root_url + link.select('@href').extract()[0]
item['level'] = 1
parent = link.select("parent::node()/parent::node()/preceding-sibling::node()")
parent_text = parent.select('text()').extract()
# category should have a parent (its department) and that parent should have been extracted earlier (above) and put in the ids dictionary, necessary for getting the department id
assert parent_text
assert parent_text[0] in self.departments_ids
if parent_text:
item['parent_text'] = parent_text[0]
item['department_text'] = item['parent_text']
item['department_id'] = self.departments_ids[item['department_text']]
# get department url from department_urls, will be availble only for extra_categories
if item['department_text'] in self.department_urls:
assert self.find_matching_key(item['department_text'], self.extra_toplevel_categories_urls)
item['department_url'] = self.department_urls[item['department_text']]
# if its parent is the special category, mark this one as special too
if (item['parent_text'] == special_item['text']):
item['special'] = 1
special = True
else:
special = False
# department_id = self.department_count
# self.department_count += 1
# item['department_text'] = item['text']
# item['department_url'] = item['url']
# item['department_id'] = department_id
yield Request(item['url'], callback = self.parseCategory, meta = {'item' : item})
class WalmartCaSpider(BaseSpider):
name = "walmartca"
allowed_domains = ["walmart.ca"]
start_urls = [
"http://www.walmart.ca/en",
]
def __init__(self, outfile=None):
self.root_url = "http://www.walmart.ca"
self.outfile = outfile
# set flag that indicates that for this spider, nr of products for each catgory should be computed
self.compute_nrproducts = True
# level that is considered to contain departments
self.DEPARTMENT_LEVEL = 1
# keep crawled items represented by (url, parent_url, department_url) pairs
# to eliminate duplicates
# (adding department_url makes sure that if one entire department is found as a subcategory of another for ex, both (and their complete category trees) will be crawled)
self.crawled = []
# last used category id, used for autoincrementing ids idenrifying categories
self.id_count = 0
# hardcoded values for special category's item count. Currently used for 'Value of the day' that typically has fixed number of products, and nowhere to extract it from page
self.special_itemcount = {'value of the day': 2}
def parse(self, response):
hxs = HtmlXPathSelector(response)
#links = hxs.select("//div[@class='MidContainer']/div[3]//a[@class='NavM']")
#parent_links = hxs.select("//div[@class='MidContainer']/div[3]//a[@class='NavXLBold']")
#parent_links = hxs.select("//div[@class='MidContainer']/div/div/div[not(@class)]//a[@class='NavXLBold']")
parent_links = hxs.select(
"//div[@class='linkGroup']/div[not (@class)]/a[@class='NavXLBold'][@href]"
)
# #TODO: check this
# item['nr_products'] = -1
# yield item
#yield Request(item['url'], callback = self.parseCategory, meta = {'item' : item})
department_id = 0
for link in parent_links:
item = CategoryItem()
#TO remove:
# # link to artificial parent category
# item['parent_catid'] = 0
item['text'] = link.select('text()').extract()[0]
item['url'] = link.select('@href').extract()[0]
# add domain if relative URL
item['url'] = Utils.add_domain(item['url'], self.root_url)
item['level'] = 1
department_id += 1
# send category page to parseCategory function to extract description and number of products and add them to the item
yield Request(item['url'], callback = self.parseCategory, meta = {'item' : item, \
'department_text' : item['text'], 'department_url' : item['url'], 'department_id' : department_id})
# parse category page and extract description and number of products
def parseCategory(self, response):
# URLs like health.walmart.com don't have body_as_unicode and generate an exception
try:
hxs = HtmlXPathSelector(response)
except AttributeError, e:
self.log("Could not get response from " + response.url +
"; original exception: " + str(e) + "\n",
level=log.WARNING)
return
item = response.meta['item']
# Add department text, url and id to item
item['department_text'] = response.meta['department_text']
item['department_url'] = response.meta['department_url']
item['department_id'] = response.meta['department_id']
# assign unique id
item['catid'] = self.id_count
self.id_count += 1
# Extract subcategories breakdown if any ("classification" field)
classification_criteria = hxs.select(
"//form[@id='refine']//h6[@class='AdvSearchSubhead']")
classification_dictionary = {}
for criterion in classification_criteria:
criterion_name = criterion.select(
".//text()[normalize-space()!='']").extract()[0].strip()
# extract subcategories by this criterion:
# find first subcategories list element following this criterion name, ignore if subcategory text starts with "See " ("See fewer", "See more")
subcategories = criterion.select(
"following-sibling::div[contains(@class,'accordionContainer')][1]/ul[@class='MainMenu AdvSearchMenu']/li/a[not(contains(text(), 'See '))]"
)
# then filter by regex only ones whose text contains at least one letter (for ex, for customers rating subcats, they have no name, only a picture with nr of starts, we don't want them)
subcategories = filter(
lambda x: x.select("text()").re(".*[A-Za-z]+.*"),
subcategories)
# if we found these, create the classification dictionary
if criterion_name and subcategories:
subcategories_list = []
for subcategory in subcategories:
subcategory_name = subcategory.select(
"@title").extract()[0]
# replace   with space, trim
subcategory_name = subcategory_name.replace(" ",
" ").strip()
# extract product count
subcategory_prodcount = subcategory.select(
"span[@class='count']/text()").extract()
# if there is no count field, extract prodcount from subcategory name
if subcategory_prodcount:
m = re.match("\(([0-9]+)\)",
subcategory_prodcount[0].strip())
# eliminate parantheses surrounding number and convert to int
if m:
subcategory_prodcount = m.group(1)
else:
subcategory_prodcount = subcategory_prodcount[
0].strip()
else:
# if there is no product count in separate element, try to extract it from subcategory name
subcategory_name = subcategory.select(
".//text()[normalize-space()!='']").extract(
)[0].replace(" ", " ").replace(u"\xa0",
" ").strip()
m = re.match("(.*)\(([0-9]+)\)", subcategory_name)
if m:
subcategory_prodcount = m.group(2)
subcategory_name = m.group(1).strip()
if subcategory_name and subcategory_prodcount:
subcategory_item = {
"name": subcategory_name,
"nr_products": int(subcategory_prodcount)
}
subcategories_list.append(subcategory_item)
classification_dictionary[criterion_name] = subcategories_list
if classification_dictionary:
item['classification'] = classification_dictionary
##########################################################################################
#
# Extract description title, text, wordcount, and keyword density (if any)
###########################################
#TODO:
# first search for the description id they usually use,
# second one is used more rarely and also with some false positives so needs to be checked for text length as well
# try to find div with detailedPageDescriptionCopyBlock id; move on only if not found
description_holder = hxs.select(
"//div[@id='detailedPageDescriptionCopyBlock']")
# flag to tell if we found it with basic rule
found = True
if not description_holder:
found = False
description_holder = hxs.select(
"//div[@class='CustomPOV ReminderBubbleSeeAll']//p/text()[string-length() > "
+ str(DESC_LEN) + "]/parent::*/parent::*")
# if none was found, try to find an element with much text (> DESC_LEN (200) characters)
# this is gonna pe a paragraph in the description, look for its parent (containing the entire description)
if not description_holder:
#description_holder = hxs.select("//*[not(self::script or self::style)]/text()[string-length() > " + str(DESC_LEN) + "]/parent::*/parent::*")
#TODO: !!does this mean string length for one paragraph is > DESC_LEN, or string length for entinre text content?
# I think it means entire text content. We're ok
description_holder = hxs.select("//p/text()[string-length() > " +
str(DESC_LEN) +
"]/parent::*/parent::*")
# select element among these with most text
if description_holder:
desc_winner = description_holder[0]
max_text = 0
for desc_candidate in description_holder:
# consider only text that is under a <p> tag and that has more than DESC_PAR_LEN (30) characters - then it's likely a description paragraph
description_texts = desc_candidate.select(
".//p//text()[string-length()>" + str(DESC_PAR_LEN) +
"]").extract()
text_len = len(" ".join(description_texts))
if text_len > max_text:
max_text = text_len
desc_winner = desc_candidate
# if text length is the same, assume one of them is parent of the other
# and select the one with greater depth (fewer children)
elif text_len == max_text and text_len != 0:
children_old = float(
desc_winner.select("count(*)").extract()[0])
children_new = float(
desc_candidate.select("count(*)").extract()[0])
if children_new < children_old:
desc_winner = desc_candidate
description_holder = desc_winner
# try to find description title in <b> tag in the holder;
# if it's not found, try to find it in the first <p> if the description
# if found there, exclude it from the description body
if description_holder:
#TODO:
# try this instead: ".//p//b/text() | .//h1/text() | .//h3/text() | .//strong/text() "
# to fix Money Center problem. but maybe it's not always inside p?
description_title = description_holder.select(
".//b/text() | .//h1/text() | .//h3/text() | .//strong/text() "
).extract()
if description_title:
# this will implicitly get thle first occurence of either a <b> element or an <h1> element,
# which is likely to be the title (the title usually comes first)
item['description_title'] = description_title[0].strip()
description_texts = description_holder.select(
"./div[position()<2]//p//text()[not(ancestor::b) and not(ancestor::h1) and not(ancestor::strong)] \
| ./p//text()[not(ancestor::b) and not(ancestor::h1) and not(ancestor::strong)]"
).extract()
# if the list is not empty and contains at least one non-whitespace item
if description_texts and reduce(
lambda x, y: x or y,
[line.strip() for line in description_texts]):
description_text = " ".join([
re.sub("\s+", " ", description_text.strip())
for description_text in description_texts
if description_text.strip()
])
# if it's larger than 4096 characters and not found with main rule it's probably not a descriptions; causes problem to PHP script as well. Ignore it
if len(description_text) < 4096 or found:
# replace all whitespace with one space, strip, and remove empty texts; then join them
item['description_text'] = description_text
# replace line breaks with space
item['description_text'] = re.sub("\n+", " ",
item['description_text'])
if 'description_text' in item:
tokenized = Utils.normalize_text(item['description_text'])
item['description_wc'] = len(tokenized)
# sometimes here there is no description title because of malformed html
# if we can find description text but not description title, title is probably malformed - get first text in div instead
if 'description_title' not in item:
desc_texts = description_holder.select(
"./text()").extract()
desc_texts = [text for text in desc_texts if text.strip()]
if desc_texts:
item['description_title'] = desc_texts[0].strip()
if 'description_title' in item:
(item['keyword_count'],
item['keyword_density']) = Utils.phrases_freq(
item['description_title'], item['description_text'])
else:
item['description_wc'] = 0
else:
item['description_wc'] = 0
#
##################################################################################
# Extract product count
# find if there is a wc field on the page
wc_field = hxs.select(
"//div[@class='mrl mod-toggleItemCount']/span/text() |\
//div[@class='SPRecordCount']/text()").extract()
if wc_field:
m1 = re.match("([0-9]+) Results", wc_field[0])
if m1:
item['nr_products'] = int(m1.group(1))
m2 = m2 = re.match(
"\s*Items\s*[0-9\-]+\s*of\s*([0-9]+)\s*total\s*", wc_field[0])
if m2:
item['nr_products'] = int(m2.group(1))
# set item count for special items (hardcoded in special_itemcount)
if item['text'].lower() in self.special_itemcount:
item['nr_products'] = self.special_itemcount[item['text'].lower()]
# Extract subcategories if no product count found
if 'nr_products' in item:
yield item
else:
# look for links to subcategory pages in menu
subcategories_links = hxs.select(
"//div[contains(@class, 'G1001 LeftNavRM')]/div[contains(@class, 'yuimenuitemlabel browseInOuter')]/a[@class='browseIn']"
)
if not subcategories_links:
# # if we haven't found them, try to find subcategories in menu on the left under a "Shop by Category" header
# subcategories_links = hxs.select("//div[@class='MainCopy']/div[@class='Header' and text()='\nShop by Category']/following-sibling::node()//a")
# if we haven't found them, try to find subcategories in menu on the left - get almost anything
subcategories_links = hxs.select(
"//div[@class='MainCopy']/div[@class='Header' and not(contains(text(),'Related Categories')) \
and not(contains(text(),'Special Offers')) and not(contains(text(),'View Top Registry Items')) and not(contains(text(),'Featured Content'))\
and not(contains(text(), 'Featured Brands'))]\
/following-sibling::node()//a")
# if we found them, create new category for each and parse it from the beginning
#TODO
########################################
# Exceptions - doesn't find anything for:
# http://photos.walmart.com/walmart/welcome?povid=cat121828-env999999-moduleA072012-lLinkGNAV5_PhotoCenter
#
#
########################################
if subcategories_links:
# new categories are subcategories of current one - calculate and store their level
parent_item = item
level = parent_item['level'] - 1
#print "URL ", response.url, " CALLING PARSEPAGE"
for subcategory in subcategories_links:
# to avoid rescraping categories reached from links in menu and reaching levels of -9,
# if level < -3 assume we've been there and skip
if level < -3:
continue
item = CategoryItem()
item['url'] = Utils.add_domain(
subcategory.select("@href").extract()[0],
self.root_url)
text = subcategory.select("text()").extract()
if text:
item['text'] = text[0].strip()
else:
# usually means it's something else than what we need
#TODO: check
continue
#print "no text for subcategory ", item, response.url
# # take care of unicode
# item['text'] = item['text'].encode("utf-8", errors=ignore)
item['level'] = level
item['parent_text'] = parent_item['text']
item['parent_url'] = parent_item['url']
item['parent_catid'] = parent_item['catid']
if 'parent_text' in parent_item:
item['grandparent_text'] = parent_item['parent_text']
if 'parent_url' in parent_item:
item['grandparent_url'] = parent_item['parent_url']
# if parent's parents are missing, level must be at least 0
if 'parent_text' not in parent_item or 'parent_url' not in parent_item:
assert level >= 0
# send subcategory items to be parsed again
# if not already crawled
if (item['url'], item['parent_url'],
response.meta['department_url']
) not in self.crawled:
yield Request(item['url'], callback = self.parseCategory, meta = {'item' : item, \
'department_text' : response.meta['department_text'], 'department_url' : response.meta['department_url'], 'department_id' : response.meta['department_id']})
self.crawled.append((item['url'], item['parent_url'],
response.meta['department_url']))
# return current item
# idea for sending parent and collecting nr products. send all of these subcats as a list in meta, pass it on, when list becomes empty, yield the parent
yield parent_item
#yield Request(item['url'], callback = self.parsePage, meta = {'item' : item, 'parent_item' : parent_item})
# if we can't find either products on the page or subcategory links
else:
#print "URL", response.url, " NO SUBCATs"
#item['nr_products'] = 0
yield item
def parse(self, response):
for url, text in self._scrape_department_links(response):
category = CategoryItem(text=text)
yield Request(url, callback=self._parse_category, meta={"category": category})
def parse(self, response):
hxs = HtmlXPathSelector(response)
# extract departments
departments = hxs.select("//h2")
department_id = 0
for department in departments:
department_item = CategoryItem()
department_text = department.select("text()").extract()[0]
department_item['department_text'] = department_text
# #TODO: add department_url, from sherwin-williams.com ...? get department list from there and match with departments from here by seeing if names match
department_item['department_id'] = department_id
department_item['text'] = department_text
department_item['level'] = 1
# get categories in department
categories = department.select("following-sibling::ul[1]/li")
# extract department url from one of its categories urls (it's not available directly)
category_ex = categories[0]
category_ex_url = Utils.add_domain(
category_ex.select("a/@href").extract()[0], self.base_url)
# extract first part of url
m = re.match("(http://www.sherwin\-williams\.com/[^/]+)/.*",
category_ex_url)
department_url = m.group(1)
department_item['department_url'] = department_url
department_item['url'] = department_url
for category in categories:
item = CategoryItem()
#TODO: special if 'Services'? or Specifications, or Ads...
category_text = category.select("a/text()").extract()[0]
category_url = Utils.add_domain(
category.select("a/@href").extract()[0], self.base_url)
item['text'] = category_text
item['url'] = category_url
# if it's not a 'products' category, mark it and all its subcategories as special
if category_text != 'Products':
item['special'] = 1
special = True
else:
special = False
item['department_id'] = department_id
item['department_text'] = department_text
item['department_url'] = department_url
item['parent_text'] = department_text
item['parent_url'] = department_url
item['level'] = 0
#TODO: do we need description_wc here as well?
yield Request(item['url'],
callback=self.parseCategory,
meta={'item': item})
# get subcategories in category
subcategories = category.select("ul/li")
for subcategory in subcategories:
item = CategoryItem()
item['text'] = subcategory.select("a/text()").extract()[0]
item['url'] = Utils.add_domain(
subcategory.select("a/@href").extract()[0],
self.base_url)
item['department_id'] = department_id
item['department_text'] = department_text
item['department_url'] = department_url
item['parent_text'] = category_text
item['parent_url'] = category_url
item['level'] = -1
# if parent is special, category is special
if special:
item['special'] = 1
yield Request(item['url'],
callback=self.parseSubcategory,
meta={'item': item})
department_id += 1
# return department
yield department_item
def parseCategory(self, response):
hxs = HtmlXPathSelector(response)
# get parent item from response, extract additional info and return it
item = response.meta['parent']
# add department name, url and id for item
item['department_text'] = response.meta['department_text']
item['department_url'] = response.meta['department_url']
item['department_id'] = response.meta['department_id']
# extract product count if available
nr_items_holder = hxs.select(
"//div[@id='showing']/strong[position()=2]/text()").extract()
if nr_items_holder:
item['nr_products'] = int(str(nr_items_holder[0]))
# extract description if available
# these are descriptions for services pages
desc_title_holder = hxs.select(
"//div[@id='searchstate']/a[position()=2]/text()").extract()
if desc_title_holder:
item['description_title'] = desc_title_holder[0].strip()
desc_content_holder = hxs.select(
"//div[@class='content']/h3/text()").extract()
if desc_content_holder:
item['description_text'] = desc_content_holder[0].strip()
tokenized = Utils.normalize_text(item['description_text'])
item['description_wc'] = len(tokenized)
(item['keyword_count'],
item['keyword_density']) = Utils.phrases_freq(
item['description_title'], item['description_text'])
else:
item['description_wc'] = 0
yield item
# extract its subcategories
#subcats_holders = hxs.select("//div[@class='narrowcontent']/ul[@class='search']")
subcats_holders = hxs.select(
"//div[@class='narrowcontent']/ul[contains(@class,'search')]")
if subcats_holders:
subcats_holder = subcats_holders[0]
# these are subcategories if they are preceded by the title "Shop ..."
title = subcats_holder.select(
"parent::node()/preceding-sibling::node()//text()").extract(
)[0]
if str(title).startswith("Shop"):
subcats = subcats_holder.select(".//li/a")
for subcat in subcats:
item = CategoryItem()
item['text'] = subcat.select("text()").extract()[0].strip()
item['url'] = Utils.add_domain(
subcat.select("@href").extract()[0],
"http://www.bestbuy.com")
parent = response.meta['parent']
item['level'] = int(response.meta['level']) - 1
# if parent was special, this category is special too
if 'special' in parent:
item['special'] = 1
item['parent_text'] = parent['text']
item['parent_url'] = parent['url']
request = Request(url = item['url'], callback = self.parseCategory, meta = {'parent' : item, 'level' : item['level'], \
'department_text' : response.meta['department_text'], 'department_url' : response.meta['department_url'], 'department_id' : response.meta['department_id']})
yield request
def parse(self, response):
# currently not extracting parents that are non-links (smaller parent categories like "resources" and "shops")
hxs = HtmlXPathSelector(response)
# # select all categories (bottom level)
# product_links = hxs.select("//div[@id='container']/div/header//nav/ul[@id='nav']//li/a")
# select all parent categories
#parent_links = hxs.select("//div[@id='container']/div/header//nav/ul[@id='nav']//h4/a")
parent_links = hxs.select(
"//div[@id='container']/div[@id='header']//nav/ul[@id='nav-touch']//h4/a"
)
#TODO: add extraction of level 3 categories (broadest: products, services,...)
# items = []
#############################################
# Extract all categories from sitemap instead of menus on category landing pages (lower level as well)
# for link in product_links:
# # retrieve parent category for this link
# parent = link.select("parent::node()/parent::node()/preceding-sibling::node()/a")
# item = CategoryItem()
# item['text'] = link.select('text()').extract()[0]
# item['url'] = link.select('@href').extract()[0]
# parent_text = parent.select('text()').extract()
# parent_url = parent.select('@href').extract()
# if parent_text:
# item['parent_text'] = parent_text[0]
# if parent_url:
# item['parent_url'] = parent_url[0]
# # mark it as special if a certain condition is checked
# if (link.select("parent::node()/parent::*[@class='nav-res']")):
# item['special'] = 1
# #TODO: add its direct parent if it's special (not among the categories). ex: shops, resources...
# # get grandparent of the category, mark item as special if grandparent is special
# grandparent = parent.select("parent::node()/parent::node()/parent::node()/parent::node()")
# if not grandparent.select('@class') or grandparent.select('@class').extract()[0] != 'nav-pro':
# item['special'] = 1
# grandparent_text = grandparent.select('a/text()').extract()
# grandparent_url = grandparent.select('a/@href').extract()
# if grandparent_text:
# item['grandparent_text'] = grandparent_text[0]
# if grandparent_url:
# item['grandparent_url'] = grandparent_url[0]
# item['level'] = 0
# items.append(item)
###############################################################
department_id = 0
for link in parent_links:
item = CategoryItem()
item['text'] = link.select('text()').extract()[0]
item['url'] = link.select('@href').extract()[0]
item['level'] = 1
parent = link.select(
"parent::node()/parent::node()/parent::node()/parent::node()")
# mark item as special if its parent is special
if not parent.select('@class').extract() or parent.select(
'@class').extract()[0] != "nav-pro":
item['special'] = 1
parent_text = parent.select('a/text()').extract()
# they don't actually have a url, only a #
#parent_url = parent.select('a/@href').extract()
if parent_text:
item['parent_text'] = parent_text[0]
# if parent_url:
# item['parent_url'] = parent_url[0]
#items.append(item)
department_id += 1
request = Request(item['url'], callback = self.parseCategory, meta = {'parent' : item, 'level' : 1, \
'department_text' : item['text'], 'department_url' : item['url'], 'department_id' : department_id})
yield request
def parseDept(self, response):
# for "copy & print" there's an exception, we don't need zipcode
# # use selenium to complete the zipcode form and get the first results page
# driver = webdriver.Firefox()
# driver.get(response.url)
# # set a hardcoded value for zipcode
# zipcode = "12345"
# textbox = driver.find_element_by_name("zipCode")
# textbox.send_keys(zipcode)
# button = driver.find_element_by_id("submitLink")
# button.click()
# cookie = {"zipcode": zipcode}
# driver.add_cookie(cookie)
# time.sleep(5)
# # convert html to "nice format"
# text_html = driver.page_source.encode('utf-8')
# #print "TEXT_HTML", text_html
# html_str = str(text_html)
# # this is a hack that initiates a "TextResponse" object (taken from the Scrapy module)
# resp_for_scrapy = TextResponse('none',200,{},html_str,[],None)
# hxs = HtmlXPathSelector(resp_for_scrapy)
#TODO: doesn't extract Televisions for ex
hxs = HtmlXPathSelector(response)
categories = hxs.select("//h2/a")
root_url = "http://www.staples.com"
# from parent's page:
item = response.meta['parent']
# add department name, url and id to item
item['department_text'] = response.meta['department_text']
item['department_url'] = response.meta['department_url']
item['department_id'] = response.meta['department_id']
# extract number of items, if any
nritems_holder = hxs.select(
"//div[@class='perpage']/span[@class='note']/text()").extract()
if nritems_holder:
m = re.findall("[0-9]+\s*items", nritems_holder[0])
if m:
item['nr_products'] = int("".join(re.findall("[0-9]+", m[0])))
# else:
# print "NOT MATCH ", nritems_holder[0]
# extract description, if any
description_texts = hxs.select(
"//h2[@class='seo short']//text() | //h2[@class='seo short long']//text()"
).extract()
if description_texts and reduce(
lambda x, y: x or y,
[line.strip() for line in description_texts]):
# replace all whitespace with one space, strip, and remove empty texts; then join them
item['description_text'] = " ".join([
re.sub("\s+", " ", description_text.strip())
for description_text in description_texts
if description_text.strip()
])
if item['description_text']:
item['description_title'] = item['text']
tokenized = Utils.normalize_text(item['description_text'])
item['description_wc'] = len(tokenized)
(item['keyword_count'],
item['keyword_density']) = Utils.phrases_freq(
item['description_title'], item['description_text'])
else:
# if no description is found
#print 'desc_holder but no desc_text ', response.URL
item['description_wc'] = 0
else:
item['description_wc'] = 0
# yield item the request came from (parent)
yield item
# extract subcategories
for category in categories:
# there are pages that don't have categories
item = CategoryItem()
text = category.select("text()").extract()
if text:
item['text'] = text[0]
url = category.select("@href").extract()
if url:
item['url'] = root_url + url[0]
item['level'] = int(response.meta['level'] - 1)
if 'text' in response.meta['parent']:
item['parent_text'] = response.meta['parent']['text']
else:
print 'no text in parent ', response.meta['parent']
item['parent_url'] = response.url
# yield the item after passing it through request and collecting additonal info
#yield item
# extract subcategories if any
zipcode = "12345"
request = Request(item['url'], callback = self.parseDept, cookies = {"zipcode" : zipcode}, \
headers = {"Cookie" : "zipcode=" + zipcode}, meta = {"dont_redirect" : True, "dont_merge_cookies" : True, \
"parent": item, "level": item['level'], \
"department_text" : response.meta["department_text"], "department_url" : response.meta["department_url"], "department_id" : response.meta["department_id"]})
yield request
def parseCategory(self, response):
hxs = HtmlXPathSelector(response)
# output received parent element after extracting additional info
item = response.meta['parent']
# add department name, url and id to item
item['department_text'] = response.meta['department_text']
item['department_url'] = response.meta['department_url']
item['department_id'] = response.meta['department_id']
# extract number of items if available
prod_count_holder = hxs.select(
"//span[@id='productCount']/text()").extract()
if prod_count_holder:
item['nr_products'] = int(prod_count_holder[0].strip())
# exract description if available
desc_holder = hxs.select("//div[@id='catalogCopyBlock']")
if desc_holder:
item['description_title'] = desc_holder.select(
"h2/text()").extract()[0]
description_texts = desc_holder.select("p/text()").extract()
# if the list is not empty and contains at least one non-whitespace item
if description_texts and reduce(
lambda x, y: x or y,
[line.strip() for line in description_texts]):
# replace all whitespace with one space, strip, and remove empty texts; then join them
item['description_text'] = " ".join([
re.sub("\s+", " ", description_text.strip())
for description_text in description_texts
if description_text.strip()
])
tokenized = Utils.normalize_text(item['description_text'])
item['description_wc'] = len(tokenized)
(item['keyword_count'],
item['keyword_density']) = Utils.phrases_freq(
item['description_title'], item['description_text'])
else:
item['description_wc'] = 0
else:
item['description_wc'] = 0
yield item
chapters = hxs.select("//li[@class='nav_cat_item_bold']")
for chapter in chapters:
#TODO: still includes some special categories (like "Coming Soon" in men)
# exclude "Brands" chapter
chapter_name = chapter.select("span/text()").extract()
if not chapter_name or "brands" in chapter_name[0]:
continue
subcats = chapter.select("ul/li/a")
for subcat in subcats:
item = CategoryItem()
text = subcat.select('text()').extract()[0]
# if it starts with "Shop all", ignore it
if re.match("Shop [aA]ll.*", text):
continue
else:
item['text'] = text
# remove unnecessary suffix from URL
url = subcat.select('@href').extract()[0]
m = re.match("(.*\?id=[0-9]+)&?.*", url)
if m:
item['url'] = m.group(1)
else:
item['url'] = url
item['level'] = int(response.meta['level']) - 1
item['parent_text'] = response.meta['parent']['text']
item['parent_url'] = response.url
#yield item
yield Request(item['url'], callback = self.parseCategory, meta = {'parent' : item, 'level' : item['level'], \
'department_text' : response.meta['department_text'], 'department_url' : response.meta['department_url'], 'department_id' : response.meta['department_id']})
def parse(self, response):
hxs = HtmlXPathSelector(response)
# if there is a captcha to solve, and we haven't exhausted our retries, try to solve it
if self.has_captcha(
response.body) and ('retry_count' not in response.meta
or response.meta['retry_count'] > 0):
yield self.solve_captcha_and_redirect(
response, self.parse
) # meta of response will contain number of retries left if set
return
links_level1 = hxs.select("//div[@id='siteDirectory']//table//a")
titles_level1 = hxs.select("//div//table//h2")
# add level 1 categories to items
# first one is a special category ("Unlimited Instant Videos"), add it separately
special_item = CategoryItem()
special_item['text'] = titles_level1[0].select('text()').extract()[0]
special_item['level'] = 2
special_item['special'] = 1
special_item['department_text'] = special_item['text']
special_item['department_id'] = self.department_count
self.department_count += 1
special_item['catid'] = self.catid
self.catid += 1
self.departments_ids[
special_item['text']] = special_item['department_id']
self.departments_cat_ids[special_item['text']] = special_item['catid']
#yield special_item
# if test category is set, and this is not it, ignore
if not self.test_category or special_item['text'] == self.test_category:
yield special_item
# the rest of the titles are not special
for title in titles_level1[1:]:
item = CategoryItem()
item['text'] = title.select('text()').extract()[0]
item['level'] = 2
item['department_text'] = item['text']
item['department_id'] = self.department_count
self.department_count += 1
item['catid'] = self.catid
self.catid += 1
self.departments_ids[item['text']] = item['department_id']
self.departments_cat_ids[item['text']] = item['catid']
# if item is found among EXTRA_TOPLEVEL_CATEGORIES_URLS, add info from that url
extra_category = self.find_matching_key(
item['text'], self.EXTRA_TOPLEVEL_CATEGORIES_URLS)
if extra_category:
item['url'] = self.EXTRA_TOPLEVEL_CATEGORIES_URLS[
extra_category]
item['department_url'] = item['url']
self.department_urls[item['text']] = item['url']
# if self.test_category is set, only send request if this is the test category
if self.test_category and item['text'] != self.test_category:
continue
# parse this category further
yield Request(item['url'],
callback=self.parseCategory,
meta={'item': item})
else:
# if test category is set and this is not it, ignore
if self.test_category and item['text'] != self.test_category:
continue
yield item
# add level 1 categories to items
for link in links_level1:
item = CategoryItem()
item['text'] = link.select('text()').extract()[0]
root_url = "http://www.amazon.com"
item['url'] = root_url + link.select('@href').extract()[0]
item['level'] = 1
parent = link.select(
"parent::node()/parent::node()/preceding-sibling::node()")
parent_text = parent.select('text()').extract()
# category should have a parent (its department) and that parent should have been extracted earlier (above) and put in the ids dictionary, necessary for getting the department id
assert parent_text
assert parent_text[0] in self.departments_ids
if parent_text:
item['parent_text'] = parent_text[0]
item['department_text'] = item['parent_text']
item['department_id'] = self.departments_ids[
item['department_text']]
item['parent_catid'] = self.departments_cat_ids[
item['department_text']]
item['catid'] = self.catid
self.catid += 1
# get department url from department_urls, will be availble only for extra_categories
if item['department_text'] in self.department_urls:
assert self.find_matching_key(
item['department_text'],
self.EXTRA_TOPLEVEL_CATEGORIES_URLS)
item['department_url'] = self.department_urls[
item['department_text']]
item['parent_url'] = item['department_url']
#TODO: leave this or not?
# Don't crawl subcategories of departments twice. If this is a department with url (extra_category), then we will crawl its subcategories. So ignore them here
#continue
# if its parent is the special category, mark this one as special too
if (item['parent_text'] == special_item['text']):
item['special'] = 1
special = True
else:
special = False
# department_id = self.department_count
# self.department_count += 1
# item['department_text'] = item['text']
# item['department_url'] = item['url']
# item['department_id'] = department_id
# if self.test_category is set, only send request if this is the test category
if self.test_category and item['text'] != self.test_category:
continue
yield Request(item['url'],
callback=self.parseCategory,
meta={'item': item})
def extractSubcategories(self, response):
# if there is a captcha to solve, and we haven't exhausted our retries, try to solve it
if self.has_captcha(
response.body) and ('retry_count' not in response.meta
or response.meta['retry_count'] > 0):
yield self.solve_captcha_and_redirect(
response, self.extractSubcategories
) # meta of response will contain number of retries left if set
return
hxs = HtmlXPathSelector(response)
# returned received item, then extract its subcategories
parent_item = response.meta['item']
yield parent_item
# extract subcategories, if level is above barrier
# extract subcategories from first menu on the left, assume this is the subcategories menu
if parent_item['level'] > self.LEVEL_BARRIER:
# check if it should be treated as a special category (exceptions to usual page structure); then extract the subcategories with the appropriate method
if self.isSpecialCategoryMenu(parent_item):
subcategories = self.extractSubcategoriesFromMenuSpecial(
hxs, parent_item)
# if no subcategories were found, try with the regular extraction as well (ex http://www.amazon.com/clothing-accessories-men-women-kids/b/ref=sd_allcat_apr/179-7724806-1781144?ie=UTF8&node=1036592)
if not subcategories:
subcategories = self.extractSubcategoriesFromMenu(hxs)
else:
subcategories = self.extractSubcategoriesFromMenu(hxs)
for (subcategory_text, subcategory_url,
subcategory_prodcount) in subcategories:
item = CategoryItem()
item['url'] = subcategory_url
item['text'] = subcategory_text
item['catid'] = self.catid
self.catid += 1
if subcategory_prodcount:
item['nr_products'] = int(subcategory_prodcount)
item['parent_text'] = parent_item['text']
item['parent_url'] = parent_item['url']
item['parent_catid'] = parent_item['catid']
# considering departments to be level 2 categories (top level) - so every category must have a department text
assert 'department_text' in parent_item
if 'department_text' in parent_item:
item['department_text'] = parent_item['department_text']
#item['department_url'] = parent_item['department_url']
item['department_id'] = parent_item['department_id']
# only level 2 categories in extra_categories have department_url
if 'department_url' in parent_item:
item['department_url'] = parent_item['department_url']
else:
assert not self.find_matching_key(
item['department_text'],
self.EXTRA_TOPLEVEL_CATEGORIES_URLS)
# else:
# # the parent must be a level 2 category - so this will be considered department
# assert parent_item['level'] == 2
# item['department_text'] = item['text']
# #item['department_url'] = item['url']
# item['department_id'] = self.department_count
# self.department_count += 1
item['level'] = parent_item['level'] - 1
# # no description extracted
# item['description_wc'] = 0
# send to parseCategory to extract description as well
yield Request(item['url'],
callback=self.parseCategory,
meta={'item': item})
def parsePage(self, response):
hxs = HtmlXPathSelector(response)
# selecting main level categories
links = hxs.select("//div[@id='sitemapLinks']/ul/li/ul/li/a")
# selecting low level categories
low_links = hxs.select("//div[@id='sitemapLinks']/ul/li/ul/li/ul/li/a")
# selecting lower level categories
lower_links = hxs.select("//div[@id='sitemapLinks']/ul/li/ul/li/ul/li/ul/li/a")
# selecting departments
departments = hxs.select("//div[@id='sitemapLinks']/ul/li/a")
items = []
root_url = "http://www.toysrus.com"
# add categories
for link in links:
# extract immediate parent
parent = link.select("parent::node()/parent::node()/parent::node()/a")
item = CategoryItem()
# add the current page url as a field
item['page_url'] = response.url
item['text'] = link.select('text()').extract()[0]
# build this into an absolute url by removing ".." prefix and adding domain
item['url'] = root_url + link.select('@href').extract()[0][2:]
item['parent_text'] = parent.select('text()').extract()[0]
item['parent_url'] = root_url + parent.select('@href').extract()[0][2:]
# this is the main level of categories
item['level'] = 0
items.append(item)
# add subcategories
for link in low_links:
# extract immediate parent
parent = link.select("parent::node()/parent::node()/parent::node()/a")
item = CategoryItem()
# add the current page url as a field
item['page_url'] = response.url
item['text'] = link.select('text()').extract()[0]
# build this into an absolute url by removing ".." prefix and adding domain
item['url'] = root_url + link.select('@href').extract()[0][2:]
item['parent_text'] = parent.select('text()').extract()[0]
item['parent_url'] = root_url + parent.select('@href').extract()[0][2:]
# extract grandparent
grandparent = parent.select("parent::node()/parent::node()/parent::node()/a")
item['grandparent_text'] = grandparent.select('text()').extract()[0]
item['grandparent_url'] = root_url + grandparent.select('@href').extract()[0][2:]
# these are subcategories
item['level'] = -1
items.append(item)
# add subsubcategories
for link in lower_links:
# extract immediate parent
parent = link.select("parent::node()/parent::node()/parent::node()/a")
item = CategoryItem()
# add the current page url as a field
item['page_url'] = response.url
item['text'] = link.select('text()').extract()[0]
# build this into an absolute url by removing ".." prefix and adding domain
item['url'] = root_url + link.select('@href').extract()[0][2:]
item['parent_text'] = parent.select('text()').extract()[0]
item['parent_url'] = root_url + parent.select('@href').extract()[0][2:]
# extract grandparent
grandparent = parent.select("parent::node()/parent::node()/parent::node()/a")
item['grandparent_text'] = grandparent.select('text()').extract()[0]
item['grandparent_url'] = root_url + grandparent.select('@href').extract()[0][2:]
# these are subsubcategories
item['level'] = -2
items.append(item)
# add departments
for link in departments:
item = CategoryItem()
# add the current page url as a field
item['page_url'] = response.url
item['text'] = link.select('text()').extract()[0]
# build this into an absolute url by removing ".." prefix and adding domain
item['url'] = root_url + link.select('@href').extract()[0][2:]
# these are departments
item['level'] = 1
# if it starts with "Save " ("Save 50% on ...") or "Buy " or contains "...% off" or starts with a date (number/number),
# mark it as special
#TODO: there are still some "Save up to..." items
special = re.match("(Save .*)|(Buy .*)|(.*[0-9]+\% off.*)|(.*[0-9]+/[0-9]+.*)", item['text'], re.UNICODE)
if special:
item['special'] = 1
items.append(item)
return items
def parsePage(self, response):
# currently selects only lowest level links, and their parents inside their fields
hxs = HtmlXPathSelector(response)
#TODO: add special categories if any
# select lowest level categories
links = hxs.select("//div[@class='siteMapSubCat']/ul/li/a")
# select parent categories
parent_links = hxs.select("//div[@class='siteMapSubCat']/h4/a")
# extract page name by getting text in url after "=" symbol
# example url: smv_10153_12605?vName=Appliances
page_name = response.url.split("=")[1]
# get partial list from previous function (containing departments)
items = response.meta['items']
root_url = "http://www.sears.com"
for link in links:
item = CategoryItem()
item['page_text'] = page_name
item['page_url'] = response.url
# add the page as the grandparent category
item['grandparent_text'] = page_name
item['grandparent_url'] = response.url
# extract parent category element
parent = link.select(
"./parent::node()/parent::node()/preceding-sibling::node()[2]/a"
)
parent_text = parent.select('text()').extract()
parent_url = parent.select('@href').extract()
if parent_text and parent_url:
item['parent_text'] = parent_text[0]
item['parent_url'] = root_url + parent_url[0]
# if you can't find a parent here go to first of the previous columns that has a parent
else:
parent = link.select(
"./parent::node()/parent::node()/parent::node()/preceding-sibling::node()[2]/h4[last()]"
)
parent_text = parent.select("a/text()").extract()
parent_url = parent.select("a/@href").extract()
index = 3
while not parent_text:
parent = link.select(
"./parent::node()/parent::node()/parent::node()/preceding-sibling::node()[%d]/h4[last()]"
% index)
index += 1
parent_text = parent.select("a/text()").extract()
parent_url = parent.select("a/@href").extract()
item['parent_text'] = parent_text[0]
item['parent_url'] = root_url + parent_url[0]
item['text'] = link.select('text()').extract()[0]
item['url'] = root_url + link.select('@href').extract()[0]
# these are subcategories
item['level'] = -1
items.append(item)
for link in parent_links:
item = CategoryItem()
item['text'] = link.select('text()').extract()[0]
item['url'] = root_url + link.select('@href').extract()[0]
item['page_text'] = page_name
item['page_url'] = response.url
# add the page as the parent category
item['parent_text'] = page_name
item['parent_url'] = response.url
# this is considered to be the main category level
item['level'] = 0
items.append(item)
# # add the page name as a department
# item = CategoryItem()
# item['text'] = page_name
# item['url'] = response.url
# item['page_text'] = page_name
# item['page_url'] = response.url
# item['level'] = 1
# items.append(item)
return items
def parse(self, response):
hxs = HtmlXPathSelector(response)
# select category links on all 3 levels
links = hxs.select("//div[@class='categories']/ul/li/ul/li/ul/li/a")
parent_links = hxs.select("//div[@class='categories']/ul/li/ul/li/a")
grandparent_links = hxs.select("//div[@class='categories']/ul/li/a")
# select special section "browse by brand"
special_links = hxs.select("//div[@class='brands']/ul/li/ul/li/a")
special_parent_links = hxs.select("//div[@class='brands']/ul/li/a")
items = []
for link in links:
# extracting parents
parent = link.select('parent::node()/parent::node()/parent::node()/a')
# extracting grandparents
grandparent = parent.select('parent::node()/parent::node()/parent::node()/a')
item = CategoryItem()
item['text'] = link.select('text()').extract()[0]
item['url'] = link.select('@href').extract()[0]
item['parent_text'] = parent.select('text()').extract()[0]
item['parent_url'] = parent.select('@href').extract()[0]
grandparent_text = grandparent.select('text()').extract()
grandparent_url = grandparent.select('@href').extract()
if grandparent_text:
item['grandparent_text'] = grandparent_text[0]
if grandparent_url:
item['grandparent_url'] = grandparent_text[0]
# this is considered more detailed than the main category level (compared to other sitemaps)
item['level'] = -1
items.append(item)
for link in parent_links:
# extracting parents
parent = link.select('parent::node()/parent::node()/parent::node()/a')
item = CategoryItem()
item['text'] = link.select('text()').extract()[0]
item['url'] = link.select('@href').extract()[0]
parent_text = parent.select('text()').extract()
parent_url = parent.select('@href').extract()
if (parent_text):
item['parent_text'] = parent_text[0]
if (parent_url):
item['parent_url'] = parent_url[0]
# this is considered the main category level
item['level'] = 0
items.append(item)
# last 2 categories need to be marked as special
for link in grandparent_links[:-2]:
item = CategoryItem()
item['text'] = link.select('text()').extract()[0]
item['url'] = link.select('@href').extract()[0]
item['level'] = 1
items.append(item)
for link in grandparent_links[-2:]:
item = CategoryItem()
item['text'] = link.select('text()').extract()[0]
item['url'] = link.select('@href').extract()[0]
item['level'] = 1
item['special'] = 1
items.append(item)
for link in special_links:
# extracting parents
parent = link.select('parent::node()/parent::node()/parent::node()/a')
item = CategoryItem()
item['text'] = link.select('text()').extract()[0]
item['url'] = link.select('@href').extract()[0]
parent_text = parent.select('text()').extract()
parent_url = parent.select('@href').extract()
if (parent_text):
item['parent_text'] = parent_text[0]
if (parent_url):
item['parent_url'] = parent_url[0]
# this is considered the main category level
item['level'] = 0
item['special'] = 1
items.append(item)
for link in special_parent_links:
item = CategoryItem()
item['text'] = link.select('text()').extract()[0]
item['url'] = link.select('@href').extract()[0]
item['level'] = 1
item['special'] = 1
items.append(item)
return items
def extract_nrprods_and_subcats(self, response):
hxs = HtmlXPathSelector(response)
item = response.meta['item']
# extract nr_products if not already extracted. necessary for extra_categories
if 'nr_products' not in item:
prod_count_holder = hxs.select("//h2[@class='resultCount']/span/text()").extract()
if prod_count_holder:
#print "DIDN'T HAVE PRODUCT COUNT", response.url
prod_count = prod_count_holder[0]
# extract number
m = re.match(".*\s*of\s*([0-9,]+)\s*Results\s*", prod_count)
if m:
item['nr_products'] = int(re.sub(",","",m.group(1)))
yield item
parent_item = item
# extract subcategories, if level is above barrier
# currently extracting subcategories for categories on any level, for level 2 this may cause duplicates (we already extract level 1)
# extract subcategories from first menu on the left, assume this is the subcategories menu
#TODO: test or make more robust
if item['level'] > self.LEVEL_BARRIER:
subcategories = hxs.select("//h2[1]/following-sibling::ul[1]/li/a")
for subcategory in subcategories:
# if we have a subcategory URL and product count with the expected format extract it, otherwise move on
if not subcategory.select("span[@class='refinementLink']"):
continue
subcategory_url = Utils.add_domain(subcategory.select("@href").extract()[0], "http://www.amazon.com")
subcategory_text = subcategory.select("span[@class='refinementLink']//text()").extract()[0].strip()
# extract product count, clean it of commas and parantheses
subcategory_prodcount_holder = subcategory.select("span[@class='narrowValue']/text()").extract()
if not subcategory_prodcount_holder:
continue
subcategory_prodcount = subcategory_prodcount_holder[0].replace(";nbsp&"," ").strip()
m = re.match("\(([0-9,]+)\)", subcategory_prodcount)
if m:
subcategory_prodcount = m.group(1).replace(",","")
item = CategoryItem()
item['url'] = subcategory_url
item['text'] = subcategory_text
item['parent_text'] = parent_item['text']
item['parent_url'] = parent_item['url']
# considering departments to be level 2 categories (top level) - so every category must have a department text
assert 'department_text' in parent_item
if 'department_text' in parent_item:
item['department_text'] = parent_item['department_text']
#item['department_url'] = parent_item['department_url']
item['department_id'] = parent_item['department_id']
# only level 2 categories in extra_categories have department_url
if 'department_url' in parent_item:
item['department_url'] = parent_item['department_url']
else:
assert not self.find_matching_key(item['department_text'], self.extra_toplevel_categories_urls)
if self.find_matching_key(item['department_text'], self.extra_toplevel_categories_urls):
print "DEPARTMENT_TEXT", item['department_text'], "--"+str(self.find_matching_key(item['department_text'], self.extra_toplevel_categories_urls))+"--"
# else:
# # the parent must be a level 2 category - so this will be considered department
# assert parent_item['level'] == 2
# item['department_text'] = item['text']
# #item['department_url'] = item['url']
# item['department_id'] = self.department_count
# self.department_count += 1
item['level'] = parent_item['level'] - 1
item['nr_products'] = subcategory_prodcount
# # no description extracted
# item['description_wc'] = 0
# send to parseCategory to extract description as well
yield Request(item['url'], callback = self.parseCategory, meta = {'item' : item})
def parse(self, response):
hxs = HtmlXPathSelector(response)
#links = hxs.select("//div[@class='MidContainer']/div[3]//a[@class='NavM']")
#parent_links = hxs.select("//div[@class='MidContainer']/div[3]//a[@class='NavXLBold']")
parent_links = hxs.select(
"//div[@class='MidContainer']/div/div/div[not(@class)]//a[@class='NavXLBold']"
)
# for link in links:
# item = CategoryItem()
# # search for the category's parent
# parents = []
# # select the preceding siblings that are a category title (have a child that is an a tag with a certain class)
# parents = link.select('parent::node()').select('preceding-sibling::node()').select('child::a[@class=\'NavXLBold\']')
# # if we found such siblings, get the last one to be the parent
# if parents:
# item['parent_text'] = parents[-1].select('text()').extract()[0]
# item['parent_url'] = parents[-1].select('@href').extract()[0]
# item['parent_url'] = Utils.add_domain(item['parent_url'], self.root_url)
# item['text'] = link.select('text()').extract()[0]
# item['url'] = link.select('@href').extract()[0]
# # add domain if relative URL
# item['url'] = Utils.add_domain(item['url'], self.root_url)
# item['level'] = 0
# to avoid duplicates, only extract highest level categories in this function (so don't return if level 0)
#yield item
# #TODO: check this
# item['nr_products'] = -1
# yield item
#yield Request(item['url'], callback = self.parseCategory, meta = {'item' : item})
department_id = 0
#TO remove:
# # artificial category - parent to all departments (root of entire sitemap tree). used to get total walmart product count
# sitemap_root = CategoryItem()
# sitemap_root['url'] = "http://www.walmart.com"
# sitemap_root['text'] = "Walmart"
# sitemap_root['department_id'] = 0
# sitemap_root['level'] = 2
# sitemap_root['catid'] = 0
# self.id_count += 1
# yield sitemap_root
for link in parent_links:
item = CategoryItem()
#TO remove:
# # link to artificial parent category
# item['parent_catid'] = 0
item['text'] = link.select('text()').extract()[0]
item['url'] = link.select('@href').extract()[0]
# add domain if relative URL
item['url'] = Utils.add_domain(item['url'], self.root_url)
item['level'] = 1
department_id += 1
# send category page to parseCategory function to extract description and number of products and add them to the item
yield Request(item['url'], callback = self.parseCategory, meta = {'item' : item, \
'department_text' : item['text'], 'department_url' : item['url'], 'department_id' : department_id})