def crawl_category(self, ctx='', **kwargs): res = requests.get(HOST) res.raise_for_status() tree = lxml.html.fromstring(res.content) dept_nodes = tree.cssselect('div#top-navigation ul.navigation li.menu-item a') for dept_node in dept_nodes: key = dept_node.text.strip() if 'brand' in key.lower(): continue combine_url = dept_node.get('href') match = re.search(r'https?://.+', combine_url) if not match: combine_url = '%s%s' % (HOST, combine_url) r = requests.get(combine_url) r.raise_for_status() t = lxml.html.fromstring(r.content) pagesize_node = None link_nodes = t.cssselect('div.atg_store_filter ul.atg_store_pager li') for link_node in link_nodes: if link_node.get('class') and 'nextLink' in link_node.get('class'): break pagesize_node = link_node pagesize = int(pagesize_node.cssselect('a')[0].text.strip()) if pagesize_node else 1 is_new = False; is_updated = False category = Category.objects(key=key).first() if not category: is_new = True category = Category(key=key) category.is_leaf = True if combine_url and combine_url != category.combine_url: category.combine_url = combine_url is_updated = True if pagesize and pagesize != category.pagesize: category.pagesize = pagesize is_updated = True category.hit_time = datetime.utcnow() category.save() print category.key; print category.cats; print category.pagesize; print category.combine_url; print is_new; print is_updated; print; common_saved.send(sender=ctx, obj_type='Category', key=category.key, url=category.combine_url, \ is_new=is_new, is_updated=((not is_new) and is_updated) )