def process_item(self, item, spider): """Main function that process URL item (second phase).""" # canonicalize expanded URL without considering the status_code # because scrapy crawling not ganrantee the success # we still try to canonicalize the URL if len(item['expanded']) > MAX_URL_LEN: item['expanded'] = item['expanded'][:MAX_URL_LEN] logger.error('Expanded URL too long, trucate it! %r', item['raw']) item['canonical'] = canonicalize(item['expanded']) if item['canonical'] is None: item['status_code'] = U_HTML_ERROR_INVALID_URL # if url could be canonicalized and if site_id is not determined # we infer it from the expanded url if item['status_code'] != U_HTML_ERROR_INVALID_URL\ and item.get('site_id', None) is None: purl = get_parsed_url(item['expanded']) if purl is not None and purl.hostname is not None: if belongs_to_domain(purl.hostname, spider.excluded_domains)\ is not None: item['status_code'] = U_HTML_ERROR_EXCLUDED_DOMAIN else: item['site_id'] = belongs_to_site(purl.hostname, self.site_tuples) else: item['status_code'] = U_HTML_ERROR_INVALID_URL # remove potential NUL byte \x00 in the HTML if 'html' in item: item['html'] = item['html'].replace(b'\x00', '') try: # update database of url table spider.session.query(Url).filter_by(id=item['id'])\ .update(dict(item), synchronize_session=False) spider.session.commit() logger.debug('Fetched html of url %r with status %i', item['raw'], item['status_code']) except SQLAlchemyError as e: logger.error(e) spider.session.rollback() raise DropItem('Fail to update database of url: %s', item) return item
def process_item(self, item, spider): """Main function that process URL item (first phase).""" # validate URL length if len(item['raw']) > MAX_URL_LEN: item['raw'] = item['raw'][:MAX_URL_LEN] logger.error('Raw URL too long, trucate it! %r', item['raw']) # parse raw URL purl = get_parsed_url(item['raw']) if purl is None or purl.hostname is None: raise DropItem('Invalide URL') site_id = belongs_to_site(purl.hostname, self.site_tuples) if site_id is None: raise DropItem('Offsite domain: %s', item) item['site_id'] = site_id # insert URL into table try: get_or_create_murl(spider.session, item, spider.platform_id) except SQLAlchemyError as e: logger.error(e) spider.session.rollback() raise DropItem('Fail to insert database of url: %s', item) return item