def parse_item(self, response):
item = UrlItem()
item['raw'] = response.url
item['expanded'] = response.url
item['canonical'] = canonicalize(item['expanded'])
try:
text = response.body
code = response.encoding
html = str(text.decode(code, 'ignore'))\
.encode('utf-8', 'ignore')
item['html'] = html
# We already have HTML, set the status_code to second phrase sucess
item['status_code'] = U_HTML_SUCCESS
except Exception as e:
logger.error(e)
yield item
def process_item(self, item, spider):
"""Main function that process URL item (second phase)."""
# canonicalize expanded URL without considering the status_code
# because scrapy crawling not ganrantee the success
# we still try to canonicalize the URL
if len(item['expanded']) > MAX_URL_LEN:
item['expanded'] = item['expanded'][:MAX_URL_LEN]
logger.error('Expanded URL too long, trucate it! %r', item['raw'])
item['canonical'] = canonicalize(item['expanded'])
if item['canonical'] is None:
item['status_code'] = U_HTML_ERROR_INVALID_URL
# if url could be canonicalized and if site_id is not determined
# we infer it from the expanded url
if item['status_code'] != U_HTML_ERROR_INVALID_URL\
and item.get('site_id', None) is None:
purl = get_parsed_url(item['expanded'])
if purl is not None and purl.hostname is not None:
if belongs_to_domain(purl.hostname, spider.excluded_domains)\
is not None:
item['status_code'] = U_HTML_ERROR_EXCLUDED_DOMAIN
else:
item['site_id'] = belongs_to_site(purl.hostname,
self.site_tuples)
else:
item['status_code'] = U_HTML_ERROR_INVALID_URL
# remove potential NUL byte \x00 in the HTML
if 'html' in item:
item['html'] = item['html'].replace(b'\x00', '')
try:
# update database of url table
spider.session.query(Url).filter_by(id=item['id'])\
.update(dict(item), synchronize_session=False)
spider.session.commit()
logger.debug('Fetched html of url %r with status %i', item['raw'],
item['status_code'])
except SQLAlchemyError as e:
logger.error(e)
spider.session.rollback()
raise DropItem('Fail to update database of url: %s', item)
return item