def process_crawler( cls, timestamp: typing.Datetime, session: typing.Session, link: Link, record: CacheRecord ) -> None: # pylint: disable=unused-argument,line-too-long """Process the :class:`~requests.Response` object. Args: timestamp: Timestamp of the worker node reference. session (requests.Session): Session object with proxy settings and cookies storage. link: Link object to be loaded. record: Cached record from the remote database. """ response = session.get(link.url) ct_type = get_content_type(response) html = response.content # submit data submit_requests(timestamp, link, response, session, html, mime_type=ct_type, html=True) # add link to queue extracted_links = cls._extract_links(link, response.content) save_requests(extracted_links, score=0, nx=True) # add link to queue save_selenium(link, single=True, score=0, nx=True)
def fetch_hosts(link: 'darc_link.Link', force: bool = False) -> None: """Fetch ``hosts.txt``. Args: link: Link object to fetch for its ``hosts.txt``. force: Force refetch ``hosts.txt``. Returns: Content of the ``hosts.txt`` file. """ if force: logger.warning('[HOSTS] Force refetch %s', link.url) hosts_path = None if force else have_hosts(link) if hosts_path is not None: logger.warning('[HOSTS] Cached %s', link.url) # pylint: disable=no-member with open(hosts_path) as hosts_file: hosts_text = hosts_file.read() else: from darc.requests import i2p_session # pylint: disable=import-outside-toplevel hosts_link = parse_link(urljoin(link.url, '/hosts.txt'), backref=link) logger.info('[HOSTS] Subscribing %s', hosts_link.url) with i2p_session() as session: try: response = session.get(hosts_link.url) except requests.RequestException: logger.pexc(message=f'[HOSTS] Failed on {hosts_link.url}') return if not response.ok: logger.error('[HOSTS] Failed on %s [%d]', hosts_link.url, response.status_code) return ct_type = get_content_type(response) if ct_type not in ['text/text', 'text/plain']: logger.error('[HOSTS] Unresolved content type on %s (%s)', hosts_link.url, ct_type) return hosts_text = response.text save_hosts(hosts_link, hosts_text) logger.info('[HOSTS] Subscribed %s', hosts_link.url) from darc.db import save_requests # pylint: disable=import-outside-toplevel # add link to queue save_requests(read_hosts(link, hosts_text))
def process_loader( cls, timestamp: typing.Datetime, driver: typing.Driver, link: Link, record: CacheRecord) -> None: # pylint: disable=unused-argument """Process the :class:`WebDriver <selenium.webdriver.Chrome>` object. Args: timestamp: Timestamp of the worker node reference. driver (selenium.webdriver.Chrome): Web driver object with proxy settings and cookies presets. link: Link object to be loaded. record: Cached record from the remote database. """ driver.get(link.url) # wait for page to finish loading if SE_WAIT is not None: time.sleep(SE_WAIT) # get HTML source html = driver.page_source if html == SE_EMPTY: print(render_error(f'[SELENIUM] Empty page from {link.url}', stem.util.term.Color.RED), file=sys.stderr) # pylint: disable=no-member save_selenium(link, single=True) return screenshot = None try: # get maximum height height = driver.execute_script('return document.body.scrollHeight') # resize window (with some magic numbers) if height < 1000: height = 1000 driver.set_window_size(1024, math.ceil(height * 1.1)) # take a full page screenshot screenshot = driver.get_screenshot_as_base64() except Exception as error: print(render_error( f'[SELENIUM] Fail to save screenshot from {link.url} <{error}>', stem.util.term.Color.RED), file=sys.stderr) # pylint: disable=no-member # submit data submit_selenium(timestamp, link, html, screenshot) # add link to queue extracted_links = cls._extract_links(link, html) save_requests(extracted_links, score=0, nx=True)
def fetch_sitemap(link: 'darc_link.Link', force: bool = False) -> None: """Fetch sitemap. The function will first fetch the ``robots.txt``, then fetch the sitemaps accordingly. Args: link: Link object to fetch for its sitemaps. force: Force refetch its sitemaps. Returns: Contents of ``robots.txt`` and sitemaps. See Also: * :func:`darc.proxy.null.read_robots` * :func:`darc.proxy.null.read_sitemap` * :func:`darc.parse.get_sitemap` """ if force: logger.warning('[ROBOTS] Force refetch %s', link.url) robots_path = None if force else have_robots(link) if robots_path is not None: logger.warning('[ROBOTS] Cached %s', link.url) with open(robots_path) as file: robots_text = file.read() else: robots_link = parse_link(urljoin(link.url, '/robots.txt'), backref=link) logger.info('[ROBOTS] Checking %s', robots_link.url) with request_session(robots_link) as session: try: response = session.get(robots_link.url) except requests.RequestException: logger.pexc(message=f'[ROBOTS] Failed on {robots_link.url}') return if response.ok: ct_type = get_content_type(response) if ct_type not in ['text/text', 'text/plain']: logger.error('[ROBOTS] Unresolved content type on %s (%s)', robots_link.url, ct_type) robots_text = '' else: robots_text = response.text save_robots(robots_link, robots_text) logger.info('[ROBOTS] Checked %s', robots_link.url) else: logger.error('[ROBOTS] Failed on %s [%d]', robots_link.url, response.status_code) robots_text = '' if force: logger.warning('[SITEMAP] Force refetch %s', link.url) sitemaps = read_robots(link, robots_text, host=link.host) for sitemap_link in sitemaps: sitemap_path = None if force else have_sitemap(sitemap_link) if sitemap_path is not None: logger.warning('[SITEMAP] Cached %s', sitemap_link.url) with open(sitemap_path) as file: sitemap_text = file.read() else: logger.info('[SITEMAP] Fetching %s', sitemap_link.url) with request_session(sitemap_link) as session: try: response = session.get(sitemap_link.url) except requests.RequestException: logger.pexc(message=f'[SITEMAP] Failed on {sitemap_link.url}') continue if not response.ok: logger.error('[SITEMAP] Failed on %s [%d]', sitemap_link.url, response.status_code) continue # check content type ct_type = get_content_type(response) if ct_type == 'application/gzip': try: sitemap_text = gzip.decompress(response.content).decode() except UnicodeDecodeError: sitemap_text = response.text elif ct_type in ['text/xml', 'text/html']: sitemap_text = response.text save_sitemap(sitemap_link, sitemap_text) else: logger.error('[SITEMAP] Unresolved content type on %s (%s)', sitemap_link.url, ct_type) continue logger.info('[SITEMAP] Fetched %s', sitemap_link.url) # get more sitemaps sitemaps.extend(get_sitemap(sitemap_link, sitemap_text, host=link.host)) # add link to queue save_requests(read_sitemap(link, sitemap_text))
def fetch_sitemap(link: Link, force: bool = False) -> None: """Fetch sitemap. The function will first fetch the ``robots.txt``, then fetch the sitemaps accordingly. Args: link: Link object to fetch for its sitemaps. force: Force refetch its sitemaps. Returns: Contents of ``robots.txt`` and sitemaps. See Also: * :func:`darc.proxy.null.read_robots` * :func:`darc.proxy.null.read_sitemap` * :func:`darc.parse.get_sitemap` """ if force: print(stem.util.term.format(f'[ROBOTS] Force refetch {link.url}', stem.util.term.Color.YELLOW)) # pylint: disable=no-member robots_path = None if force else have_robots(link) if robots_path is not None: print(stem.util.term.format(f'[ROBOTS] Cached {link.url}', stem.util.term.Color.YELLOW)) # pylint: disable=no-member with open(robots_path) as file: robots_text = file.read() else: robots_link = parse_link(urljoin(link.url, '/robots.txt')) print(f'[ROBOTS] Checking {robots_link.url}') with request_session(robots_link) as session: try: response = session.get(robots_link.url) except requests.RequestException as error: print(render_error(f'[ROBOTS] Failed on {robots_link.url} <{error}>', stem.util.term.Color.RED), file=sys.stderr) # pylint: disable=no-member return if response.ok: ct_type = get_content_type(response) if ct_type not in ['text/text', 'text/plain']: print(render_error(f'[ROBOTS] Unresolved content type on {robots_link.url} ({ct_type})', stem.util.term.Color.RED), file=sys.stderr) # pylint: disable=no-member robots_text = '' else: robots_text = response.text save_robots(robots_link, robots_text) print(f'[ROBOTS] Checked {robots_link.url}') else: print(render_error(f'[ROBOTS] Failed on {robots_link.url} [{response.status_code}]', stem.util.term.Color.RED), file=sys.stderr) # pylint: disable=no-member robots_text = '' if force: print(stem.util.term.format(f'[SITEMAP] Force refetch {link.url}', stem.util.term.Color.YELLOW)) # pylint: disable=no-member sitemaps = read_robots(link, robots_text, host=link.host) for sitemap_link in sitemaps: sitemap_path = None if force else have_sitemap(sitemap_link) if sitemap_path is not None: print(stem.util.term.format(f'[SITEMAP] Cached {sitemap_link.url}', stem.util.term.Color.YELLOW)) # pylint: disable=no-member with open(sitemap_path) as file: sitemap_text = file.read() else: print(f'[SITEMAP] Fetching {sitemap_link.url}') with request_session(sitemap_link) as session: try: response = session.get(sitemap_link.url) except requests.RequestException as error: print(render_error(f'[SITEMAP] Failed on {sitemap_link.url} <{error}>', stem.util.term.Color.RED), file=sys.stderr) # pylint: disable=no-member continue if not response.ok: print(render_error(f'[SITEMAP] Failed on {sitemap_link.url} [{response.status_code}]', stem.util.term.Color.RED), file=sys.stderr) # pylint: disable=no-member continue # check content type ct_type = get_content_type(response) if ct_type == 'application/gzip': try: sitemap_text = gzip.decompress(response.content).decode() except UnicodeDecodeError: sitemap_text = response.text elif ct_type in ['text/xml', 'text/html']: sitemap_text = response.text save_sitemap(sitemap_link, sitemap_text) else: print(render_error(f'[SITEMAP] Unresolved content type on {sitemap_link.url} ({ct_type})', stem.util.term.Color.RED), file=sys.stderr) # pylint: disable=no-member continue print(f'[SITEMAP] Fetched {sitemap_link.url}') # get more sitemaps sitemaps.extend(get_sitemap(sitemap_link, sitemap_text, host=link.host)) # add link to queue save_requests(read_sitemap(link, sitemap_text))
def main(argv: 'Optional[List[str]]' = None) -> int: """Entrypoint. Args: argv: Optional command line arguments. Returns: Exit code. """ parser = get_parser() args = parser.parse_args(argv) pid = os.getpid() with open(PATH_ID, 'w') as file: print(pid, file=file) # wait for Redis if _WAIT_REDIS: if not FLAG_DB: _redis_command('set', 'darc', pid) if FLAG_DB: while True: try: with DB: _db_operation(DB.create_tables, [ HostnameQueueModel, RequestsQueueModel, SeleniumQueueModel, ]) except Exception: logger.pexc(LOG_WARNING, category=DatabaseOperaionFailed, line='DB.create_tables([HostnameQueueModel, ...]') continue break if SAVE_DB: while True: try: with DB_WEB: _db_operation(DB_WEB.create_tables, [ HostnameModel, URLModel, URLThroughModel, RobotsModel, SitemapModel, HostsModel, RequestsModel, RequestsHistoryModel, SeleniumModel, ]) except Exception: logger.pexc(LOG_WARNING, category=DatabaseOperaionFailed, line='DB.create_tables([HostnameModel, ...]') continue break logger.debug('-*- Initialisation -*-') if DEBUG and not FLAG_DB: # nuke the db _redis_command('delete', 'queue_hostname') _redis_command('delete', 'queue_requests') _redis_command('delete', 'queue_selenium') link_list = [] for link in filter( None, map(lambda s: s.strip(), args.link)): # type: ignore[name-defined,var-annotated] logger.pline(LOG_DEBUG, link) link_list.append(link) if args.file is not None: for path in args.file: with open(path) as file: for line in filter(None, map(lambda s: s.strip(), file)): if line.startswith('#'): continue logger.pline(LOG_DEBUG, line) link_list.append(line) # write to database link_pool = [parse_link(link, backref=None) for link in link_list] save_requests(link_pool, score=0, nx=True) logger.pline(LOG_DEBUG, logger.horizon) # init link file if not os.path.isfile(PATH_LN): with open(PATH_LN, 'w') as file: print('proxy,scheme,host,hash,link', file=file) try: process(args.type) except BaseException: traceback.print_exc() _exit() return 0
def fetch_hosts(link: Link): """Fetch ``hosts.txt``. Args: link: Link object to fetch for its ``hosts.txt``. Returns: Content of the ``hosts.txt`` file. """ hosts_path = have_hosts(link) if hosts_path is not None: print( stem.util.term.format(f'[HOSTS] Cached {link.url}', stem.util.term.Color.YELLOW)) # pylint: disable=no-member with open(hosts_path) as hosts_file: hosts_text = hosts_file.read() else: from darc.requests import i2p_session # pylint: disable=import-outside-toplevel hosts_link = parse_link(urljoin(link.url, '/hosts.txt')) print(f'[HOSTS] Subscribing {hosts_link.url}') with i2p_session() as session: try: response = session.get(hosts_link.url) except requests.RequestException as error: print(render_error( f'[HOSTS] Failed on {hosts_link.url} <{error}>', stem.util.term.Color.RED), file=sys.stderr) # pylint: disable=no-member return if not response.ok: print(render_error( f'[HOSTS] Failed on {hosts_link.url} [{response.status_code}]', stem.util.term.Color.RED), file=sys.stderr) # pylint: disable=no-member return ct_type = get_content_type(response) if ct_type not in ['text/text', 'text/plain']: print(render_error( f'[HOSTS] Unresolved content type on {hosts_link.url} ({ct_type}', stem.util.term.Color.RED), file=sys.stderr) # pylint: disable=no-member return hosts_text = response.text save_hosts(hosts_link, hosts_text) print(f'[HOSTS] Subscribed {hosts_link.url}') from darc.db import save_requests # pylint: disable=import-outside-toplevel # add link to queue save_requests(read_hosts(hosts_text))
def crawler(link: Link): """Single :mod:`requests` crawler for a entry link. Args: link: URL to be crawled by :mod:`requests`. The function will first parse the URL using :func:`~darc.link.parse_link`, and check if need to crawl the URL (c.f. :data:`~darc.const.PROXY_WHITE_LIST`, :data:`~darc.const.PROXY_BLACK_LIST`, :data:`~darc.const.LINK_WHITE_LIST` and :data:`~darc.const.LINK_BLACK_LIST`); if true, then crawl the URL with :mod:`requests`. If the URL is from a brand new host, :mod:`darc` will first try to fetch and save ``robots.txt`` and sitemaps of the host (c.f. :func:`~darc.proxy.null.save_robots` and :func:`~darc.proxy.null.save_sitemap`), and extract then save the links from sitemaps (c.f. :func:`~darc.proxy.null.read_sitemap`) into link database for future crawling (c.f. :func:`~darc.db.save_requests`). .. note:: A host is new if :func:`~darc.db.have_hostname` returns :data:`True`. If :func:`darc.proxy.null.fetch_sitemap` and/or :func:`darc.proxy.i2p.fetch_hosts` failed when fetching such documents, the host will be removed from the hostname database through :func:`~darc.db.drop_hostname`, and considered as new when next encounter. Also, if the submission API is provided, :func:`~darc.submit.submit_new_host` will be called and submit the documents just fetched. If ``robots.txt`` presented, and :data:`~darc.const.FORCE` is :data:`False`, :mod:`darc` will check if allowed to crawl the URL. .. note:: The root path (e.g. ``/`` in https://www.example.com/) will always be crawled ignoring ``robots.txt``. At this point, :mod:`darc` will call the customised hook function from :mod:`darc.sites` to crawl and get the final response object. :mod:`darc` will save the session cookies and header information, using :func:`~darc.save.save_headers`. .. note:: If :exc:`requests.exceptions.InvalidSchema` is raised, the link will be saved by :func:`~darc.proxy.null.save_invalid`. Further processing is dropped, and the link will be removed from the :mod:`requests` database through :func:`~darc.db.drop_requests`. If :exc:`~darc.error.LinkNoReturn` is raised, the link will be removed from the :mod:`requests` database through :func:`~darc.db.drop_requests`. If the content type of response document is not ignored (c.f. :data:`~darc.const.MIME_WHITE_LIST` and :data:`~darc.const.MIME_BLACK_LIST`), :func:`~darc.submit.submit_requests` will be called and submit the document just fetched. If the response document is HTML (``text/html`` and ``application/xhtml+xml``), :func:`~darc.parse.extract_links` will be called then to extract all possible links from the HTML document and save such links into the database (c.f. :func:`~darc.db.save_requests`). And if the response status code is between ``400`` and ``600``, the URL will be saved back to the link database (c.f. :func:`~darc.db.save_requests`). If **NOT**, the URL will be saved into :mod:`selenium` link database to proceed next steps (c.f. :func:`~darc.db.save_selenium`). """ print(f'[REQUESTS] Requesting {link.url}') try: if match_proxy(link.proxy): print(render_error( f'[REQUESTS] Ignored proxy type from {link.url} ({link.proxy})', stem.util.term.Color.YELLOW), file=sys.stderr) # pylint: disable=no-member drop_requests(link) return if match_host(link.host): print(render_error( f'[REQUESTS] Ignored hostname from {link.url} ({link.proxy})', stem.util.term.Color.YELLOW), file=sys.stderr) # pylint: disable=no-member drop_requests(link) return # timestamp timestamp = datetime.now() # if it's a new host if not have_hostname(link): partial = False if link.proxy not in ('zeronet', 'freenet'): # fetch sitemap.xml try: fetch_sitemap(link) except Exception: error = f'[Error fetching sitemap of {link.url}]' + os.linesep + traceback.format_exc() + '-' * shutil.get_terminal_size().columns # pylint: disable=line-too-long print(render_error(error, stem.util.term.Color.CYAN), file=sys.stderr) # pylint: disable=no-member partial = True if link.proxy == 'i2p': # fetch hosts.txt try: fetch_hosts(link) except Exception: error = f'[Error subscribing hosts from {link.url}]' + os.linesep + traceback.format_exc() + '-' * shutil.get_terminal_size().columns # pylint: disable=line-too-long print(render_error(error, stem.util.term.Color.CYAN), file=sys.stderr) # pylint: disable=no-member partial = True # submit data / drop hostname from db if partial: drop_hostname(link) submit_new_host(timestamp, link, partial=partial) if not FORCE and not check_robots(link): print(render_error( f'[REQUESTS] Robots disallowed link from {link.url}', stem.util.term.Color.YELLOW), file=sys.stderr) # pylint: disable=no-member return with request_session(link) as session: try: # requests session hook response = crawler_hook(link, session) except requests.exceptions.InvalidSchema as error: print(render_error( f'[REQUESTS] Failed on {link.url} <{error}>', stem.util.term.Color.RED), file=sys.stderr) # pylint: disable=no-member save_invalid(link) drop_requests(link) return except requests.RequestException as error: print(render_error( f'[REQUESTS] Failed on {link.url} <{error}>', stem.util.term.Color.RED), file=sys.stderr) # pylint: disable=no-member save_requests(link, single=True) return except LinkNoReturn: print(render_error( f'[REQUESTS] Removing from database: {link.url}', stem.util.term.Color.YELLOW), file=sys.stderr) # pylint: disable=no-member drop_requests(link) return # save headers save_headers(timestamp, link, response, session) # check content type ct_type = get_content_type(response) if ct_type not in ['text/html', 'application/xhtml+xml']: print(render_error( f'[REQUESTS] Generic content type from {link.url} ({ct_type})', stem.util.term.Color.YELLOW), file=sys.stderr) # pylint: disable=no-member # probably hosts.txt if link.proxy == 'i2p' and ct_type in [ 'text/plain', 'text/text' ]: text = response.text save_requests(read_hosts(text)) if match_mime(ct_type): drop_requests(link) return # submit data data = response.content submit_requests(timestamp, link, response, session, data, mime_type=ct_type, html=False) return html = response.content if not html: print(render_error( f'[REQUESTS] Empty response from {link.url}', stem.util.term.Color.RED), file=sys.stderr) # pylint: disable=no-member save_requests(link, single=True) return # submit data submit_requests(timestamp, link, response, session, html, mime_type=ct_type, html=True) # add link to queue save_requests(extract_links(link, html), score=0, nx=True) if not response.ok: print(render_error( f'[REQUESTS] Failed on {link.url} [{response.status_code}]', stem.util.term.Color.RED), file=sys.stderr) # pylint: disable=no-member save_requests(link, single=True) return # add link to queue save_selenium(link, single=True, score=0, nx=True) except Exception: error = f'[Error from {link.url}]' + os.linesep + traceback.format_exc() + '-' * shutil.get_terminal_size().columns # pylint: disable=line-too-long print(render_error(error, stem.util.term.Color.CYAN), file=sys.stderr) # pylint: disable=no-member save_requests(link, single=True) print(f'[REQUESTS] Requested {link.url}')
def loader(link: Link): """Single :mod:`selenium` loader for a entry link. Args: Link: URL to be crawled by :mod:`selenium`. The function will first parse the URL using :func:`~darc.link.parse_link` and start loading the URL using :mod:`selenium` with Google Chrome. At this point, :mod:`darc` will call the customised hook function from :mod:`darc.sites` to load and return the original :class:`selenium.webdriver.Chrome` object. .. note:: If :exc:`~darc.error.LinkNoReturn` is raised, the link will be removed from the :mod:`selenium` database through :func:`~darc.db.drop_selenium`. If successful, the rendered source HTML document will be saved, and a full-page screenshot will be taken and saved. .. note:: When taking full-page screenshot, :func:`~darc.crawl.loader` will use :javascript:`document.body.scrollHeight` to get the total height of web page. If the page height is *less than* **1,000 pixels**, then :mod:`darc` will by default set the height as **1,000 pixels**. Later :mod:`darc` will tell :mod:`selenium` to resize the window (in *headless* mode) to **1,024 pixels** in width and **110%** of the page height in height, and take a *PNG* screenshot. If the submission API is provided, :func:`~darc.submit.submit_selenium` will be called and submit the document just loaded. Later, :func:`~darc.parse.extract_links` will be called then to extract all possible links from the HTML document and save such links into the :mod:`requests` database (c.f. :func:`~darc.db.save_requests`). .. seealso:: * :data:`darc.const.SE_EMPTY` * :data:`darc.const.SE_WAIT` """ print(f'[SELENIUM] Loading {link.url}') try: # timestamp timestamp = datetime.now() # retrieve source from Chrome with request_driver(link) as driver: try: # selenium driver hook driver = loader_hook(link, driver) except urllib3.exceptions.HTTPError as error: print(render_error( f'[SELENIUM] Fail to load {link.url} <{error}>', stem.util.term.Color.RED), file=sys.stderr) # pylint: disable=no-member save_selenium(link, single=True) return except selenium.common.exceptions.WebDriverException as error: print(render_error( f'[SELENIUM] Fail to load {link.url} <{error}>', stem.util.term.Color.RED), file=sys.stderr) # pylint: disable=no-member save_selenium(link, single=True) return except LinkNoReturn: print(render_error( f'[SELENIUM] Removing from database: {link.url}', stem.util.term.Color.YELLOW), file=sys.stderr) # pylint: disable=no-member drop_selenium(link) return # get HTML source html = driver.page_source if html == SE_EMPTY: print(render_error(f'[SELENIUM] Empty page from {link.url}', stem.util.term.Color.RED), file=sys.stderr) # pylint: disable=no-member save_selenium(link, single=True) return screenshot = None try: # get maximum height height = driver.execute_script( 'return document.body.scrollHeight') # resize window (with some magic numbers) if height < 1000: height = 1000 driver.set_window_size(1024, math.ceil(height * 1.1)) # take a full page screenshot screenshot = driver.get_screenshot_as_base64() except Exception as error: print(render_error( f'[SELENIUM] Fail to save screenshot from {link.url} <{error}>', stem.util.term.Color.RED), file=sys.stderr) # pylint: disable=no-member # submit data submit_selenium(timestamp, link, html, screenshot) # add link to queue save_requests(extract_links(link, html), score=0, nx=True) except Exception: error = f'[Error from {link.url}]' + os.linesep + traceback.format_exc() + '-' * shutil.get_terminal_size().columns # pylint: disable=line-too-long print(render_error(error, stem.util.term.Color.CYAN), file=sys.stderr) # pylint: disable=no-member save_selenium(link, single=True) print(f'[SELENIUM] Loaded {link.url}')
def main(argv: typing.Optional[typing.List[str]] = None) -> int: """Entrypoint. Args: argv: Optional command line arguments. Returns: Exit code. """ parser = get_parser() args = parser.parse_args(argv) pid = os.getpid() with open(PATH_ID, 'w') as file: print(pid, file=file) # wait for Redis if _WAIT_REDIS: if not FLAG_DB: _redis_command('set', 'darc', pid) if FLAG_DB: while True: try: with DB: _db_operation(DB.create_tables, [ HostnameQueueModel, RequestsQueueModel, SeleniumQueueModel, ]) except Exception as error: warning = warnings.formatwarning( error, DatabaseOperaionFailed, __file__, 102, # type: ignore[arg-type] 'DB.create_tables([HostnameQueueModel, ...])') print(render_error(warning, stem.util.term.Color.YELLOW), end='', file=sys.stderr) # pylint: disable=no-member continue break if SAVE_DB: while True: try: with DB_WEB: _db_operation(DB_WEB.create_tables, [ HostnameModel, URLModel, RobotsModel, SitemapModel, HostsModel, RequestsModel, RequestsHistoryModel, SeleniumModel, ]) except Exception as error: warning = warnings.formatwarning( error, DatabaseOperaionFailed, __file__, 117, # type: ignore[arg-type] 'DB.create_tables([HostnameModel, ...])') print(render_error(warning, stem.util.term.Color.YELLOW), end='', file=sys.stderr) # pylint: disable=no-member continue break if DEBUG: print( stem.util.term.format('-*- Initialisation -*-', stem.util.term.Color.MAGENTA)) # pylint: disable=no-member # nuke the db if not FLAG_DB: _redis_command('delete', 'queue_hostname') _redis_command('delete', 'queue_requests') _redis_command('delete', 'queue_selenium') link_list = list() for link in filter( None, map(lambda s: s.strip(), args.link)): # type: ignore[name-defined,var-annotated] if DEBUG: print(stem.util.term.format(link, stem.util.term.Color.MAGENTA)) # pylint: disable=no-member link_list.append(link) if args.file is not None: for path in args.file: with open(path) as file: for line in filter(None, map(lambda s: s.strip(), file)): if line.startswith('#'): continue if DEBUG: print( stem.util.term.format( line, stem.util.term.Color.MAGENTA)) # pylint: disable=no-member link_list.append(line) # write to database link_pool = [parse_link(link) for link in link_list] save_requests(link_pool, score=0, nx=True) if DEBUG: print( stem.util.term.format('-' * shutil.get_terminal_size().columns, stem.util.term.Color.MAGENTA)) # pylint: disable=no-member # init link file if not os.path.isfile(PATH_LN): with open(PATH_LN, 'w') as file: print('proxy,scheme,host,hash,link', file=file) try: process(args.type) except BaseException: traceback.print_exc() _exit() return 0
def main(): """Entrypoint.""" parser = get_parser() args = parser.parse_args() pid = os.getpid() with open(PATH_ID, 'w') as file: print(pid, file=file) # wait for Redis if _WAIT_REDIS: if not FLAG_DB: _redis_command('set', 'darc', pid) if FLAG_DB: while True: with contextlib.suppress(Exception): with DB: DB.create_tables([ HostnameQueueModel, RequestsQueueModel, SeleniumQueueModel, ]) break if SAVE_DB: while True: with contextlib.suppress(Exception): with DB_WEB: DB_WEB.create_tables([ HostnameModel, URLModel, RobotsModel, SitemapModel, HostsModel, RequestsModel, RequestsHistoryModel, SeleniumModel, ]) break if DEBUG: print(stem.util.term.format('-*- Initialisation -*-', stem.util.term.Color.MAGENTA)) # pylint: disable=no-member # nuke the db if not FLAG_DB: _redis_command('delete', 'queue_hostname') _redis_command('delete', 'queue_requests') _redis_command('delete', 'queue_selenium') link_list = list() for link in filter(None, map(lambda s: s.strip(), args.link)): if DEBUG: print(stem.util.term.format(link, stem.util.term.Color.MAGENTA)) # pylint: disable=no-member link_list.append(link) if args.file is not None: for path in args.file: with open(path) as file: for line in filter(None, map(lambda s: s.strip(), file)): if line.startswith('#'): continue if DEBUG: print(stem.util.term.format(line, stem.util.term.Color.MAGENTA)) # pylint: disable=no-member link_list.append(line) # write to database link_pool = [parse_link(link) for link in link_list] save_requests(link_pool, score=0, nx=True) if DEBUG: print(stem.util.term.format('-' * shutil.get_terminal_size().columns, stem.util.term.Color.MAGENTA)) # pylint: disable=no-member # init link file if not os.path.isfile(PATH_LN): with open(PATH_LN, 'w') as file: print('proxy,scheme,host,hash,link', file=file) try: process(args.type) except BaseException: traceback.print_exc() _exit()
def crawler(link: 'darc_link.Link') -> None: """Single :mod:`requests` crawler for an entry link. Args: link: URL to be crawled by :mod:`requests`. The function will first parse the URL using :func:`~darc.link.parse_link`, and check if need to crawl the URL (c.f. :data:`~darc.const.PROXY_WHITE_LIST`, :data:`~darc.const.PROXY_BLACK_LIST`, :data:`~darc.const.LINK_WHITE_LIST` and :data:`~darc.const.LINK_BLACK_LIST`); if true, then crawl the URL with :mod:`requests`. If the URL is from a brand new host, :mod:`darc` will first try to fetch and save ``robots.txt`` and sitemaps of the host (c.f. :func:`~darc.proxy.null.save_robots` and :func:`~darc.proxy.null.save_sitemap`), and extract then save the links from sitemaps (c.f. :func:`~darc.proxy.null.read_sitemap`) into link database for future crawling (c.f. :func:`~darc.db.save_requests`). .. note:: A host is new if :func:`~darc.db.have_hostname` returns :data:`True`. If :func:`darc.proxy.null.fetch_sitemap` and/or :func:`darc.proxy.i2p.fetch_hosts` failed when fetching such documents, the host will be removed from the hostname database through :func:`~darc.db.drop_hostname`, and considered as new when next encounter. Also, if the submission API is provided, :func:`~darc.submit.submit_new_host` will be called and submit the documents just fetched. If ``robots.txt`` presented, and :data:`~darc.const.FORCE` is :data:`False`, :mod:`darc` will check if allowed to crawl the URL. .. note:: The root path (e.g. ``/`` in https://www.example.com/) will always be crawled ignoring ``robots.txt``. At this point, :mod:`darc` will call the customised hook function from :mod:`darc.sites` to crawl and get the final response object. :mod:`darc` will save the session cookies and header information, using :func:`~darc.save.save_headers`. .. note:: If :exc:`requests.exceptions.InvalidSchema` is raised, the link will be saved by :func:`~darc.proxy.null.save_invalid`. Further processing is dropped, and the link will be removed from the :mod:`requests` database through :func:`~darc.db.drop_requests`. If :exc:`~darc.error.LinkNoReturn` is raised, the link will be removed from the :mod:`requests` database through :func:`~darc.db.drop_requests`. If the content type of response document is not ignored (c.f. :data:`~darc.const.MIME_WHITE_LIST` and :data:`~darc.const.MIME_BLACK_LIST`), :func:`~darc.submit.submit_requests` will be called and submit the document just fetched. If the response document is HTML (``text/html`` and ``application/xhtml+xml``), :func:`~darc.parse.extract_links` will be called then to extract all possible links from the HTML document and save such links into the database (c.f. :func:`~darc.db.save_requests`). And if the response status code is between ``400`` and ``600``, the URL will be saved back to the link database (c.f. :func:`~darc.db.save_requests`). If **NOT**, the URL will be saved into :mod:`selenium` link database to proceed next steps (c.f. :func:`~darc.db.save_selenium`). """ logger.info('[REQUESTS] Requesting %s', link.url) try: if match_proxy(link.proxy): logger.warning('[REQUESTS] Ignored proxy type from %s (%s)', link.url, link.proxy) drop_requests(link) return if match_host(link.host): logger.warning('[REQUESTS] Ignored hostname from %s (%s)', link.url, link.proxy) drop_requests(link) return # timestamp timestamp = datetime.now() # get the session object in advance session = request_session(link) # check whether schema supported by :mod:`requests` try: session.get_adapter(link.url) # test for adapter requests_supported = True except requests.exceptions.InvalidSchema: requests_supported = False # if need to test for new host if requests_supported: # if it's a new host flag_have, force_fetch = have_hostname(link) if not flag_have or force_fetch: partial = False if link.proxy not in ('zeronet', 'freenet'): # fetch sitemap.xml try: fetch_sitemap(link, force=force_fetch) except Exception: logger.ptb('[Error fetching sitemap of %s]', link.url) partial = True if link.proxy == 'i2p': # fetch hosts.txt try: fetch_hosts(link, force=force_fetch) except Exception: logger.ptb('[Error subscribing hosts from %s]', link.url) partial = True # submit data / drop hostname from db if partial: drop_hostname(link) submit_new_host(timestamp, link, partial=partial, force=force_fetch) if not FORCE and not check_robots(link): logger.warning('[REQUESTS] Robots disallowed link from %s', link.url) return # reuse the session object with session: try: # requests session hook response = crawler_hook(timestamp, session, link) except requests.exceptions.InvalidSchema: logger.pexc(message=f'[REQUESTS] Fail to crawl {link.url}') save_invalid(link) drop_requests(link) return except requests.RequestException as error: logger.pexc(message=f'[REQUESTS] Fail to crawl {link.url}') save_requests(link, single=True) return except LinkNoReturn as error: logger.pexc(LOG_WARNING, f'[REQUESTS] Removing from database: {link.url}') if error.drop: drop_requests(link) return # save headers save_headers(timestamp, link, response, session) # check content type ct_type = get_content_type(response) if ct_type not in ['text/html', 'application/xhtml+xml']: logger.warning('[REQUESTS] Generic content type from %s (%s)', link.url, ct_type) # probably hosts.txt if link.proxy == 'i2p' and ct_type in [ 'text/plain', 'text/text' ]: text = response.text save_requests(read_hosts(link, text)) if match_mime(ct_type): drop_requests(link) return # submit data data = response.content submit_requests(timestamp, link, response, session, data, mime_type=ct_type, html=False) return html = response.content if not html: logger.error('[REQUESTS] Empty response from %s', link.url) save_requests(link, single=True) return # submit data submit_requests(timestamp, link, response, session, html, mime_type=ct_type, html=True) # add link to queue save_requests(extract_links(link, html), score=0, nx=True) if not response.ok: logger.error('[REQUESTS] Failed on %s [%d]', link.url, response.status_code) save_requests(link, single=True) return # add link to queue save_selenium(link, single=True, score=0, nx=True) except Exception: if SAVE_DB: with contextlib.suppress(Exception): host = HostnameModel.get_or_none( HostnameModel.hostname == link.host) # type: Optional[HostnameModel] if host is not None: host.alive = False host.save() with contextlib.suppress(Exception): url = URLModel.get_or_none( URLModel.hash == link.name) # type: Optional[URLModel] if url is not None: url.alias = False url.save() logger.ptb('[Error from %s]', link.url) save_requests(link, single=True) logger.info('[REQUESTS] Requested %s', link.url)
def loader(link: 'darc_link.Link') -> None: """Single :mod:`selenium` loader for an entry link. Args: Link: URL to be crawled by :mod:`selenium`. The function will first parse the URL using :func:`~darc.link.parse_link` and start loading the URL using :mod:`selenium` with Google Chrome. At this point, :mod:`darc` will call the customised hook function from :mod:`darc.sites` to load and return the original :class:`selenium.webdriver.chrome.webdriver.WebDriver` object. .. note:: If :exc:`~darc.error.LinkNoReturn` is raised, the link will be removed from the :mod:`selenium` database through :func:`~darc.db.drop_selenium`. If successful, the rendered source HTML document will be saved, and a full-page screenshot will be taken and saved. .. note:: When taking full-page screenshot, :func:`~darc.crawl.loader` will use :javascript:`document.body.scrollHeight` to get the total height of web page. If the page height is *less than* **1,000 pixels**, then :mod:`darc` will by default set the height as **1,000 pixels**. Later :mod:`darc` will tell :mod:`selenium` to resize the window (in *headless* mode) to **1,024 pixels** in width and **110%** of the page height in height, and take a *PNG* screenshot. If the submission API is provided, :func:`~darc.submit.submit_selenium` will be called and submit the document just loaded. Later, :func:`~darc.parse.extract_links` will be called then to extract all possible links from the HTML document and save such links into the :mod:`requests` database (c.f. :func:`~darc.db.save_requests`). .. seealso:: * :data:`darc.const.SE_EMPTY` * :data:`darc.const.SE_WAIT` """ logger.info('[SELENIUM] Loading %s', link.url) try: # timestamp timestamp = datetime.now() # retrieve source from Chrome with request_driver(link) as driver: try: # selenium driver hook driver = loader_hook(timestamp, driver, link) except urllib3_exceptions.HTTPError: logger.pexc(message=f'[SELENIUM] Fail to load {link.url}') save_selenium(link, single=True) return except selenium_exceptions.WebDriverException as error: logger.pexc(message=f'[SELENIUM] Fail to load {link.url}') save_selenium(link, single=True) return except LinkNoReturn as error: logger.pexc(LOG_WARNING, f'[SELENIUM] Removing from database: {link.url}') if error.drop: drop_selenium(link) return # get HTML source html = driver.page_source if html == SE_EMPTY: logger.error('[SELENIUM] Empty page from %s', link.url) save_selenium(link, single=True) return screenshot = None try: # get maximum height height = driver.execute_script( 'return document.body.scrollHeight') # resize window (with some magic numbers) driver.set_window_size(1024, math.ceil(max(height, 1000) * 1.1)) # take a full page screenshot screenshot = driver.get_screenshot_as_base64() except Exception: logger.pexc( message= f'[SELENIUM] Fail to save screenshot from {link.url}') # submit data submit_selenium(timestamp, link, html, screenshot) # add link to queue save_requests(extract_links(link, html), score=0, nx=True) except Exception: logger.ptb('[Error from %s]', link.url) save_selenium(link, single=True) logger.info('[SELENIUM] Loaded %s', link.url)