def test_custom_function(default_params, xpi, server):
""" Test `custom_function` with an inline func that collects links """
table_name = TableName("page_links")
manager_params, browser_params = default_params
path = manager_params.data_directory / "crawl-data.sqlite"
db = sqlite3.connect(path)
cur = db.cursor()
cur.execute(
"""CREATE TABLE IF NOT EXISTS %s (
top_url TEXT, link TEXT,
visit_id INTEGER, browser_id INTEGER);"""
% table_name
)
cur.close()
db.close()
storage_provider = SQLiteStorageProvider(path)
manager = TaskManager(manager_params, browser_params, storage_provider, None)
cs = command_sequence.CommandSequence(url_a)
cs.get(sleep=0, timeout=60)
cs.append_command(CollectLinksCommand(table_name, "http"))
manager.execute_command_sequence(cs)
manager.close()
query_result = db_utils.query_db(
path,
"SELECT top_url, link FROM page_links;",
as_tuple=True,
)
assert PAGE_LINKS == set(query_result)
def test_saving(self):
manager_params, browser_params = self.get_config()
manager = TaskManager(manager_params, browser_params)
manager.get("http://example.com")
manager.close()
assert isfile(
join(browser_params[0].profile_archive_dir, "profile.tar.gz"))
def test_display_shutdown(self):
manager_params, browser_params = self.get_config()
TEST_SITE = BASE_TEST_URL + "/test_pages/simple_a.html"
manager = TaskManager(manager_params, browser_params)
port = manager.browsers[0].display_port
sequence = CommandSequence(TEST_SITE)
sequence.get()
sequence.append_command(ExceptionCommand)
manager.execute_command_sequence(sequence)
manager.close()
assert not os.path.exists("/tmp/.X%s-lock" % port)
def test_local_callbacks(self):
manager_params, browser_params = self.get_config()
TEST_SITE = BASE_TEST_URL + "/test_pages/simple_a.html"
manager = TaskManager(manager_params, browser_params)
def callback(argument: List[int], success: bool):
argument.extend([1, 2, 3])
my_list = []
sequence = CommandSequence(
TEST_SITE, reset=True, blocking=True, callback=partial(callback, my_list)
)
sequence.get()
manager.execute_command_sequence(sequence)
manager.close()
assert my_list == [1, 2, 3]
def test_profile_saved_when_launch_crashes(self):
manager_params, browser_params = self.get_config()
browser_params[0].proxy = True
browser_params[0].save_content = "script"
manager = TaskManager(manager_params, browser_params)
manager.get("http://example.com")
# Kill the LevelDBAggregator
# This will cause the proxy launch to crash
manager.ldb_status_queue.put("DIE")
manager.browsers[0]._SPAWN_TIMEOUT = 2 # Have timeout occur quickly
manager.browsers[0]._UNSUCCESSFUL_SPAWN_LIMIT = 2 # Quick timeout
manager.get("example.com") # Cause a selenium crash
# The browser will fail to launch due to the proxy crashes
try:
manager.get("http://example.com")
except CommandExecutionError:
pass
manager.close()
assert isfile(join(browser_params[0].profile_archive_dir, "profile.tar.gz"))
def test_seed_persistance(self):
manager_params, browser_params = self.get_test_config(num_browsers=1)
browser_params[0].seed_tar = "."
command_sequences = []
for _ in range(2):
cs = CommandSequence(url="https://example.com", reset=True)
cs.get()
cs.append_command(TestConfigSetCommand("test_pref", True))
command_sequences.append(cs)
manager = TaskManager(manager_params, browser_params)
for cs in command_sequences:
manager.execute_command_sequence(cs)
manager.close()
query_result = db_utils.query_db(
manager_params.database_name,
"SELECT * FROM crawl_history;",
)
assert len(query_result) > 0
for row in query_result:
assert row[
"command_status"] == "ok", f"Command {tuple(row)} was not ok"
# memory_watchdog and process_watchdog are useful for large scale cloud crawls.
# Please refer to docs/Configuration.md#platform-configuration-options for more information
# manager_params.memory_watchdog = True
# manager_params.process_watchdog = True
# Instantiates the measurement platform
# Commands time out by default after 60 seconds
manager = TaskManager(manager_params, browser_params)
# Visits the sites
for site in sites:
# Parallelize sites over all number of browsers set above.
command_sequence = CommandSequence(
site,
reset=True,
callback=lambda success, val=site: print("CommandSequence {} done".
format(val)),
)
# Start by visiting the page
command_sequence.append_command(GetCommand(url=site, sleep=10), timeout=60)
# Have a look at custom_command.py to see how to implement your own command
command_sequence.append_command(LinkCountingCommand())
# Run commands across the three browsers (simple parallelization)
manager.execute_command_sequence(command_sequence)
# Shuts down the browsers and waits for the data to finish logging
manager.close()