def test_seed_persistence(default_params, task_manager_creator):
manager_params, browser_params = default_params
p = Path("profile.tar.gz")
for browser_param in browser_params:
browser_param.seed_tar = p
manager, db = task_manager_creator(default_params)
command_sequences = []
for _ in range(2):
cs = CommandSequence(url=BASE_TEST_URL)
cs.get()
cs.append_command(AssertConfigSetCommand("test_pref", True))
command_sequences.append(cs)
for cs in command_sequences:
manager.execute_command_sequence(cs)
manager.close()
query_result = db_utils.query_db(
db,
"SELECT * FROM crawl_history;",
)
assert len(query_result) > 0
for row in query_result:
assert row[
"command_status"] == "ok", f"Command {tuple(row)} was not ok"
def test_profile_recovery(monkeypatch, default_params, task_manager_creator,
testcase, stateful, seed_tar):
"""Test browser profile recovery in various scenarios."""
manager_params, browser_params = default_params
manager_params.num_browsers = 1
browser_params[0].seed_tar = seed_tar
manager, db = task_manager_creator((manager_params, browser_params[:1]))
manager.get(BASE_TEST_URL, reset=not stateful)
if testcase == "normal_operation":
pass
elif testcase == "on_crash":
# Cause a selenium crash to force browser to restart
manager.get("example.com", reset=not stateful)
elif testcase == "on_crash_during_launch":
# Cause a selenium crash to force browser to restart
manager.get("example.com", reset=not stateful)
# This will cause browser restarts to fail
monkeypatch.setenv("FIREFOX_BINARY", "/tmp/NOTREAL")
# Let the launch succeed after some failed launch attempts
def undo_monkeypatch():
time.sleep(5) # This should be smaller than _SPAWN_TIMEOUT
monkeypatch.undo()
Thread(target=undo_monkeypatch).start()
elif testcase == "on_timeout":
# Set a very low timeout to cause a restart
manager.get("about:config", reset=not stateful, timeout=0.1)
cs = CommandSequence("about:config", reset=not stateful)
expected_value = True if seed_tar else False
cs.append_command(AssertConfigSetCommand("test_pref", expected_value))
tar_directory = manager_params.data_directory / "browser_profile"
tar_path = tar_directory / "profile.tar.gz"
cs.dump_profile(tar_path, True)
manager.execute_command_sequence(cs)
manager.close()
# Check that a consistent profile is used for stateful crawls but
# not for stateless crawls
with tarfile.open(tar_path) as tar:
tar.extractall(tar_directory)
ff_db = tar_directory / "places.sqlite"
rows = db_utils.query_db(ff_db, "SELECT url FROM moz_places")
places = [url for (url, ) in rows]
if stateful:
assert BASE_TEST_URL in places
else:
assert BASE_TEST_URL not in places
# Check if seed_tar was loaded on restart
rows = db_utils.query_db(
db,
"SELECT command_status FROM crawl_history WHERE"
" command='AssertConfigSetCommand'",
)
assert rows[0][0] == "ok"
def test_display_shutdown(self):
manager_params, browser_params = self.get_config()
TEST_SITE = BASE_TEST_URL + "/test_pages/simple_a.html"
manager = TaskManager(manager_params, browser_params)
port = manager.browsers[0].display_port
sequence = CommandSequence(TEST_SITE)
sequence.get()
sequence.append_command(ExceptionCommand)
manager.execute_command_sequence(sequence)
manager.close()
assert not os.path.exists("/tmp/.X%s-lock" % port)
def test_assertion_error_propagation(
task_manager_creator, default_params, testing, expectation
):
"""Test that assertion errors bubble up through the TaskManager when running tests"""
manager_params, browser_params = default_params
manager_params.num_browsers = 1
manager_params.testing = testing
manager, _ = task_manager_creator((manager_params, browser_params[:1]))
cs = CommandSequence("http://example.com", blocking=True)
cs.append_command(CrashingAssertionCommand())
with expectation:
with manager:
manager.execute_command_sequence(cs)
def test_display_shutdown(task_manager_creator, default_params):
"""Test the XVFB display option to see if it runs and deletes the lockfile upon shutdown"""
manager_params, browser_params = default_params
for browser_param in browser_params:
browser_param.display_mode = "xvfb"
TEST_SITE = BASE_TEST_URL + "/test_pages/simple_a.html"
manager, db = task_manager_creator((manager_params, browser_params))
port = manager.browsers[0].display_port
sequence = CommandSequence(TEST_SITE)
sequence.get()
sequence.append_command(ExceptionCommand())
manager.execute_command_sequence(sequence)
manager.close()
assert not os.path.exists("/tmp/.X%s-lock" % port)
def test_seed_persistance(self):
manager_params, browser_params = self.get_test_config(num_browsers=1)
browser_params[0].seed_tar = "."
command_sequences = []
for _ in range(2):
cs = CommandSequence(url="https://example.com", reset=True)
cs.get()
cs.append_command(TestConfigSetCommand("test_pref", True))
command_sequences.append(cs)
manager = TaskManager(manager_params, browser_params)
for cs in command_sequences:
manager.execute_command_sequence(cs)
manager.close()
query_result = db_utils.query_db(
manager_params.database_name,
"SELECT * FROM crawl_history;",
)
assert len(query_result) > 0
for row in query_result:
assert row[
"command_status"] == "ok", f"Command {tuple(row)} was not ok"
# memory_watchdog and process_watchdog are useful for large scale cloud crawls.
# Please refer to docs/Configuration.md#platform-configuration-options for more information
# manager_params.memory_watchdog = True
# manager_params.process_watchdog = True
# Instantiates the measurement platform
# Commands time out by default after 60 seconds
manager = TaskManager(manager_params, browser_params)
# Visits the sites
for site in sites:
# Parallelize sites over all number of browsers set above.
command_sequence = CommandSequence(
site,
reset=True,
callback=lambda success, val=site: print("CommandSequence {} done".
format(val)),
)
# Start by visiting the page
command_sequence.append_command(GetCommand(url=site, sleep=10), timeout=60)
# Have a look at custom_command.py to see how to implement your own command
command_sequence.append_command(LinkCountingCommand())
# Run commands across the three browsers (simple parallelization)
manager.execute_command_sequence(command_sequence)
# Shuts down the browsers and waits for the data to finish logging
manager.close()
def processSite(site):
# The list of sites that we wish to crawl
NUM_BROWSERS = 1
sites = [site]
# Loads the default ManagerParams
# and NUM_BROWSERS copies of the default BrowserParams
manager_params = ManagerParams(num_browsers=NUM_BROWSERS)
browser_params = [
BrowserParams(display_mode="headless") for _ in range(NUM_BROWSERS)
]
# Update browser configuration (use this for per-browser settings)
for i in range(NUM_BROWSERS):
# Record HTTP Requests and Responses
browser_params[i].http_instrument = True
# Record cookie changes
browser_params[i].cookie_instrument = True
# Record Navigations
browser_params[i].navigation_instrument = True
# Record JS Web API calls
browser_params[i].js_instrument = True
# Record the callstack of all WebRequests made
browser_params[i].callstack_instrument = True
# Record DNS resolution
browser_params[i].dns_instrument = True
# Update TaskManager configuration (use this for crawl-wide settings)
manager_params.data_directory = Path("./datadir/")
manager_params.log_directory = Path("./datadir/")
# memory_watchdog and process_watchdog are useful for large scale cloud crawls.
# Please refer to docs/Configuration.md#platform-configuration-options for more information
# manager_params.memory_watchdog = True
# manager_params.process_watchdog = True
# Commands time out by default after 60 seconds
with TaskManager(
manager_params,
browser_params,
SQLiteStorageProvider(Path("./datadir/crawl-data.sqlite")),
None,
) as manager:
# Visits the sites
for index, site in enumerate(sites):
def callback(success: bool, val: str = site) -> None:
print(
f"CommandSequence for {val} ran {'successfully' if success else 'unsuccessfully'}"
)
# Parallelize sites over all number of browsers set above.
command_sequence = CommandSequence(
site,
site_rank=index,
callback=callback,
)
# Start by visiting the page
command_sequence.append_command(GetCommand(url=site, sleep=3),
timeout=60)
# Have a look at custom_command.py to see how to implement your own command
command_sequence.append_command(LinkCountingCommand())
# Run commands across all browsers (simple parallelization)
manager.execute_command_sequence(command_sequence)
return (None)
def crawl(sites, db_filename):
'''
sites ihe list of sites that we wish to crawl
db_filename is the file name of the output database
'''
# Loads the default ManagerParams
# and NUM_BROWSERS copies of the default BrowserParams
NUM_BROWSERS = 12
manager_params = ManagerParams(num_browsers=NUM_BROWSERS)
browser_params = [
BrowserParams(display_mode="headless") for _ in range(NUM_BROWSERS)
]
# Update browser configuration (use this for per-browser settings)
for i in range(NUM_BROWSERS):
# Record HTTP Requests and Responses
browser_params[i].http_instrument = True
# Record cookie changes
browser_params[i].cookie_instrument = True
# Record Navigations
browser_params[i].navigation_instrument = True
# Record JS Web API calls
browser_params[i].js_instrument = True
# Record the callstack of all WebRequests made
browser_params[i].callstack_instrument = True
# Record DNS resolution
browser_params[i].dns_instrument = True
browser_params[i].bot_mitigation = True
# Update TaskManager configuration (use this for crawl-wide settings)
manager_params.data_directory = Path("./datadir/")
manager_params.log_directory = Path("./datadir/")
# Commands time out by default after 60 seconds
with TaskManager(
manager_params,
browser_params,
SQLiteStorageProvider(
Path("./datadir/{}.sqlite".format(db_filename))),
None,
) as manager:
# Visits the sites
for index, site in enumerate(sites):
def callback(success: bool, val: str = site) -> None:
print(
f"CommandSequence for {val} ran {'successfully' if success else 'unsuccessfully'}"
)
# Parallelize sites over all number of browsers set above.
command_sequence = CommandSequence(
site,
site_rank=index,
reset=True,
callback=callback,
)
# Start by visiting the page
command_sequence.append_command(GetCommand(url=site, sleep=3),
timeout=60)
# Run commands across the three browsers (simple parallelization)
manager.execute_command_sequence(command_sequence)