def test_saving(self):
manager_params, browser_params = self.get_config()
manager = task_manager.TaskManager(manager_params, browser_params)
manager.get("http://example.com")
manager.close()
assert isfile(
join(browser_params[0]["profile_archive_dir"], "profile.tar.gz"))
def test_http_stacktrace(self):
test_url = utilities.BASE_TEST_URL + "/http_stacktrace.html"
manager_params, browser_params = self.get_config()
manager = task_manager.TaskManager(manager_params, browser_params)
manager.get(test_url, sleep=10)
db = manager_params["db"]
manager.close()
rows = db_utils.query_db(
db,
(
"SELECT hr.url, c.call_stack"
" FROM callstacks c"
" JOIN http_requests hr"
" ON c.request_id=hr.request_id"
" AND c.visit_id= hr.visit_id"
" AND c.browser_id = hr.browser_id;"
),
)
print("Printing callstacks contents")
observed_records = set()
for row in rows:
print(row["call_stack"])
url, call_stack = row
test_urls = (
"inject_pixel.js",
"test_image.png",
"Blank.gif",
)
if url.endswith(test_urls):
observed_records.add(call_stack)
assert HTTP_STACKTRACES == observed_records
def test_extension_gets_correct_visit_id(self):
manager_params, browser_params = self.get_config()
manager = task_manager.TaskManager(manager_params, browser_params)
url_a = utilities.BASE_TEST_URL + "/simple_a.html"
url_b = utilities.BASE_TEST_URL + "/simple_b.html"
manager.get(url_a)
manager.get(url_b)
manager.close()
qry_res = db_utils.query_db(
manager_params.database_name,
"SELECT visit_id, site_url FROM site_visits")
# Construct dict mapping site_url to visit_id
visit_ids = dict()
for row in qry_res:
visit_ids[row[1]] = row[0]
simple_a_visit_id = db_utils.query_db(
manager_params.database_name,
"SELECT visit_id FROM javascript WHERE "
"symbol=?",
("window.navigator.userAgent", ),
)
simple_b_visit_id = db_utils.query_db(
manager_params.database_name,
"SELECT visit_id FROM javascript WHERE "
"symbol=?",
("window.navigator.platform", ),
)
assert visit_ids[url_a] == simple_a_visit_id[0][0]
assert visit_ids[url_b] == simple_b_visit_id[0][0]
def test_content_saving(self, tmpdir):
""" check that content is saved and hashed correctly """
test_url = utilities.BASE_TEST_URL + "/http_test_page.html"
manager_params, browser_params = self.get_test_config(str(tmpdir))
browser_params[0]["http_instrument"] = True
browser_params[0]["save_content"] = True
manager = task_manager.TaskManager(manager_params, browser_params)
manager.get(url=test_url, sleep=1)
manager.close()
db = manager_params["db"]
rows = db_utils.query_db(db, "SELECT * FROM http_responses;")
disk_content = dict()
for row in rows:
if "MAGIC_REDIRECT" in row["url"] or "404" in row["url"]:
continue
path = urlparse(row["url"]).path
with open(os.path.join(BASE_PATH, path[1:]), "rb") as f:
content = f.read()
chash = sha256(content).hexdigest()
assert chash == row["content_hash"]
disk_content[chash] = content
ldb_content = dict()
for chash, content in db_utils.get_content(str(tmpdir)):
chash = chash.decode("ascii")
ldb_content[chash] = content
for k, v in disk_content.items():
assert v == ldb_content[k]
def test_document_saving(http_params, xpi, server):
""" check that document content is saved and hashed correctly """
test_url = utilities.BASE_TEST_URL + "/http_test_page.html"
expected_hashes = {
"2390eceab422db15bc45940b7e042e83e6cbd5f279f57e714bc4ad6cded7f966",
"25343f42d9ffa5c082745f775b172db87d6e14dfbc3160b48669e06d727bfc8d",
}
manager_params, browser_params = http_params()
for browser_param in browser_params:
browser_param.http_instrument = True
browser_param.save_content = "main_frame,sub_frame"
structured_storage = SQLiteStorageProvider(
db_path=manager_params.data_directory / "crawl-data.sqlite")
ldb_path = Path(manager_params.data_directory) / "content.ldb"
unstructured_storage = LevelDbProvider(db_path=ldb_path)
manager = task_manager.TaskManager(manager_params, browser_params,
structured_storage,
unstructured_storage)
manager.get(url=test_url, sleep=1)
manager.close()
for chash, content in db_utils.get_content(ldb_path):
chash = chash.decode("ascii").lower()
pyhash = sha256(content).hexdigest().lower()
assert pyhash == chash # Verify expected key (sha256 of content)
assert chash in expected_hashes
expected_hashes.remove(chash)
assert len(expected_hashes) == 0 # All expected hashes have been seen
def test_javascript_saving(http_params, xpi, server):
""" check that javascript content is saved and hashed correctly """
test_url = utilities.BASE_TEST_URL + "/http_test_page.html"
manager_params, browser_params = http_params()
for browser_param in browser_params:
browser_param.http_instrument = True
browser_param.save_content = "script"
structured_storage = SQLiteStorageProvider(
db_path=manager_params.data_directory / "crawl-data.sqlite")
ldb_path = Path(manager_params.data_directory) / "content.ldb"
unstructured_storage = LevelDbProvider(db_path=ldb_path)
manager = task_manager.TaskManager(manager_params, browser_params,
structured_storage,
unstructured_storage)
manager.get(url=test_url, sleep=1)
manager.close()
expected_hashes = {
"0110c0521088c74f179615cd7c404816816126fa657550032f75ede67a66c7cc",
"b34744034cd61e139f85f6c4c92464927bed8343a7ac08acf9fb3c6796f80f08",
}
for chash, content in db_utils.get_content(ldb_path):
chash = chash.decode("ascii").lower()
pyhash = sha256(content).hexdigest().lower()
assert pyhash == chash # Verify expected key (sha256 of content)
assert chash in expected_hashes
expected_hashes.remove(chash)
assert len(expected_hashes) == 0 # All expected hashes have been seen
def test_browse_site_visits_table_valid(self, display_mode):
"""Check that CommandSequence.browse() populates db correctly."""
# Run the test crawl
manager_params, browser_params = self.get_config(display_mode)
manager = task_manager.TaskManager(manager_params, browser_params)
# Set up two sequential browse commands to two URLS
cs_a = command_sequence.CommandSequence(url_a, site_rank=0)
cs_a.browse(num_links=1, sleep=1)
cs_b = command_sequence.CommandSequence(url_b, site_rank=1)
cs_b.browse(num_links=1, sleep=1)
manager.execute_command_sequence(cs_a)
manager.execute_command_sequence(cs_b)
manager.close()
qry_res = db_utils.query_db(
manager_params["db"], "SELECT site_url, site_rank"
" FROM site_visits")
# We had two separate page visits
assert len(qry_res) == 2
assert qry_res[0][0] == url_a
assert qry_res[0][1] == 0
assert qry_res[1][0] == url_b
assert qry_res[1][1] == 1
def test_save_screenshot_valid(self, display_mode):
"""Check that 'save_screenshot' works"""
# Run the test crawl
manager_params, browser_params = self.get_config(display_mode)
manager = task_manager.TaskManager(manager_params, browser_params)
cs = command_sequence.CommandSequence(url_a)
cs.get(sleep=1)
cs.save_screenshot("test")
cs.screenshot_full_page("test_full")
manager.execute_command_sequence(cs)
manager.close()
# Check that viewport image is not blank
pattern = os.path.join(str(self.tmpdir), "screenshots", "1-*-test.png")
screenshot = glob.glob(pattern)[0]
im = Image.open(screenshot)
bands = im.split()
is_blank = all(band.getextrema() == (255, 255) for band in bands)
assert not is_blank
# Check that full page screenshot is not blank
pattern = os.path.join(str(self.tmpdir), "screenshots",
"1-*-test_full.png")
screenshot = glob.glob(pattern)[0]
im = Image.open(screenshot)
bands = im.split()
is_blank = all(band.getextrema() == (255, 255) for band in bands)
assert not is_blank
def test_get_site_visits_table_valid(self, display_mode):
"""Check that get works and populates db correctly."""
# Run the test crawl
manager_params, browser_params = self.get_config(display_mode)
manager = task_manager.TaskManager(manager_params, browser_params)
# Set up two sequential get commands to two URLS
cs_a = command_sequence.CommandSequence(url_a)
cs_a.get(sleep=1)
cs_b = command_sequence.CommandSequence(url_b)
cs_b.get(sleep=1)
# Perform the get commands
manager.execute_command_sequence(cs_a)
manager.execute_command_sequence(cs_b)
manager.close()
qry_res = db_utils.query_db(manager_params["db"],
"SELECT site_url FROM site_visits")
# We had two separate page visits
assert len(qry_res) == 2
assert qry_res[0][0] == url_a
assert qry_res[1][0] == url_b
def test_crash(self):
manager_params, browser_params = self.get_config()
manager_params["failure_limit"] = 0
manager = task_manager.TaskManager(manager_params, browser_params)
with pytest.raises(CommandExecutionError):
manager.get("http://example.com") # So we have a profile
manager.get("example.com") # Selenium requires scheme prefix
manager.get("example.com") # Requires two commands to shut down
def visit(self, page_url, data_dir="", sleep_after=0):
"""Visit a test page with the given parameters."""
manager_params, browser_params = self.get_config(data_dir)
manager = task_manager.TaskManager(manager_params, browser_params)
if not page_url.startswith("http"):
page_url = utilities.BASE_TEST_URL + page_url
manager.get(url=page_url, sleep=sleep_after)
manager.close()
return manager_params.database_name
def test_get_http_tables_valid(self, display_mode):
"""Check that get works and populates http tables correctly."""
# Run the test crawl
manager_params, browser_params = self.get_config(display_mode)
manager = task_manager.TaskManager(manager_params, browser_params)
# Set up two sequential get commands to two URLS
cs_a = command_sequence.CommandSequence(url_a)
cs_a.get(sleep=1)
cs_b = command_sequence.CommandSequence(url_b)
cs_b.get(sleep=1)
manager.execute_command_sequence(cs_a)
manager.execute_command_sequence(cs_b)
manager.close()
qry_res = db_utils.query_db(
manager_params.database_name,
"SELECT visit_id, site_url FROM site_visits")
# Construct dict mapping site_url to visit_id
visit_ids = dict()
for row in qry_res:
visit_ids[row[1]] = row[0]
qry_res = db_utils.query_db(
manager_params.database_name,
"SELECT visit_id FROM http_requests"
" WHERE url = ?",
(url_a, ),
)
assert qry_res[0][0] == visit_ids[url_a]
qry_res = db_utils.query_db(
manager_params.database_name,
"SELECT visit_id FROM http_requests"
" WHERE url = ?",
(url_b, ),
)
assert qry_res[0][0] == visit_ids[url_b]
qry_res = db_utils.query_db(
manager_params.database_name,
"SELECT visit_id FROM http_responses"
" WHERE url = ?",
(url_a, ),
)
assert qry_res[0][0] == visit_ids[url_a]
qry_res = db_utils.query_db(
manager_params.database_name,
"SELECT visit_id FROM http_responses"
" WHERE url = ?",
(url_b, ),
)
assert qry_res[0][0] == visit_ids[url_b]
def test_record_file_upload(self):
"""Test that we correctly capture the uploaded file contents.
We upload a CSS file and a PNG file to test both text based and
binary files.
File uploads are not expected in the crawl data, but we make sure we
correctly parse the POST data in this very common scenario.
Firefox is currently not able to return the FormData with the file
contents, currently only the filenames are returned. This is due to
a limitation in the current API implementation:
https://searchfox.org/mozilla-central/rev/b3b401254229f0a26f7ee625ef5f09c6c31e3949/toolkit/components/extensions/webrequest/WebRequestUpload.jsm#339
Therefore, the test is currently skipped.
"""
img_file_path = os.path.abspath("test_pages/shared/test_image.png")
css_file_path = os.path.abspath("test_pages/shared/test_style.css")
def type_filenames_into_form(**kwargs):
"""Simulate typing into the file upload input fields."""
driver = kwargs["driver"]
img_file_upload_element = driver.find_element_by_id("upload-img")
css_file_upload_element = driver.find_element_by_id("upload-css")
img_file_upload_element.send_keys(img_file_path)
css_file_upload_element.send_keys(css_file_path)
sleep(5) # wait for the form submission (3 sec after onload)
manager_params, browser_params = self.get_config()
manager = task_manager.TaskManager(manager_params, browser_params)
test_url = utilities.BASE_TEST_URL + "/post_file_upload.html"
cs = command_sequence.CommandSequence(test_url)
cs.get(sleep=0, timeout=60)
cs.run_custom_function(type_filenames_into_form, ())
manager.execute_command_sequence(cs)
manager.close()
post_body = self.get_post_request_body_from_db(
manager_params.database_name)
# Binary strings get put into the database as-if they were latin-1.
with open(img_file_path, "rb") as f:
img_file_content = f.read().strip().decode("latin-1")
with open(css_file_path, "rt") as f:
css_file_content = f.read().strip()
# POST data is stored as JSON in the DB
post_body_decoded = json.loads(post_body)
expected_body = {
u"username": u"name surname+",
u"upload-css": css_file_content,
u"upload-img": img_file_content,
}
assert expected_body == post_body_decoded
def test_custom_function(self):
""" Test `custom_function` with an inline func that collects links """
from openwpm.socket_interface import ClientSocket
def collect_links(table_name, scheme, **kwargs):
""" Collect links with `scheme` and save in table `table_name` """
driver = kwargs["driver"]
manager_params = kwargs["manager_params"]
browser_id = kwargs["command"].browser_id
visit_id = kwargs["command"].visit_id
link_urls = [
x
for x in (element.get_attribute("href")
for element in driver.find_elements_by_tag_name("a"))
if x.startswith(scheme + "://")
]
current_url = driver.current_url
sock = ClientSocket()
sock.connect(*manager_params.aggregator_address)
query = ("CREATE TABLE IF NOT EXISTS %s ("
"top_url TEXT, link TEXT, "
"visit_id INTEGER, browser_id INTEGER);" % table_name)
sock.send(("create_table", query))
for link in link_urls:
query = (
table_name,
{
"top_url": current_url,
"link": link,
"visit_id": visit_id,
"browser_id": browser_id,
},
)
sock.send(query)
sock.close()
manager_params, browser_params = self.get_config()
manager = task_manager.TaskManager(manager_params, browser_params)
cs = command_sequence.CommandSequence(url_a)
cs.get(sleep=0, timeout=60)
cs.run_custom_function(collect_links, ("page_links", "http"))
manager.execute_command_sequence(cs)
manager.close()
query_result = db_utils.query_db(
manager_params.database_name,
"SELECT top_url, link FROM page_links;",
as_tuple=True,
)
assert PAGE_LINKS == set(query_result)
def test_crash_profile(self):
manager_params, browser_params = self.get_config()
manager_params["failure_limit"] = 2
manager = task_manager.TaskManager(manager_params, browser_params)
try:
manager.get("http://example.com") # So we have a profile
manager.get("example.com") # Selenium requires scheme prefix
manager.get("example.com") # Selenium requires scheme prefix
manager.get("example.com") # Selenium requires scheme prefix
manager.get("example.com") # Requires two commands to shut down
except CommandExecutionError:
pass
assert isfile(
join(browser_params[0]["profile_archive_dir"], "profile.tar.gz"))
def test_commit_on_timeout(self):
TEST_SITE = "%s/s3_aggregator.html" % BASE_TEST_URL
manager_params, browser_params = self.get_config(num_browsers=1)
manager_params.s3_directory = "s3-aggregator-tests-2"
manager = task_manager.TaskManager(manager_params, browser_params)
manager.get(TEST_SITE, sleep=1)
dataset = LocalS3Dataset(manager_params.s3_bucket, manager_params.s3_directory)
with pytest.raises((FileNotFoundError, OSError)):
requests = dataset.load_table("http_requests")
time.sleep(45) # Current timeout
dataset2 = LocalS3Dataset(manager_params.s3_bucket, manager_params.s3_directory)
requests = dataset2.load_table("http_requests")
assert TEST_SITE in requests.top_level_url.unique()
manager.close()
def test_parse_neterror_integration(self):
manager_params, browser_params = self.get_config()
manager = task_manager.TaskManager(manager_params, browser_params)
manager.get("http://website.invalid")
manager.close()
get_command = db_utils.query_db(
manager_params["db"],
"SELECT command_status, error FROM crawl_history WHERE command = \"<class 'openwpm.commands.types.GetCommand'>\"",
as_tuple=True,
)[0]
assert get_command[0] == "neterror"
assert get_command[1] == "dnsNotFound"
def test_custom_function(self):
""" Test `custom_function` with an inline func that collects links """
manager_params, browser_params = self.get_config()
manager = task_manager.TaskManager(manager_params, browser_params)
cs = command_sequence.CommandSequence(url_a)
cs.get(sleep=0, timeout=60)
cs.append_command(CollectLinksCommand("http", "page_links"))
manager.execute_command_sequence(cs)
manager.close()
query_result = db_utils.query_db(
manager_params.database_name,
"SELECT top_url, link FROM page_links;",
as_tuple=True,
)
assert PAGE_LINKS == set(query_result)
def test_command_duration(self):
manager_params, browser_params = self.get_config()
manager = task_manager.TaskManager(manager_params, browser_params)
manager.get(url=TEST_URL, sleep=5)
manager.close()
get_command = db_utils.query_db(
manager_params.database_name,
"SELECT duration FROM crawl_history WHERE command = \"<class 'openwpm.commands.types.GetCommand'>\"",
as_tuple=True,
)[0]
assert get_command[0] > (5 * 1000
) # milliseconds conversion for sleep time
assert get_command[0] <= (
(5 * 1000) + 2 * 1000
) # milliseconds conversion for sleep time + time duration a command took (milliseconds)
def visit(self,
page_url: str,
data_dir: Optional[Path] = None,
sleep_after: int = 0) -> Path:
"""Visit a test page with the given parameters."""
manager_params, browser_params = self.get_config(data_dir)
if data_dir:
db_path = data_dir / "crawl-data.sqlite"
else:
db_path = self.tmpdir / "crawl-data.sqlite"
structured_provider = SQLiteStorageProvider(db_path)
manager = task_manager.TaskManager(manager_params, browser_params,
structured_provider, None)
if not page_url.startswith("http"):
page_url = utilities.BASE_TEST_URL + page_url
manager.get(url=page_url, sleep=sleep_after)
manager.close()
return db_path
def test_recursive_dump_page_source_valid(self, display_mode):
"""Check that 'recursive_dump_page_source' works"""
# Run the test crawl
manager_params, browser_params = self.get_config(display_mode)
manager = task_manager.TaskManager(manager_params, browser_params)
cs = command_sequence.CommandSequence(NESTED_FRAMES_URL)
cs.get(sleep=1)
cs.recursive_dump_page_source()
manager.execute_command_sequence(cs)
manager.close()
outfile = os.path.join(str(self.tmpdir), "sources", "1-*.json.gz")
src_file = glob.glob(outfile)[0]
with gzip.GzipFile(src_file, "rb") as f:
visit_source = json.loads(f.read().decode("utf-8"))
observed_parents = dict()
def verify_frame(frame, parent_frames=[]):
# Verify structure
observed_parents[frame["doc_url"]] = list(parent_frames) # copy
# Verify source
path = urlparse(frame["doc_url"]).path
expected_source = ""
with open("." + path, "r") as f:
expected_source = re.sub(r"\s", "", f.read().lower())
if expected_source.startswith("<!doctypehtml>"):
expected_source = expected_source[14:]
observed_source = re.sub(r"\s", "", frame["source"].lower())
if observed_source.startswith("<!doctypehtml>"):
observed_source = observed_source[14:]
assert observed_source == expected_source
# Verify children
parent_frames.append(frame["doc_url"])
for key, child_frame in frame["iframes"].items():
verify_frame(child_frame, parent_frames)
parent_frames.pop()
verify_frame(visit_source)
assert EXPECTED_PARENTS == observed_parents
def test_basic_properties(self):
TEST_SITE = "%s/s3_aggregator.html" % BASE_TEST_URL
NUM_VISITS = 2
NUM_BROWSERS = 4
manager_params, browser_params = self.get_config(num_browsers=NUM_BROWSERS)
manager = task_manager.TaskManager(manager_params, browser_params)
for _ in range(NUM_VISITS * NUM_BROWSERS):
manager.get(TEST_SITE, sleep=1)
manager.close()
dataset = LocalS3Dataset(
manager_params["s3_bucket"], manager_params["s3_directory"]
)
# Test visit_id consistency
visit_ids = defaultdict(set)
expected_tables = dict(PQ_SCHEMAS)
# We don't expect incomplete visits to exist
# since the visit shouldn't be interrupted
expected_tables.pop("incomplete_visits")
for table_name in expected_tables:
table = dataset.load_table(table_name)
visit_ids[table_name] = table.visit_id.unique()
actual = len(visit_ids[table_name])
expected = NUM_VISITS * NUM_BROWSERS
assert actual == expected, (
f"Table {table_name} had {actual} " f"visit_ids, we expected {expected}"
)
for vid in visit_ids[table_name]:
assert (vid >= 0) and (vid < (1 << 53))
for table_name, ids in visit_ids.items():
assert set(ids) == set(visit_ids["site_visits"])
# Ensure http table is created
assert TEST_SITE in dataset.load_table("http_requests").top_level_url.unique()
# Ensure config directory is created and contains the correct number
# of configuration files
config_file = dataset.list_files("config", prepend_root=True)
assert len(config_file) == 1 # only one instance started in test
config = json.loads(str(dataset.get_file(config_file[0]), "utf-8"))
assert len(config["browser_params"]) == NUM_BROWSERS
def test_document_saving(self, tmpdir):
""" check that document content is saved and hashed correctly """
test_url = utilities.BASE_TEST_URL + "/http_test_page.html"
expected_hashes = {
"2390eceab422db15bc45940b7e042e83e6cbd5f279f57e714bc4ad6cded7f966",
"25343f42d9ffa5c082745f775b172db87d6e14dfbc3160b48669e06d727bfc8d",
}
manager_params, browser_params = self.get_test_config(str(tmpdir))
browser_params[0]["http_instrument"] = True
browser_params[0]["save_content"] = "main_frame,sub_frame"
manager = task_manager.TaskManager(manager_params, browser_params)
manager.get(url=test_url, sleep=1)
manager.close()
for chash, content in db_utils.get_content(str(tmpdir)):
chash = chash.decode("ascii").lower()
pyhash = sha256(content).hexdigest().lower()
assert pyhash == chash # Verify expected key (sha256 of content)
assert chash in expected_hashes
expected_hashes.remove(chash)
assert len(expected_hashes) == 0 # All expected hashes have been seen
def test_javascript_saving(self, tmpdir):
""" check that javascript content is saved and hashed correctly """
test_url = utilities.BASE_TEST_URL + "/http_test_page.html"
manager_params, browser_params = self.get_test_config(str(tmpdir))
browser_params[0]["http_instrument"] = True
browser_params[0]["save_content"] = "script"
manager = task_manager.TaskManager(manager_params, browser_params)
manager.get(url=test_url, sleep=1)
manager.close()
expected_hashes = {
"0110c0521088c74f179615cd7c404816816126fa657550032f75ede67a66c7cc",
"b34744034cd61e139f85f6c4c92464927bed8343a7ac08acf9fb3c6796f80f08",
}
for chash, content in db_utils.get_content(str(tmpdir)):
chash = chash.decode("ascii").lower()
pyhash = sha256(content).hexdigest().lower()
assert pyhash == chash # Verify expected key (sha256 of content)
assert chash in expected_hashes
expected_hashes.remove(chash)
assert len(expected_hashes) == 0 # All expected hashes have been seen
def test_s3_callbacks(self):
TEST_SITE = BASE_TEST_URL + "/test_pages/simple_a.html"
manager_params, browser_params = self.get_config()
dataset = LocalS3Dataset(manager_params.s3_bucket, manager_params.s3_directory)
manager = task_manager.TaskManager(manager_params, browser_params)
queue = Queue()
def ensure_site_in_s3(success: bool):
# Ensure http table is created
queue.put(
TEST_SITE in dataset.load_table("http_requests").top_level_url.unique()
)
sequence = CommandSequence(
TEST_SITE, reset=True, blocking=True, callback=ensure_site_in_s3
)
sequence.get()
manager.execute_command_sequence(sequence)
manager.close()
assert queue.get()
def test_dump_page_source_valid(self, display_mode):
"""Check that 'dump_page_source' works and source is saved properly."""
# Run the test crawl
manager_params, browser_params = self.get_config(display_mode)
manager = task_manager.TaskManager(manager_params, browser_params)
cs = command_sequence.CommandSequence(url_a)
cs.get(sleep=1)
cs.dump_page_source(suffix="test")
manager.execute_command_sequence(cs)
manager.close()
# Source filename is of the follow structure:
# `sources/<visit_id>-<md5_of_url>(-suffix).html`
# thus for this test we expect `sources/1-<md5_of_test_url>-test.html`.
outfile = os.path.join(str(self.tmpdir), "sources", "1-*-test.html")
source_file = glob.glob(outfile)[0]
with open(source_file, "rb") as f:
actual_source = f.read()
with open("./test_pages/expected_source.html", "rb") as f:
expected_source = f.read()
assert actual_source == expected_source
def test_profile_saved_when_launch_crashes(self):
manager_params, browser_params = self.get_config()
browser_params[0]["proxy"] = True
browser_params[0]["save_content"] = "script"
manager = task_manager.TaskManager(manager_params, browser_params)
manager.get("http://example.com")
# Kill the LevelDBAggregator
# This will cause the proxy launch to crash
manager.ldb_status_queue.put("DIE")
manager.browsers[0]._SPAWN_TIMEOUT = 2 # Have timeout occur quickly
manager.browsers[0]._UNSUCCESSFUL_SPAWN_LIMIT = 2 # Quick timeout
manager.get("example.com") # Cause a selenium crasht
# The browser will fail to launch due to the proxy crashes
try:
manager.get("http://example.com")
except CommandExecutionError:
pass
manager.close()
assert isfile(
join(browser_params[0]["profile_archive_dir"], "profile.tar.gz"))
def test_js_profile_cookies(self):
""" Check that profile cookies set by JS are saved """
# Run the test crawl
manager_params, browser_params = self.get_config()
browser_params[0].cookie_instrument = True
manager = task_manager.TaskManager(manager_params, browser_params)
url = utilities.BASE_TEST_URL + "/js_cookie.html"
cs = command_sequence.CommandSequence(url)
cs.get(sleep=3, timeout=120)
manager.execute_command_sequence(cs)
manager.close()
# Check that the JS cookie we stored is recorded
qry_res = db_utils.query_db(
manager_params.database_name,
("SELECT visit_id, record_type, change_cause, is_http_only, "
"is_host_only, is_session, host, is_secure, name, path, "
"value, same_site FROM javascript_cookies"),
as_tuple=True,
)
assert len(qry_res) == 1 # we store only one cookie
cookies = qry_res[0] # take the first cookie
# compare URL, domain, name, value, origin, path
assert cookies == expected_js_cookie
def test_content_saving(http_params, xpi, server):
""" check that content is saved and hashed correctly """
test_url = utilities.BASE_TEST_URL + "/http_test_page.html"
manager_params, browser_params = http_params()
for browser_param in browser_params:
browser_param.http_instrument = True
browser_param.save_content = True
db = manager_params.data_directory / "crawl-data.sqlite"
structured_storage = SQLiteStorageProvider(db_path=db)
ldb_path = Path(manager_params.data_directory) / "content.ldb"
unstructured_storage = LevelDbProvider(db_path=ldb_path)
manager = task_manager.TaskManager(manager_params, browser_params,
structured_storage,
unstructured_storage)
manager.get(url=test_url, sleep=1)
manager.close()
rows = db_utils.query_db(db, "SELECT * FROM http_responses;")
disk_content = dict()
for row in rows:
if "MAGIC_REDIRECT" in row["url"] or "404" in row["url"]:
continue
path = urlparse(row["url"]).path
with open(os.path.join(BASE_PATH, path[1:]), "rb") as f:
content = f.read()
chash = sha256(content).hexdigest()
assert chash == row["content_hash"]
disk_content[chash] = content
ldb_content = dict()
for chash, content in db_utils.get_content(ldb_path):
chash = chash.decode("ascii")
ldb_content[chash] = content
for k, v in disk_content.items():
assert v == ldb_content[k]
def test_seed_persistance(self):
def test_config_is_set(*args, **kwargs):
driver = kwargs["driver"]
driver.get("about:config")
result = driver.execute_script("""
var prefs = Components
.classes["@mozilla.org/preferences-service;1"]
.getService(Components.interfaces.nsIPrefBranch);
try {
return prefs.getBoolPref("test_pref")
} catch (e) {
return false;
}
""")
assert result
manager_params, browser_params = self.get_test_config(num_browsers=1)
browser_params[0]["seed_tar"] = "."
command_sequences = []
for _ in range(2):
cs = CommandSequence(url="https://example.com", reset=True)
cs.get()
cs.run_custom_function(test_config_is_set)
command_sequences.append(cs)
manager = task_manager.TaskManager(manager_params, browser_params)
for cs in command_sequences:
manager.execute_command_sequence(cs)
manager.close()
query_result = db_utils.query_db(
manager_params["db"],
"SELECT * FROM crawl_history;",
)
assert len(query_result) > 0
for row in query_result:
assert row[
"command_status"] == "ok", f"Command {tuple(row)} was not ok"