def test_update_visit(self):
test_url = "http://example.com"
start_time = strftime("%Y%m%d-%H%M%S")
be = cm.BrowserEvent()
be.event_type = cm.EVENT_NEW_VISIT
vi = cm.VisitInfo()
vi.url = test_url
vi.start_time = start_time
vi.out_db = self.test_db
vi.duration = 0
vi.incomplete = 1
vi.visit_id = dbu.insert_to_db(dbu.DBCmd.ADD_VISIT, be, vi)
vi_read = dbu.get_db_entry(self.test_db, dbu.DBCmd.VISIT_BY_ID,
vi.visit_id)
self.assertEqual(vi.duration, vi_read.duration)
self.assertEqual(vi.incomplete, vi_read.incomplete)
vi.duration = 33
vi.incomplete = 0
dbu.insert_to_db(dbu.DBCmd.UPDATE_VISIT, be, vi)
vi_read = dbu.get_db_entry(self.test_db, dbu.DBCmd.VISIT_BY_ID,
vi.visit_id)
self.assertEqual(vi.duration, vi_read.duration)
self.assertEqual(vi.incomplete, vi_read.incomplete)
def test_update_visit(self):
test_url = "http://example.com"
start_time = strftime("%Y%m%d-%H%M%S")
be = cm.BrowserEvent()
be.event_type = cm.EVENT_NEW_VISIT
vi = cm.VisitInfo()
vi.url = test_url
vi.start_time = start_time
vi.out_db = self.test_db
vi.duration = 0
vi.incomplete = 1
vi.visit_id = dbu.insert_to_db(dbu.DBCmd.ADD_VISIT, be, vi)
vi_read = dbu.get_db_entry(self.test_db, dbu.DBCmd.VISIT_BY_ID,
vi.visit_id)
self.assertEqual(vi.duration, vi_read.duration)
self.assertEqual(vi.incomplete, vi_read.incomplete)
vi.duration = 33
vi.incomplete = 0
dbu.insert_to_db(dbu.DBCmd.UPDATE_VISIT, be, vi)
vi_read = dbu.get_db_entry(self.test_db, dbu.DBCmd.VISIT_BY_ID,
vi.visit_id)
self.assertEqual(vi.duration, vi_read.duration)
self.assertEqual(vi.incomplete, vi_read.incomplete)
def test_lso_db_ops(self):
lso_events = lso.parse_strace_logs(self.vi, test_lso=self.lso_file)
dbu.insert_to_db(dbu.DBCmd.ADD_LSO_ITEMS, lso_events, self.vi)
lso_items_db = dbu.get_db_entry(self.vi.out_db,
dbu.DBCmd.GET_FLASH_COOKIES,
self.vi.visit_id).fetchall()
self.assertEqual(len(lso_items_db), 1)
lso_event = lso_items_db[0]
page_url, domain, filename, local_path, key, content = lso_event[2:8]
self.assertEqual(page_url, self.vi.url)
self.assertEqual(domain, cm.ONLINE_TEST_HOST)
self.assertEqual(filename, TEST_LSO_FILENAME)
self.assertEqual(local_path, TEST_LSO_REL_PATH)
self.assertEqual(key, TEST_LSO_KEYNAME)
self.assertEqual(content, TEST_LSO_VALUE)
def test_lso_db_ops(self):
lso_events = lso.parse_strace_logs(self.vi, test_lso=self.lso_file)
dbu.insert_to_db(dbu.DBCmd.ADD_LSO_ITEMS, lso_events, self.vi)
lso_items_db = dbu.get_db_entry(self.vi.out_db,
dbu.DBCmd.GET_FLASH_COOKIES,
self.vi.visit_id).fetchall()
self.assertEqual(len(lso_items_db), 1)
lso_event = lso_items_db[0]
page_url, domain, filename, local_path, key, content = lso_event[2:8]
self.assertEqual(page_url, self.vi.url)
self.assertEqual(domain, cm.ONLINE_TEST_HOST)
self.assertEqual(filename, TEST_LSO_FILENAME)
self.assertEqual(local_path, TEST_LSO_REL_PATH)
self.assertEqual(key, TEST_LSO_KEYNAME)
self.assertEqual(content, TEST_LSO_VALUE)
def process_crawler_output(ff_log_file, visit_info, flash=1):
# wait until tmp files are merged to db, otherwise we won't
# find the recently added items in db
db_jobs = {}
flash_cookies = []
sleep_until_sqlite_checkpoint(visit_info.profile_dir)
cache_entries = cu.get_ff_cache(visit_info.profile_dir)
db_jobs[dbu.DBCmd.ADD_CACHE_ITEMS] = cache_entries
js_calls, all_calls = ff_log_parser(ff_log_file, visit_info)
db_jobs[dbu.DBCmd.ADD_CANVAS] = js_calls
cookies = cookie.get_ff_cookies(visit_info.profile_dir)
db_jobs[dbu.DBCmd.ADD_COOKIES] = cookies
local_storage_entries = ls.get_ff_local_storage(visit_info.profile_dir)
db_jobs[dbu.DBCmd.ADD_LOCALSTORAGE_ITEMS] = local_storage_entries
indexed_db_entries = indexedDB.gen_ff_indexedDB(visit_info)
db_jobs[dbu.DBCmd.ADD_INDEXEDDB_ITEMS] = indexed_db_entries
if flash:
time.sleep(5) # strace logs may not be available immediately
flash_cookies = lso.parse_strace_logs(visit_info)
db_jobs[dbu.DBCmd.ADD_LSO_ITEMS] = flash_cookies
http_msgs = list(mitm.parse_mitm_dump(visit_info.http_dump))
db_jobs[dbu.DBCmd.ADD_HTTP_HEADERS] = http_msgs
dbu.insert_to_db(dbu.DBCmd.ADD_ALL_VISIT_DATA, db_jobs, visit_info)
return {
"calls": all_calls,
"cookies": cookies,
"flash_cookies": flash_cookies,
"local_storage": local_storage_entries,
"indexed_db": indexed_db_entries,
"cache": cache_entries,
"http_msgs": http_msgs
}
def process_crawler_output(ff_log_file, visit_info, flash=1):
# wait until tmp files are merged to db, otherwise we won't
# find the recently added items in db
db_jobs = {}
flash_cookies = []
sleep_until_sqlite_checkpoint(visit_info.profile_dir)
cache_entries = cu.get_ff_cache(visit_info.profile_dir)
db_jobs[dbu.DBCmd.ADD_CACHE_ITEMS] = cache_entries
js_calls, all_calls = ff_log_parser(ff_log_file, visit_info)
db_jobs[dbu.DBCmd.ADD_CANVAS] = js_calls
cookies = cookie.get_ff_cookies(visit_info.profile_dir)
db_jobs[dbu.DBCmd.ADD_COOKIES] = cookies
local_storage_entries = ls.get_ff_local_storage(visit_info.profile_dir)
db_jobs[dbu.DBCmd.ADD_LOCALSTORAGE_ITEMS] = local_storage_entries
indexed_db_entries = indexedDB.gen_ff_indexedDB(visit_info)
db_jobs[dbu.DBCmd.ADD_INDEXEDDB_ITEMS] = indexed_db_entries
if flash:
time.sleep(5) # strace logs may not be available immediately
flash_cookies = lso.parse_strace_logs(visit_info)
db_jobs[dbu.DBCmd.ADD_LSO_ITEMS] = flash_cookies
http_msgs = list(mitm.parse_mitm_dump(visit_info.http_dump))
db_jobs[dbu.DBCmd.ADD_HTTP_HEADERS] = http_msgs
dbu.insert_to_db(dbu.DBCmd.ADD_ALL_VISIT_DATA, db_jobs, visit_info)
return {"calls": all_calls,
"cookies": cookies,
"flash_cookies": flash_cookies,
"local_storage": local_storage_entries,
"indexed_db": indexed_db_entries,
"cache": cache_entries,
"http_msgs": http_msgs}
def test_r_w_visit_to_db(self):
test_url = "http://example.com"
start_time = strftime("%Y%m%d-%H%M%S")
be = cm.BrowserEvent()
be.event_type = cm.EVENT_NEW_VISIT
vi = cm.VisitInfo()
vi.url = test_url
vi.start_time = start_time
vi.out_db = self.test_db
visit_id = dbu.insert_to_db(dbu.DBCmd.ADD_VISIT, be, vi)
vi_read = dbu.get_db_entry(self.test_db, dbu.DBCmd.VISIT_BY_ID,
visit_id)
self.assertEqual(vi.url, vi_read.url)
self.assertEqual(vi.start_time, vi_read.start_time)
def test_r_w_visit_to_db(self):
test_url = "http://example.com"
start_time = strftime("%Y%m%d-%H%M%S")
be = cm.BrowserEvent()
be.event_type = cm.EVENT_NEW_VISIT
vi = cm.VisitInfo()
vi.url = test_url
vi.start_time = start_time
vi.out_db = self.test_db
visit_id = dbu.insert_to_db(dbu.DBCmd.ADD_VISIT, be, vi)
vi_read = dbu.get_db_entry(self.test_db, dbu.DBCmd.VISIT_BY_ID,
visit_id)
self.assertEqual(vi.url, vi_read.url)
self.assertEqual(vi.start_time, vi_read.start_time)
def test_r_w_canvas_to_db(self):
be = cm.BrowserEvent()
be.event_type = cm.EVENT_TODATAURL
be.url = "http://example.com"
be.js_file = "http://example.com/fp.js"
be.js_line = 5
be.txt = "data:asdsads"
vi = cm.VisitInfo()
vi.visit_id = 1
vi.out_db = self.test_db
canvas_ev_id = dbu.insert_to_db(dbu.DBCmd.ADD_CANVAS, be, vi)
self.assertGreater(canvas_ev_id, 0)
visit_id, data_url_id, event_time, be_db = \
dbu.get_db_entry(self.test_db, dbu.DBCmd.CANVAS_BY_ID,
canvas_ev_id)
self.assertEqual(vi.visit_id, visit_id)
self.assertEqual(data_url_id, 1)
self.assertEqual(be_db.event_type, be.event_type)
self.assertEqual(be_db.url, be.url)
self.assertEqual(be_db.js_file, be.js_file)
self.assertEqual(be_db.js_line, be.js_line)
self.assertEqual(event_time, 0)
def test_r_w_canvas_to_db(self):
be = cm.BrowserEvent()
be.event_type = cm.EVENT_TODATAURL
be.url = "http://example.com"
be.js_file = "http://example.com/fp.js"
be.js_line = 5
be.txt = "data:asdsads"
vi = cm.VisitInfo()
vi.visit_id = 1
vi.out_db = self.test_db
canvas_ev_id = dbu.insert_to_db(dbu.DBCmd.ADD_CANVAS, be, vi)
self.assertGreater(canvas_ev_id, 0)
visit_id, data_url_id, event_time, be_db = \
dbu.get_db_entry(self.test_db, dbu.DBCmd.CANVAS_BY_ID,
canvas_ev_id)
self.assertEqual(vi.visit_id, visit_id)
self.assertEqual(data_url_id, 1)
self.assertEqual(be_db.event_type, be.event_type)
self.assertEqual(be_db.url, be.url)
self.assertEqual(be_db.js_file, be.js_file)
self.assertEqual(be_db.js_line, be.js_line)
self.assertEqual(event_time, 0)
def visit_page(url_tuple,
timeout=cm.HARD_TIME_OUT,
wait_on_site=cm.WAIT_ON_SITE,
pre_crawl_sleep=False,
out_dir=cm.BASE_TMP_DIR,
flash_support=cm.FLASH_ENABLE,
cookie_support=cm.COOKIE_ALLOW_ALL):
driver = None
visit_info = cm.VisitInfo()
try:
visit_info.rank, visit_info.url = url_tuple
except:
# When rank of the page is not provided, we'll use rank=0
visit_info.rank, visit_info.url = 0, url_tuple
visit_info.sys_log = join(
out_dir, "syscall-%s-%s.log" % (visit_info.rank, ut.rand_str()))
visit_info.http_log = join(
out_dir, "http-%s-%s.log" % (visit_info.rank, ut.rand_str()))
visit_info.http_dump = join(
out_dir, "mitm-%s-%s.dmp" % (visit_info.rank, ut.rand_str()))
visit_info.start_time = strftime("%Y%m%d-%H%M%S")
visit_info.out_dir = out_dir
visit_info.out_db = join(visit_info.out_dir, cm.DB_FILENAME)
visit_info.err_log = join(out_dir, "error.log")
visit_info.debug_log = join(out_dir, "debug.log")
be = cm.BrowserEvent()
be.event_type = cm.EVENT_NEW_VISIT
visit_info.ff_log = open_log_file(out_dir, visit_info.url)
if not visit_info.url[:5] in ('data:', 'http:', 'https', 'file:'):
visit_info.url = 'http://' + visit_info.url
try:
visit_info.visit_id = dbu.insert_to_db(dbu.DBCmd.ADD_VISIT, be,
visit_info)
cm.print_debug(
visit_info, "Visiting: %s %s (%s)" %
(visit_info.visit_id, visit_info.url, visit_info.rank))
setup_nspr_logging(visit_info.http_log)
visit_info.vdisplay = start_xvfb()
port, visit_info.mitm_proc = start_mitm_capture(visit_info.http_dump)
driver, visit_info.profile_dir, visit_info.sel_proc =\
get_browser(visit_info.ff_log, port, flash_support, cookie_support)
if flash_support:
visit_info.strace_proc = log_syscalls(visit_info.sel_proc,
visit_info.sys_log)
#############################################################
driver_get(driver, visit_info, cm.SOFT_TIMEOUT) # real visit
#############################################################
time.sleep(wait_on_site)
close_driver(driver, timeout=10)
stop_strace(visit_info.strace_proc)
result_dict = process_crawler_output(visit_info.ff_log, visit_info,
flash_support)
cm.print_debug(
visit_info, "Visit OK: %s %s (%s)" %
(visit_info.visit_id, visit_info.url, visit_info.rank))
visit_info.incomplete = 0
dbu.insert_to_db(dbu.DBCmd.UPDATE_VISIT, be, visit_info)
quit_driver(driver)
stop_xvfb(visit_info.vdisplay)
remove_visit_files(visit_info)
except (cm.TimeExceededError, sel_exceptions.TimeoutException) as texc:
err_str = "Visit to %s(%s) timed out %s" % \
(visit_info.url, visit_info.rank, texc)
cm.print_error(visit_info, err_str)
clean_up(visit_info, driver)
return None
except Exception as exc:
err_str = "Exception visiting %s(%s) %s %s" % \
(visit_info.url, visit_info.rank, exc, traceback.format_exc())
cm.print_error(visit_info, err_str)
clean_up(visit_info, driver)
return None
else:
return result_dict
def visit_page(url_tuple, timeout=cm.HARD_TIME_OUT,
wait_on_site=cm.WAIT_ON_SITE, pre_crawl_sleep=False,
out_dir=cm.BASE_TMP_DIR, flash_support=cm.FLASH_ENABLE,
cookie_support=cm.COOKIE_ALLOW_ALL):
driver = None
visit_info = cm.VisitInfo()
try:
visit_info.rank, visit_info.url = url_tuple
except:
# When rank of the page is not provided, we'll use rank=0
visit_info.rank, visit_info.url = 0, url_tuple
visit_info.sys_log = join(out_dir, "syscall-%s-%s.log" %
(visit_info.rank, ut.rand_str()))
visit_info.http_log = join(out_dir, "http-%s-%s.log" %
(visit_info.rank, ut.rand_str()))
visit_info.http_dump = join(out_dir, "mitm-%s-%s.dmp" %
(visit_info.rank, ut.rand_str()))
visit_info.start_time = strftime("%Y%m%d-%H%M%S")
visit_info.out_dir = out_dir
visit_info.out_db = join(visit_info.out_dir, cm.DB_FILENAME)
visit_info.err_log = join(out_dir, "error.log")
visit_info.debug_log = join(out_dir, "debug.log")
be = cm.BrowserEvent()
be.event_type = cm.EVENT_NEW_VISIT
visit_info.ff_log = open_log_file(out_dir, visit_info.url)
if not visit_info.url[:5] in ('data:', 'http:', 'https', 'file:'):
visit_info.url = 'http://' + visit_info.url
try:
visit_info.visit_id = dbu.insert_to_db(dbu.DBCmd.ADD_VISIT, be,
visit_info)
cm.print_debug(visit_info, "Visiting: %s %s (%s)" %
(visit_info.visit_id, visit_info.url, visit_info.rank))
setup_nspr_logging(visit_info.http_log)
visit_info.vdisplay = start_xvfb()
port, visit_info.mitm_proc = start_mitm_capture(visit_info.http_dump)
driver, visit_info.profile_dir, visit_info.sel_proc =\
get_browser(visit_info.ff_log, port, flash_support, cookie_support)
if flash_support:
visit_info.strace_proc = log_syscalls(visit_info.sel_proc,
visit_info.sys_log)
#############################################################
driver_get(driver, visit_info, cm.SOFT_TIMEOUT) # real visit
#############################################################
time.sleep(wait_on_site)
close_driver(driver, timeout=10)
stop_strace(visit_info.strace_proc)
result_dict = process_crawler_output(visit_info.ff_log, visit_info,
flash_support)
cm.print_debug(visit_info, "Visit OK: %s %s (%s)" %
(visit_info.visit_id, visit_info.url, visit_info.rank))
visit_info.incomplete = 0
dbu.insert_to_db(dbu.DBCmd.UPDATE_VISIT, be, visit_info)
quit_driver(driver)
stop_xvfb(visit_info.vdisplay)
remove_visit_files(visit_info)
except (cm.TimeExceededError, sel_exceptions.TimeoutException) as texc:
err_str = "Visit to %s(%s) timed out %s" % \
(visit_info.url, visit_info.rank, texc)
cm.print_error(visit_info, err_str)
clean_up(visit_info, driver)
return None
except Exception as exc:
err_str = "Exception visiting %s(%s) %s %s" % \
(visit_info.url, visit_info.rank, exc, traceback.format_exc())
cm.print_error(visit_info, err_str)
clean_up(visit_info, driver)
return None
else:
return result_dict
