def dump_flash_cookies(start_time, visit_id, webdriver, browser_params, manager_params): """ Save newly changed Flash LSOs to database We determine which LSOs to save by the `start_time` timestamp. This timestamp should be taken prior to calling the `get` for which creates these changes. """ # Set up a connection to DataAggregator tab_restart_browser(webdriver) # kills traffic sock = clientsocket() sock.connect(*manager_params['aggregator_address']) # Flash cookies flash_cookies = get_flash_cookies(start_time) for cookie in flash_cookies: query = ("INSERT INTO flash_cookies (crawl_id, visit_id, domain, " "filename, local_path, key, content) VALUES (?,?,?,?,?,?,?)", (browser_params['crawl_id'], visit_id, cookie.domain, cookie.filename, cookie.local_path, cookie.key, cookie.content)) sock.send(query) # Close connection to db sock.close()
def dump_storage_vectors(top_url, start_time, webdriver, browser_params): """ Grab the newly changed items in supported storage vectors """ # Set up a connection to DataAggregator tab_restart_browser( webdriver) # kills traffic so we can cleanly record data sock = clientsocket() sock.connect(*browser_params['aggregator_address']) # Wait for SQLite Checkpointing - never happens when browser open # Flash cookies flash_cookies = get_flash_cookies(start_time) for cookie in flash_cookies: query = ( "INSERT INTO flash_cookies (crawl_id, page_url, domain, filename, local_path, \ key, content) VALUES (?,?,?,?,?,?,?)", (browser_params['crawl_id'], top_url, cookie.domain, cookie.filename, cookie.local_path, cookie.key, cookie.content)) sock.send(query) # Cookies rows = get_cookies(browser_params['profile_path'], start_time) if rows is not None: for row in rows: query = ( "INSERT INTO profile_cookies (crawl_id, page_url, baseDomain, name, value, \ host, path, expiry, accessed, creationTime, isSecure, isHttpOnly) \ VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", (browser_params['crawl_id'], top_url) + row) sock.send(query) # localStorage - TODO this doesn't have a modified time support #rows = get_localStorage(profile_dir, start_time) #if rows is not None: # for row in rows: # query = ("INSERT INTO localStorage (crawl_id, page_url, scope, KEY, value) \ # VALUES (?,?,?,?)",(crawl_id, top_url) + row) # sock.send(query) # Close connection to db sock.close()
def dump_storage_vectors(top_url, start_time, webdriver, browser_params, manager_params): """ Grab the newly changed items in supported storage vectors """ # Set up a connection to DataAggregator tab_restart_browser(webdriver) # kills traffic so we can cleanly record data sock = clientsocket() sock.connect(*manager_params['aggregator_address']) # Wait for SQLite Checkpointing - never happens when browser open # Flash cookies flash_cookies = get_flash_cookies(start_time) for cookie in flash_cookies: query = ("INSERT INTO flash_cookies (crawl_id, page_url, domain, filename, local_path, \ key, content) VALUES (?,?,?,?,?,?,?)", (browser_params['crawl_id'], top_url, cookie.domain, cookie.filename, cookie.local_path, cookie.key, cookie.content)) sock.send(query) # Cookies rows = get_cookies(browser_params['profile_path'], start_time) if rows is not None: for row in rows: query = ("INSERT INTO profile_cookies (crawl_id, page_url, baseDomain, name, value, \ host, path, expiry, accessed, creationTime, isSecure, isHttpOnly) \ VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", (browser_params['crawl_id'], top_url) + row) sock.send(query) # localStorage - TODO this doesn't have a modified time support #rows = get_localStorage(profile_dir, start_time) #if rows is not None: # for row in rows: # query = ("INSERT INTO localStorage (crawl_id, page_url, scope, KEY, value) \ # VALUES (?,?,?,?)",(crawl_id, top_url) + row) # sock.send(query) # Close connection to db sock.close()
def dump_flash_cookies(start_time, visit_id, webdriver, browser_params, manager_params): """ Save newly changed Flash LSOs to database We determine which LSOs to save by the `start_time` timestamp. This timestamp should be taken prior to calling the `get` for which creates these changes. """ # Set up a connection to DataAggregator tab_restart_browser(webdriver) # kills traffic so we can cleanly record data sock = clientsocket() sock.connect(*manager_params['aggregator_address']) # Flash cookies flash_cookies = get_flash_cookies(start_time) for cookie in flash_cookies: query = ("INSERT INTO flash_cookies (crawl_id, visit_id, domain, filename, local_path, \ key, content) VALUES (?,?,?,?,?,?,?)", (browser_params['crawl_id'], visit_id, cookie.domain, cookie.filename, cookie.local_path, cookie.key, cookie.content)) sock.send(query) # Close connection to db sock.close()