class ChromeDriver: """ This class allows for using the production Chrome browser with webXray. Requirements are Selenium, Chrome, and ChromeDriver. Pros: Production browser which is largely identical to real-world use By turning headless off it is very similar to a 'real' session By turning headless on the CPU/Mem usage is lower than otherwise Cons: Less testing with webxray than phantomjs, does not handle many paralell instances very well In headless mode prior to 64.0.3254.0, the cookie database does not get created and no cookies are returned """ def __init__(self, ua=False, dnt=False): """ set various global options here """ self.dnt = dnt # set here if you want to use headless mode for Chrome self.headless = True # if you want to get potentially dangerous requests, set # this to true. # false by default for hopefully obvious reasons self.allow_insecure = False # if you have trouble getting chrome to start # change these values manually self.chromedriver_path = None self.chrome_binary_path = None # we want to give our browsers a full minute to try to # download content, but gracefully timeout thereafter self.page_timeout_seconds = 60 # Set ua if we have it, see get_ua_for_headless # for details self.ua = ua # useful for various tasks self.utilities = Utilities() return None # init def create_chromedriver(self): """ Since we have many functions we can perform we consolidate chromedriver code here. """ # set up options object chrome_options = Options() # if we have chrome binary set it up if self.chrome_binary_path: chrome_options.binary_location = self.chrome_binary_path # live dangerously if self.allow_insecure: chrome_options.add_argument('--allow-running-insecure-content') # thank god for this option chrome_options.add_argument('--mute-audio') # if we are headless we also mix up window size a bit if self.headless: chrome_options.add_argument('headless') chrome_options.add_argument('disable-gpu') window_x = random.randrange(1050, 1920) window_y = random.randrange(900, 1080) chrome_options.add_argument('window-size=%sx%s' % (window_x, window_y)) # if we have a ua set it here if self.ua: chrome_options.add_argument('user-agent=' + self.ua) # attempt to start driver, fail gracefull otherwise try: # if we have chromedriver path set it up if self.chromedriver_path: driver = webdriver.Chrome(self.chromedriver_path, desired_capabilities={ 'loggingPrefs': { 'performance': 'ALL' } }, chrome_options=chrome_options) else: driver = webdriver.Chrome( desired_capabilities={ 'loggingPrefs': { 'performance': 'ALL' } }, chrome_options=chrome_options, ) except: return None # allow one minute before we kill it, seperate from browser_wait driver.set_page_load_timeout(self.page_timeout_seconds) return driver # init_headless_driver def get_ua_for_headless(self): """ Using chrome in headless sends a 'Headless' ua string, here we figure out the current ua and remove the 'Headless' part to help with compatability This requires firing up a new browser instance and destroying it, so this should be called once and resused if possible and this is not in __init___ on purpose """ driver = self.create_chromedriver() if driver != None: ua = driver.execute_script('return navigator.userAgent') driver.quit() return ua.replace('Headless', '') else: return None # get_ua_for_headless def get_webxray_scan_data(self, url, browser_wait): """ This function loads the page, monitors network traffic, and returns relevant data/logs. IMPORTANT: headless will miss all cookies in chrome versions < 64.0.3254.0 This uses the chrome performance log to get network traffic details, see following for details: - https://gist.githubusercontent.com/klepikov/5457750/raw/ecedc6dd4eed82f318db91adb923627716fb6b58/test.py - https://sites.google.com/a/chromium.org/chromedriver/logging/performance-log """ driver = self.create_chromedriver() # we can't start Chrome, return error message as result if driver == None: return ({ 'success': False, 'result': 'Unable to launch Chrome instance' }) # allow one minute before we kill it, seperate from browser_wait driver.set_page_load_timeout(60) # start the page load process, return error message if we fail try: if self.dnt: print("DNT on") driver._client.set_header_overrides(headers={'DNT': 1}) driver.get(url) except: driver.quit() return ({'success': False, 'result': 'Unable to load page'}) # if the page has an alert window open it will throw the following exception when trying # to get the current_url: selenium.common.exceptions.UnexpectedAlertPresentException # in theory we should be able to set an option for UNEXPECTED_ALERT_BEHAVIOUR to ACCEPT # but it does not seem to be supported by chromedriver at present # in some cases we can catch the items we need before an alert fires, otherwise # we fail gracefully, but this is a bug that needs resolution try: final_url = driver.current_url title = driver.title page_source = driver.page_source except: # quit the driver or it will never die! driver.quit() return ({ 'success': False, 'result': 'Unable to load page, possible javascript alert issue' }) # handle odd bug where title is a 'webelement' object if not isinstance(title, str): title = None # while the browser may be finished loading the page, scripts may still making # additional requests, so we wait to let all that finish time.sleep(browser_wait) # We use the Chrome performance log get network traffic. Chrome performance log outputs a # number of independent 'message' events which are keyed to a 'requestId'. What we want # to send upstream is a dictionary keyed on the requested url so we do a lot of processing # here to stitch together a coherent log in the format expected by wbxr. # # There are two types of network events we are concerned with: normal http # requests (initiated by Network.requestWillBeSent) and websocket requests (initiated # by Network.webSocketCreated). # # For normal events, we add entries to the 'requests' dictionary which we key to the requested # url. The reason for this is a single requestId may correspond with many urls in # cases where a request results in redirects occuring. However, data from the # Network.loadingFinished event does not include the url, so we key that seperately # in the load_finish_data dict and then attach it later on. Note that if a request to # x.com results in redirects to y.com and z.com, all three will end up sharing # the same loadingFinished data. # # webSocket events are a special case in that they are not strictly HTTP events, but # they do two things we are concerned with: potentially linking a user to # a third-party domain and setting cookies. The url contacted is only exposed in the # first event, Network.webSocketCreated, so we must use the requestId to tie together # subsequent Network.webSocketWillSendHandshakeRequest and # Network.webSocketHandshakeResponseReceived events. We use the dictionary websocket_requests # to keep track of such events, and we then reprocess them to be keyed to the url in our # normal requests log. Note that to keep track of websocket request we use 'websocket' # for content type, and there may be a better way to handle this. # http requests are keyed to URL requests = {} # these events are keyed to requestID load_finish_data = {} websocket_requests = {} # to get page load time we will figure out when the first request and final load finished occured first_start_time = None last_end_time = None # for debuging duplicate_keys = [] # crunch through all the chrome logs here, the main event! for log_item in driver.get_log('performance'): for key, this_log_item in log_item.items(): # we are only interested in message events if key == 'message': # we have to read in this value to get json data log_item_data = json.loads(this_log_item) message_type = log_item_data['message']['method'] ################################ # normal http event processing # ################################ # we have a new http event, create new empty entry keyed to url # and keep track of start time info if message_type == 'Network.requestWillBeSent': this_request = log_item_data['message']['params'][ 'request'] this_url = this_request['url'] # skip if not http(s) if not re.match('^https?://', this_url): continue # the presence of 'redirectResponse' means a prior request is redirected # so we update the status of the original request here and # then continue processing the current request if 'redirectResponse' in log_item_data['message'][ 'params']: redirect_info = log_item_data['message']['params'][ 'redirectResponse'] original_url = redirect_info['url'] # the request was received, mark it requests[original_url].update({'received': True}) # record status code and text requests[original_url].update( {'status': redirect_info['status']}) requests[original_url].update( {'status_text': redirect_info['statusText']}) # try to get response headers, fail gracefully as they are already None try: requests[this_url].update({ 'response_headers': this_response['headersText'] }) except: pass try: requests[this_url].update({ 'content_type': this_response['headers']['Content-Type'] }) except: pass # if a new request we initialize entry if this_url not in requests: requests[this_url] = {} # we use this to get the load_finish_data later on requests[this_url].update({ 'request_id': log_item_data['message']['params']['requestId'] }) # we set received to false to start with requests[this_url].update({'received': False}) # initialze response values to None in case we don't get response requests[this_url].update({'end_time': None}) requests[this_url].update({'status': None}) requests[this_url].update({'status_text': None}) requests[this_url].update( {'response_headers': None}) requests[this_url].update({'content_type': None}) requests[this_url].update({'body_size': None}) requests[this_url].update({'end_time': None}) requests[this_url].update({'user_agent': None}) requests[this_url].update({'referer': None}) # each request has a start_time, we use this to figure out the time it took to download this_start_time = log_item_data['message'][ 'params']['timestamp'] requests[this_url].update( {'start_time': this_start_time}) # update global start time to measure page load time if first_start_time == None or this_start_time < first_start_time: first_start_time = this_start_time # get the request headers requests[this_url].update( {'request_headers': this_request['headers']}) # these can fail, if so, we ignore try: requests[this_url].update({ 'user_agent': this_request['headers']['User-Agent'] }) except: pass try: requests[this_url].update({ 'referer': this_request['headers']['Referer'] }) except: pass # this_url already exists, log else: duplicate_keys.append(this_url) continue # we have received a response to our request, update appropriately if message_type == 'Network.responseReceived': this_response = log_item_data['message']['params'][ 'response'] this_url = this_response['url'] # skip if not http(s) if not re.match('^https?://', this_url): continue # the request was received, mark it requests[this_url].update({'received': True}) # record status code and text requests[this_url].update( {'status': this_response['status']}) requests[this_url].update( {'status_text': this_response['statusText']}) # try to get response headers, fail gracefully as they are already None try: requests[this_url].update({ 'response_headers': this_response['headersText'] }) except: pass try: requests[this_url].update({ 'content_type': this_response['headers']['Content-Type'] }) except: pass # load finish events are keyed to requestId and may apply to many requested urls # so we keep this in a seperate dictionary to be relinked when we're done if message_type == 'Network.loadingFinished': this_request_id = log_item_data['message']['params'][ 'requestId'] this_end_time = log_item_data['message']['params'][ 'timestamp'] # update global end time if last_end_time == None or this_end_time > last_end_time: last_end_time = this_end_time if this_request_id not in load_finish_data: load_finish_data[this_request_id] = {} # size is updated during loading and is shown in logs, but we only want the final size which is here load_finish_data[this_request_id].update({ 'body_size': log_item_data['message']['params'] ['encodedDataLength'] }) # we use this to calculate the total time for all requests load_finish_data[this_request_id].update( {'end_time': this_end_time}) ############################## # webSocket event processing # ############################## # we have a new websocket, create new empty entry keyed to requestId # this will be rekeyed to url # note we ignore timing data for websockets if message_type == 'Network.webSocketCreated': this_url = log_item_data['message']['params']['url'] this_request_id = log_item_data['message']['params'][ 'requestId'] if this_request_id not in websocket_requests: websocket_requests[this_request_id] = {} websocket_requests[this_request_id].update( {'url': this_url}) websocket_requests[this_request_id].update( {'content_type': 'websocket'}) websocket_requests[this_request_id].update( {'received': False}) websocket_requests[this_request_id].update( {'end_time': None}) websocket_requests[this_request_id].update( {'status': None}) websocket_requests[this_request_id].update( {'status_text': None}) websocket_requests[this_request_id].update( {'response_headers': None}) websocket_requests[this_request_id].update( {'body_size': None}) websocket_requests[this_request_id].update( {'end_time': None}) websocket_requests[this_request_id].update( {'start_time': None}) websocket_requests[this_request_id].update( {'user_agent': None}) websocket_requests[this_request_id].update( {'referer': None}) # websocket request made, update relevant fields if message_type == 'Network.webSocketWillSendHandshakeRequest': this_request = log_item_data['message']['params'][ 'request'] this_request_id = log_item_data['message']['params'][ 'requestId'] websocket_requests[this_request_id].update( {'request_headers': this_request['headers']}) websocket_requests[this_request_id].update({ 'user_agent': this_request['headers']['User-Agent'] }) # websocket response received, update relevant fields if message_type == 'Network.webSocketHandshakeResponseReceived': this_response = log_item_data['message']['params'][ 'response'] this_request_id = log_item_data['message']['params'][ 'requestId'] websocket_requests[this_request_id].update( {'received': True}) websocket_requests[this_request_id].update( {'status': this_response['status']}) websocket_requests[this_request_id].update( {'status_text': this_response['statusText']}) websocket_requests[this_request_id].update( {'response_headers': this_response['headersText']}) # end log processing loop # append load finish info to requests for this_url in requests: this_request_id = requests[this_url]['request_id'] if this_request_id in load_finish_data: requests[this_url].update({ 'body_size': load_finish_data[this_request_id]['body_size'] }) # load_time is start time minus end time, # multiplied by 1k to convert to miliseconds load_time = (load_finish_data[this_request_id]['end_time'] - requests[this_url]['start_time']) * 1000 # we shouldn't be getting <=0, but make it null if this happens if load_time <= 0: requests[this_url].update({'load_time': load_time}) else: requests[this_url].update({'load_time': None}) else: requests[this_url].update({'body_size': None}) requests[this_url].update({'load_time': None}) # append websocket data to requests data for item in websocket_requests: requests[websocket_requests[item] ['url']] = websocket_requests[item] # return all the links for later processing all_links = [] try: links = driver.find_elements_by_tag_name('a') for link in links: all_links.append( [link.get_attribute('text'), link.get_attribute('href')]) except: pass # get the page meta description try: meta_desc = driver.find_element_by_xpath( "//meta[@name='description']").get_attribute("content") except: meta_desc = None # get the language of the page try: lang = driver.find_element_by_xpath('/html').get_attribute('lang') except: lang = None # get all the cookies # the selenium get_cookies method does not return third-party cookies # so we open the cookie db directly from the chrome profile # note that in headless mode this does not work in chrome versions # prior to 64.0.3254.0 and no cookies will be returned cookies = [] try: conn = sqlite3.connect( driver.capabilities['chrome']['userDataDir'] + '/Default/Cookies') c = conn.cursor() c.execute( "SELECT name,is_secure,path,host_key,expires_utc,is_httponly,value FROM cookies" ) for cookie in c.fetchall(): cookies.append({ 'name': cookie[0], 'secure': cookie[1], 'path': cookie[2], 'domain': cookie[3], 'expiry': cookie[4], 'httponly': cookie[5], 'value': cookie[6] }) except: return ({ 'success': False, 'result': 'Cookie database not loaded, if this message appears often something is fundamentally wrong and requires attention!' }) if self.headless == True: browser_version = driver.capabilities['version'] + ' [headless]' else: browser_version = driver.capabilities['version'] # other parts of webxray expect this data format, common to all browser drivers used return_dict = { 'browser_type': driver.capabilities['browserName'], 'browser_version': browser_version, 'browser_wait': browser_wait, 'start_url': url, 'final_url': final_url, 'title': title, 'meta_desc': meta_desc, 'lang': lang, 'load_time': int((last_end_time - first_start_time) * 1000), 'processed_requests': requests, 'cookies': cookies, 'all_links': all_links, 'source': page_source } # quit the driver or it will never die! driver.quit() return ({'success': True, 'result': return_dict}) # get_webxray_scan_data def get_page_source_text_title_lang(self, url): """ Given a url, this function attempts to load the page, inject the Readability.js library, execute it, and extract only the text of a given page. Function returns the success value (True/False), page source, the extracted text, and the page title in case of success; in case of failure returns an error message. """ # set up sql_driver for logging errors driver = self.create_chromedriver() # browser hasn't started and error already printed to cli if driver == None: return ({ 'success': False, 'result': 'Unable to launch Chrome instance' }) # starts the page load process, quits driver and returns nothing if we fail try: driver.get(url) except: driver.quit() return ({'success': False, 'result': 'Unable to load page'}) # if we can't get source something is wrong, return None try: page_source = driver.page_source except: driver.quit() return ({'success': False, 'result': 'Unable to read page source'}) # if page title fails put in a null value try: page_title = driver.title except: page_title = None # get the language of the page try: page_lang = driver.find_element_by_xpath('/html').get_attribute( 'lang') except: page_lang = None # inject the locally downloaded copy of readability into the page # and extract the content # # NOTE: you must download readability on your own and place in the # appropriate directory readability_js = open(os.path.dirname(os.path.abspath(__file__)) + '/resources/policyxray/readability.js', 'r', encoding='utf-8').read() try: readabilty_result = json.loads( driver.execute_script(""" %s var readabilityObj = new Readability("%s", document); return JSON.stringify(readabilityObj.parse(), null, 2); """ % (readability_js, url))) driver.quit() except: driver.quit() return ({ 'success': False, 'result': 'Unable to inject Readability.js' }) # readability failure, return None if readabilty_result == None: return ({'success': False, 'result': 'Empty Readability result'}) # readability has HTML formatting, remove it (and extra spaces), return None on failure try: page_text = re.sub( '\s+', ' ', re.sub('&.+?;', ' ', re.sub('<.+?>', ' ', readabilty_result['content']))) except: return ({ 'success': False, 'result': 'Unable to remove HTML from Readability result' }) # the page text is empty, return None if len(page_text) == 0: return ({ 'success': False, 'result': 'Empty result after HTML removal' }) # looks good, return tuple return ({ 'success': True, 'result': { 'page_source': page_source, 'page_text': page_text, 'page_title': page_title, 'page_lang': page_lang } }) # get_page_source_text_title_lang def get_absolute_page_links(self, url): """ Returns all links on the page, changes relative links to absolute links as well. """ # initialize browser instance driver = self.create_chromedriver() # browser hasn't started and error already printed to cli if driver == None: return ({ 'success': False, 'result': 'Unable to launch Chrome instance' }) # allow one minute before we kill it driver.set_page_load_timeout(60) # starts the page load process, quits driver and returns nothing if we fail try: driver.get(url) except: driver.quit() return ({'success': False, 'result': 'Unable to load page'}) # page has now been loaded, we process all the links all_links = set() # either there are no links or something has gone wrong try: links = driver.find_elements_by_tag_name('a') except: driver.quit() return ({'success': False, 'result': 'Unable to extract links'}) # process the links for link in links: # to cope with: # selenium.common.exceptions.StaleElementReferenceException: Message: stale element reference: element is not attached to the page document try: this_link = link.get_attribute('href') this_link_text = re.sub( '\s+', ' ', re.sub('[\\n|\\r|\\t]', ' ', link.get_attribute('text').strip())) except: continue # sometimes can be empty if this_link == None: continue if len(this_link) == 0: continue # add in the absolute url from the link to our set all_links.add((this_link_text, self.utilities.get_absolute_url_from_page_link( url, this_link))) driver.quit() return ({'success': True, 'result': all_links})
class OutputStore: """ This class receives data from the browser, processes it, and stores it in the db """ def __init__(self, db_engine, db_name): self.db_engine = db_engine self.db_name = db_name self.utilities = Utilities() self.url_parser = ParseURL() # init def store(self, url, browser_output, store_source=False, store_1p=True, get_file_hashes=False, hash_3p_only=False): """ this is the primary function of this class, it takes the url of the given page and the request and cookie data generated by the browser data is cleaned up with some minor analysis (eg file types) and stored for later in-depth analysis. there is an option to store first party requests as well as third, turned on by default to save disk space turn off store_1p there is also an option to get file hashes, this introduces serious overhead and is turned off by default """ # open up a sql connection if self.db_engine == 'mysql': from webxray.MySQLDriver import MySQLDriver sql_driver = MySQLDriver(self.db_name) elif self.db_engine == 'sqlite': from webxray.SQLiteDriver import SQLiteDriver sql_driver = SQLiteDriver(self.db_name) elif self.db_engine == 'postgres': from webxray.PostgreSQLDriver import PostgreSQLDriver sql_driver = PostgreSQLDriver(self.db_name) else: print('INVALED DB ENGINE FOR %s, QUITTING!' % db_engine) exit() # get the ip, fqdn, domain, pubsuffix, and tld # we need the domain to figure out if cookies/elements are third-party origin_ip_fqdn_domain_pubsuffix_tld = self.url_parser.get_ip_fqdn_domain_pubsuffix_tld(url) # if we can't get page domain info we fail gracefully if origin_ip_fqdn_domain_pubsuffix_tld is None: sql_driver.log_error(url, 'Could not parse TLD for %s' % url) return False origin_ip = origin_ip_fqdn_domain_pubsuffix_tld[0] origin_fqdn = origin_ip_fqdn_domain_pubsuffix_tld[1] origin_domain = origin_ip_fqdn_domain_pubsuffix_tld[2] origin_pubsuffix = origin_ip_fqdn_domain_pubsuffix_tld[3] origin_tld = origin_ip_fqdn_domain_pubsuffix_tld[4] # sql_driver.add_domain both stores the new domain and returns its db row id # if it is already in db just return the existing id page_domain_id = sql_driver.add_domain(origin_ip, origin_fqdn, origin_domain, origin_pubsuffix, origin_tld) # figure out the privacy policy url and text, starts null priv_policy_url = None priv_policy_url_text = None # read in our list of privacy link terms from the json file in webxray/resources/policyxray privacy_policy_term_list = self.utilities.get_privacy_policy_term_list() # we reverse links return from browser to check footer links first as that is where policy links tend to be all_links = browser_output['all_links'] all_links.reverse() # if we have links search for privacy policy if len(all_links) > 0: # links are tuple for link_text,link_url in all_links: # makes sure we have text, skip links without if link_text: # need lower for string matching link_text = link_text.lower().strip() # not a link we can use if 'javascript' in link_text: continue # see if the link_text is in our term list if link_text in privacy_policy_term_list: # if the link_url is relative this will convert to absolute priv_policy_url = self.utilities.get_absolute_url_from_page_link(url,link_url) priv_policy_url_text = link_text break # if the final page is https (often after a redirect), mark it appropriately if browser_output['final_url'][:5] == 'https': page_is_ssl = True else: page_is_ssl = False if store_source: # handles issue where postgres will crash on inserting null character source = browser_output['source'].replace('\x00',' ') else: source = None # add page page_id = sql_driver.add_page( browser_output['browser_type'], browser_output['browser_version'], browser_output['browser_wait'], browser_output['title'], browser_output['meta_desc'], url, browser_output['final_url'], priv_policy_url, priv_policy_url_text, page_is_ssl, source, browser_output['load_time'], page_domain_id ) # store cookies for cookie in browser_output['cookies']: # get the ip, fqdn, domain, pubsuffix, and tld # we need the domain to figure out if cookies/elements are third-party # note: # url_parser fails on non-http, we should fix this, right now a lame hack is to prepend http:// cookie_ip_fqdn_domain_pubsuffix_tld = self.url_parser.get_ip_fqdn_domain_pubsuffix_tld('http://'+cookie['domain']) # something went wrong, log and fail gracefully if cookie_ip_fqdn_domain_pubsuffix_tld is None: sql_driver.log_error(url, 'Error parsing cookie with domain: '+cookie['domain']) continue # otherwise, everything went fine cookie_ip = cookie_ip_fqdn_domain_pubsuffix_tld[0] cookie_fqdn = cookie_ip_fqdn_domain_pubsuffix_tld[1] cookie_domain = cookie_ip_fqdn_domain_pubsuffix_tld[2] cookie_pubsuffix = cookie_ip_fqdn_domain_pubsuffix_tld[3] cookie_tld = cookie_ip_fqdn_domain_pubsuffix_tld[4] # mark third-party cookies if origin_domain != cookie_domain: is_3p_cookie = True else: is_3p_cookie = False # this is a first party cookie, see if we want to store it if is_3p_cookie is False and store_1p is False: continue # sql_driver.add_domain both stores the new domain and returns its id cookie_domain_id = sql_driver.add_domain(cookie_ip, cookie_fqdn, cookie_domain, cookie_pubsuffix, cookie_tld) # name and domain are required, so if they fail we just continue try: name = cookie['name'] except: continue try: domain = cookie_domain except: continue # these are optional, fill with null values if fail try: secure = cookie['secure'] except: secure = None try: path = cookie['path'] except: path = None try: httponly = cookie['httponly'] except: httponly = None try: expiry = cookie['expiry'] except: expiry = None try: value = cookie['value'] except: value = None # all done with this cookie sql_driver.add_cookie( page_id, name, secure, path, domain, httponly, expiry, value, is_3p_cookie, cookie_domain_id ) # process requests now for request in browser_output['processed_requests']: # if the request starts with the following we can't parse anyway, so skip if re.match('^(data|about|chrome|blob).+', request): continue # get the ip, fqdn, domain, pubsuffix, and tld # we need the domain to figure out if cookies/elements are third-party element_ip_fqdn_domain_pubsuffix_tld = self.url_parser.get_ip_fqdn_domain_pubsuffix_tld(request) # problem with this request, log and fail gracefully if element_ip_fqdn_domain_pubsuffix_tld is None: sql_driver.log_error(url, 'Error parsing element request: '+request) continue element_ip = element_ip_fqdn_domain_pubsuffix_tld[0] element_fqdn = element_ip_fqdn_domain_pubsuffix_tld[1] element_domain = element_ip_fqdn_domain_pubsuffix_tld[2] element_pubsuffix = element_ip_fqdn_domain_pubsuffix_tld[3] element_tld = element_ip_fqdn_domain_pubsuffix_tld[4] # sql_driver.add_domain both stores the new domain and returns its db row id element_domain_id = sql_driver.add_domain(element_ip, element_fqdn, element_domain, element_pubsuffix, element_tld) # mark third-party elements based on domain if origin_domain != element_domain: is_3p_element = True else: is_3p_element = False # if we are not storing 1p elements continue if is_3p_element is False and store_1p is False: continue if request[:5] == 'https' or request[:3] == 'wss': element_is_ssl = True else: element_is_ssl = False try: received = browser_output['processed_requests'][request]['received'] except: received = None # get domain of referer and determine if page leaked by referer try: referer = browser_output['processed_requests'][request]['referer'] except: referer = None if referer and len(referer) != 0: referer_ip_fqdn_domain_pubsuffix_tld = self.url_parser.get_ip_fqdn_domain_pubsuffix_tld(referer) if referer_ip_fqdn_domain_pubsuffix_tld: if referer_ip_fqdn_domain_pubsuffix_tld[2] == origin_domain: page_domain_in_referer = True else: page_domain_in_referer = False else: page_domain_in_referer = None sql_driver.log_error(url, 'Error parsing referer header: '+referer) else: page_domain_in_referer = None try: start_time_offset = browser_output['processed_requests'][request]['start_time_offset'] except: start_time_offset = None try: load_time = browser_output['processed_requests'][request]['load_time'] except: load_time = None try: status = browser_output['processed_requests'][request]['status'] except: status = None try: status_text = browser_output['processed_requests'][request]['status_text'] except: status_text = None try: content_type = browser_output['processed_requests'][request]['content_type'] except: content_type = None try: body_size = browser_output['processed_requests'][request]['body_size'] except: body_size = None try: request_headers = str(browser_output['processed_requests'][request]['request_headers']) except: request_headers = None try: response_headers = str(browser_output['processed_requests'][request]['response_headers']) except: response_headers = None # consider anything before the "?" to be the element_url try: element_url = re.search('^(.+?)\?.+$', request).group(1) except: element_url = request # consider anything after the "?" to be the args try: element_args = re.search('^.+(\?.+)$', request).group(1) # start url args except: element_args = None # attempt to parse off the extension try: element_extension = re.search('\.([0-9A-Za-z]+)$', element_url).group(1).lower() except: element_extension = None # lists of common extensions, can be expanded image_extensions = ['png', 'jpg', 'jpgx', 'jpeg', 'gif', 'svg', 'bmp', 'tif', 'tiff', 'webp', 'srf'] script_extensions = ['js', 'javascript'] data_extensions = ['json', 'jsonp', 'xml'] font_extentions = ['woff', 'ttf', 'otf'] static_extentions = ['html', 'htm', 'shtml'] dynamic_extentions = ['php', 'asp', 'jsp', 'aspx', 'ashx', 'pl', 'cgi', 'fcgi'] # figure out what type of element it is if element_extension in image_extensions: element_type = 'image' elif element_extension in script_extensions: element_type = 'javascript' elif element_extension in data_extensions: element_type = 'data_structured' elif element_extension == 'css': element_type = 'style_sheet' elif element_extension in font_extentions: element_type = 'font' elif element_extension in static_extentions: element_type = 'page_static' elif element_extension == dynamic_extentions: element_type = 'page_dynamic' elif element_extension == 'swf' or element_extension == 'fla': element_type = 'Shockwave Flash' else: element_type = None # file hashing has non-trivial overhead and off by default # # what this does is uses the same ua/referer as the actual request # so we are just replaying the last one to get similar response # note that we aren't sending the same cookies so that could be an issue # otherwise it is equivalent to a page refresh in theory # option to hash only 3p elements observed here if (get_file_hashes and hash_3p_only and is_3p_element) or (get_file_hashes and hash_3p_only == False): replay_element_request = urllib.request.Request( request, headers = { 'User-Agent' : browser_output['processed_requests'][request]['user_agent'], 'Referer' : referer, 'Accept' : '*/*' } ) try: file_md5 = hashlib.md5(urllib.request.urlopen(replay_element_request,timeout=10).read()).hexdigest() except: file_md5 = None else: file_md5 = None # final tasks is to truncate the request if it is # over 2k characters as it is likely # binary data and may cause problems inserting # into TEXT fields in database # # TODO: # better handle binary data in general if len(request) >= 2000: request = request[:2000] if len(element_url) >= 2000: element_url = element_url[:2000] # store request sql_driver.add_element( page_id, request, element_url, is_3p_element, element_is_ssl, received, referer, page_domain_in_referer, start_time_offset, load_time, status, status_text, content_type, body_size, request_headers, response_headers, file_md5, element_extension, element_type, element_args, element_domain_id ) # close db connection sql_driver.close() return True
class OutputStore: """ This class receives data from the browser, processes it, and stores it in the db """ def __init__(self, db_engine, db_name): self.db_engine = db_engine self.db_name = db_name self.utilities = Utilities() self.url_parser = ParseURL() # init def store(self, url, browser_output, store_source=False, store_1p=True, get_file_hashes=False, hash_3p_only=False): """ this is the primary function of this class, it takes the url of the given page and the request and cookie data generated by the browser data is cleaned up with some minor analysis (eg file types) and stored for later in-depth analysis. there is an option to store first party requests as well as third, turned on by default to save disk space turn off store_1p there is also an option to get file hashes, this introduces serious overhead and is turned off by default """ # open up a sql connection if self.db_engine == 'mysql': from webxray.MySQLDriver import MySQLDriver sql_driver = MySQLDriver(self.db_name) elif self.db_engine == 'sqlite': from webxray.SQLiteDriver import SQLiteDriver sql_driver = SQLiteDriver(self.db_name) elif self.db_engine == 'postgres': from webxray.PostgreSQLDriver import PostgreSQLDriver sql_driver = PostgreSQLDriver(self.db_name) else: print('INVALED DB ENGINE FOR %s, QUITTING!' % db_engine) exit() # get the ip, fqdn, domain, pubsuffix, and tld # we need the domain to figure out if cookies/elements are third-party origin_ip_fqdn_domain_pubsuffix_tld = self.url_parser.get_ip_fqdn_domain_pubsuffix_tld( url) # if we can't get page domain info we fail gracefully if origin_ip_fqdn_domain_pubsuffix_tld is None: sql_driver.log_error(url, 'Could not parse TLD for %s' % url) return False origin_ip = origin_ip_fqdn_domain_pubsuffix_tld[0] origin_fqdn = origin_ip_fqdn_domain_pubsuffix_tld[1] origin_domain = origin_ip_fqdn_domain_pubsuffix_tld[2] origin_pubsuffix = origin_ip_fqdn_domain_pubsuffix_tld[3] origin_tld = origin_ip_fqdn_domain_pubsuffix_tld[4] # sql_driver.add_domain both stores the new domain and returns its db row id # if it is already in db just return the existing id page_domain_id = sql_driver.add_domain(origin_ip, origin_fqdn, origin_domain, origin_pubsuffix, origin_tld) # figure out the privacy policy url and text, starts null priv_policy_url = None priv_policy_url_text = None # read in our list of privacy link terms from the json file in webxray/resources/policyxray privacy_policy_term_list = self.utilities.get_privacy_policy_term_list( ) # we reverse links return from browser to check footer links first as that is where policy links tend to be all_links = browser_output['all_links'] all_links.reverse() # if we have links search for privacy policy if len(all_links) > 0: # links are tuple for link_text, link_url in all_links: # makes sure we have text, skip links without if link_text: # need lower for string matching link_text = link_text.lower().strip() # not a link we can use if 'javascript' in link_text: continue # see if the link_text is in our term list if link_text in privacy_policy_term_list: # if the link_url is relative this will convert to absolute priv_policy_url = self.utilities.get_absolute_url_from_page_link( url, link_url) priv_policy_url_text = link_text break # if the final page is https (often after a redirect), mark it appropriately if browser_output['final_url'][:5] == 'https': page_is_ssl = True else: page_is_ssl = False if store_source: # handles issue where postgres will crash on inserting null character source = browser_output['source'].replace('\x00', ' ') else: source = None # add page page_id = sql_driver.add_page( browser_output['browser_type'], browser_output['browser_version'], browser_output['browser_wait'], browser_output['title'], browser_output['meta_desc'], url, browser_output['final_url'], priv_policy_url, priv_policy_url_text, page_is_ssl, source, browser_output['load_time'], page_domain_id) # store cookies for cookie in browser_output['cookies']: # get the ip, fqdn, domain, pubsuffix, and tld # we need the domain to figure out if cookies/elements are third-party # note: # url_parser fails on non-http, we should fix this, right now a lame hack is to prepend http:// cookie_ip_fqdn_domain_pubsuffix_tld = self.url_parser.get_ip_fqdn_domain_pubsuffix_tld( 'http://' + cookie['domain']) # something went wrong, log and fail gracefully if cookie_ip_fqdn_domain_pubsuffix_tld is None: sql_driver.log_error( url, 'Error parsing cookie with domain: ' + cookie['domain']) continue # otherwise, everything went fine cookie_ip = cookie_ip_fqdn_domain_pubsuffix_tld[0] cookie_fqdn = cookie_ip_fqdn_domain_pubsuffix_tld[1] cookie_domain = cookie_ip_fqdn_domain_pubsuffix_tld[2] cookie_pubsuffix = cookie_ip_fqdn_domain_pubsuffix_tld[3] cookie_tld = cookie_ip_fqdn_domain_pubsuffix_tld[4] # mark third-party cookies if origin_domain != cookie_domain: is_3p_cookie = True else: is_3p_cookie = False # this is a first party cookie, see if we want to store it if is_3p_cookie is False and store_1p is False: continue # sql_driver.add_domain both stores the new domain and returns its id cookie_domain_id = sql_driver.add_domain(cookie_ip, cookie_fqdn, cookie_domain, cookie_pubsuffix, cookie_tld) # name and domain are required, so if they fail we just continue try: name = cookie['name'] except: continue try: domain = cookie_domain except: continue # these are optional, fill with null values if fail try: secure = cookie['secure'] except: secure = None try: path = cookie['path'] except: path = None try: httponly = cookie['httponly'] except: httponly = None try: expiry = cookie['expiry'] except: expiry = None try: value = cookie['value'] except: value = None # all done with this cookie sql_driver.add_cookie(page_id, name, secure, path, domain, httponly, expiry, value, is_3p_cookie, cookie_domain_id) # process requests now for request in browser_output['processed_requests']: # if the request starts with the following we can't parse anyway, so skip if re.match('^(data|about|chrome|blob).+', request): continue # get the ip, fqdn, domain, pubsuffix, and tld # we need the domain to figure out if cookies/elements are third-party element_ip_fqdn_domain_pubsuffix_tld = self.url_parser.get_ip_fqdn_domain_pubsuffix_tld( request) # problem with this request, log and fail gracefully if element_ip_fqdn_domain_pubsuffix_tld is None: sql_driver.log_error( url, 'Error parsing element request: ' + request) continue element_ip = element_ip_fqdn_domain_pubsuffix_tld[0] element_fqdn = element_ip_fqdn_domain_pubsuffix_tld[1] element_domain = element_ip_fqdn_domain_pubsuffix_tld[2] element_pubsuffix = element_ip_fqdn_domain_pubsuffix_tld[3] element_tld = element_ip_fqdn_domain_pubsuffix_tld[4] # sql_driver.add_domain both stores the new domain and returns its db row id element_domain_id = sql_driver.add_domain(element_ip, element_fqdn, element_domain, element_pubsuffix, element_tld) # mark third-party elements based on domain if origin_domain != element_domain: is_3p_element = True else: is_3p_element = False # if we are not storing 1p elements continue if is_3p_element is False and store_1p is False: continue if request[:5] == 'https': element_is_ssl = True else: element_is_ssl = False try: received = browser_output['processed_requests'][request][ 'received'] except: received = None # get domain of referer and determine if page leaked by referer try: referer = browser_output['processed_requests'][request][ 'referer'] except: referer = None if referer and len(referer) != 0: referer_ip_fqdn_domain_pubsuffix_tld = self.url_parser.get_ip_fqdn_domain_pubsuffix_tld( referer) if referer_ip_fqdn_domain_pubsuffix_tld: if referer_ip_fqdn_domain_pubsuffix_tld[ 2] == origin_domain: page_domain_in_referer = True else: page_domain_in_referer = False else: page_domain_in_referer = None sql_driver.log_error( url, 'Error parsing referer header: ' + referer) else: page_domain_in_referer = None try: start_time_offset = browser_output['processed_requests'][ request]['start_time_offset'] except: start_time_offset = None try: load_time = browser_output['processed_requests'][request][ 'load_time'] except: load_time = None try: status = browser_output['processed_requests'][request][ 'status'] except: status = None try: status_text = browser_output['processed_requests'][request][ 'status_text'] except: status_text = None try: content_type = browser_output['processed_requests'][request][ 'content_type'] except: content_type = None try: body_size = browser_output['processed_requests'][request][ 'body_size'] except: body_size = None try: request_headers = str(browser_output['processed_requests'] [request]['request_headers']) except: request_headers = None try: response_headers = str(browser_output['processed_requests'] [request]['response_headers']) except: response_headers = None # consider anything before the "?" to be the element_url try: element_url = re.search('^(.+?)\?.+$', request).group(1) except: element_url = request # consider anything after the "?" to be the args try: element_args = re.search('^.+(\?.+)$', request).group(1) # start url args except: element_args = None # attempt to parse off the extension try: element_extension = re.search('\.([0-9A-Za-z]+)$', element_url).group(1).lower() except: element_extension = None # lists of common extensions, can be expanded image_extensions = [ 'png', 'jpg', 'jpgx', 'jpeg', 'gif', 'svg', 'bmp', 'tif', 'tiff', 'webp', 'srf' ] script_extensions = ['js', 'javascript'] data_extensions = ['json', 'jsonp', 'xml'] font_extentions = ['woff', 'ttf', 'otf'] static_extentions = ['html', 'htm', 'shtml'] dynamic_extentions = [ 'php', 'asp', 'jsp', 'aspx', 'ashx', 'pl', 'cgi', 'fcgi' ] # figure out what type of element it is if element_extension in image_extensions: element_type = 'image' elif element_extension in script_extensions: element_type = 'javascript' elif element_extension in data_extensions: element_type = 'data_structured' elif element_extension == 'css': element_type = 'style_sheet' elif element_extension in font_extentions: element_type = 'font' elif element_extension in static_extentions: element_type = 'page_static' elif element_extension == dynamic_extentions: element_type = 'page_dynamic' elif element_extension == 'swf' or element_extension == 'fla': element_type = 'Shockwave Flash' else: element_type = None # file hashing has non-trivial overhead and off by default # # what this does is uses the same ua/referer as the actual request # so we are just replaying the last one to get similar response # note that we aren't sending the same cookies so that could be an issue # otherwise it is equivalent to a page refresh in theory # option to hash only 3p elements observed here if (get_file_hashes and hash_3p_only and is_3p_element) or (get_file_hashes and hash_3p_only == False): replay_element_request = urllib.request.Request( request, headers={ 'User-Agent': browser_output['processed_requests'][request] ['user_agent'], 'Referer': referer, 'Accept': '*/*' }) try: file_md5 = hashlib.md5( urllib.request.urlopen(replay_element_request, timeout=10).read()).hexdigest() except: file_md5 = None else: file_md5 = None # store request sql_driver.add_element( page_id, request, element_url, is_3p_element, element_is_ssl, received, referer, page_domain_in_referer, start_time_offset, load_time, status, status_text, content_type, body_size, request_headers, response_headers, file_md5, element_extension, element_type, element_args, element_domain_id) # close db connection sql_driver.close() return True
class ChromeDriver: """ This class allows for using the production Chrome browser with webXray. Requirements are Selenium, Chrome, and ChromeDriver. Pros: Production browser which is largely identical to real-world use By turning headless off it is very similar to a 'real' session By turning headless on the CPU/Mem usage is lower than otherwise Cons: Less testing with webxray than phantomjs, does not handle many paralell instances very well In headless mode prior to 64.0.3254.0, the cookie database does not get created and no cookies are returned """ def __init__(self,ua=False): """ set various global options here """ # set here if you want to use headless mode for Chrome self.headless = True # if you want to get potentially dangerous requests, set # this to true. # false by default for hopefully obvious reasons self.allow_insecure = False # if you have trouble getting chrome to start # change these values manually self.chromedriver_path = None self.chrome_binary_path = None # we want to give our browsers a full minute to try to # download content, but gracefully timeout thereafter self.page_timeout_seconds = 60 # Set ua if we have it, see get_ua_for_headless # for details self.ua = ua # useful for various tasks self.utilities = Utilities() return None # init def create_chromedriver(self): """ Since we have many functions we can perform we consolidate chromedriver code here. """ # set up options object chrome_options = Options() # if we have chrome binary set it up if self.chrome_binary_path: chrome_options.binary_location = self.chrome_binary_path # live dangerously if self.allow_insecure: chrome_options.add_argument('--allow-running-insecure-content') # thank god for this option chrome_options.add_argument('--mute-audio') # if we are headless we also mix up window size a bit if self.headless: chrome_options.add_argument('headless') chrome_options.add_argument('disable-gpu') window_x = random.randrange(1050,1920) window_y = random.randrange(900,1080) chrome_options.add_argument('window-size=%sx%s' % (window_x,window_y)) # if we have a ua set it here if self.ua: chrome_options.add_argument('user-agent='+self.ua) # 'desired_capabilities' is essentially a second way to set options for chrome # we set loggingPrefs to turn on the performance log which we need to analyze network traffic # see: https://sites.google.com/a/chromium.org/chromedriver/logging/performance-log # pageLoadStrategy is set to 'none' to make sure we don't get stuck on pages that never finish loading # once 'eager' is implemented in chromedriver that may be preferable # see: https://w3c.github.io/webdriver/#dfn-table-of-page-load-strategies chrome_capabilities = { 'loggingPrefs': {'performance': 'ALL'}, 'pageLoadStrategy': 'none' } # attempt to start driver, fail gracefull otherwise try: # if we have chromedriver path set it up if self.chromedriver_path: driver = webdriver.Chrome( self.chromedriver_path, desired_capabilities=chrome_capabilities, chrome_options=chrome_options ) else: driver = webdriver.Chrome( desired_capabilities=chrome_capabilities, chrome_options=chrome_options ) except: return None # allow one minute before we kill it, seperate from browser_wait driver.set_page_load_timeout(self.page_timeout_seconds) return driver # init_headless_driver def get_ua_for_headless(self): """ Using chrome in headless sends a 'Headless' ua string, here we figure out the current ua and remove the 'Headless' part to help with compatability This requires firing up a new browser instance and destroying it, so this should be called once and resused if possible and this is not in __init___ on purpose """ driver = self.create_chromedriver() if driver != None: ua = driver.execute_script('return navigator.userAgent') driver.quit() return ua.replace('Headless','') else: return None # get_ua_for_headless def get_webxray_scan_data(self, url, browser_wait): """ This function loads the page, monitors network traffic, and returns relevant data/logs. IMPORTANT: headless will miss all cookies in chrome versions < 64.0.3254.0 This uses the chrome performance log to get network traffic details, see following for details: - https://gist.githubusercontent.com/klepikov/5457750/raw/ecedc6dd4eed82f318db91adb923627716fb6b58/test.py - https://sites.google.com/a/chromium.org/chromedriver/logging/performance-log """ driver = self.create_chromedriver() # we can't start Chrome, return error message as result if driver == None: return({ 'success': False, 'result': 'Unable to launch Chrome instance' }) # allow one minute before we kill it, seperate from browser_wait driver.set_page_load_timeout(self.page_timeout_seconds) # start the page load process, return error message if we fail try: driver.get(url) except Exception as e: driver.quit() return({ 'success': False, 'result': 'Unable to load page: '+str(e).replace('\n', ' ') }) # while the browser may be finished loading the page, scripts may still making # additional requests, so we wait to let all that finish time.sleep(browser_wait) # if the page has an alert window open it will throw the following exception when trying # to get the current_url: selenium.common.exceptions.UnexpectedAlertPresentException # in theory we should be able to set an option for UNEXPECTED_ALERT_BEHAVIOUR to ACCEPT # but it does not seem to be supported by chromedriver at present # in some cases we can catch the items we need before an alert fires, otherwise # we fail gracefully, but this is a bug that needs resolution try: final_url = driver.current_url title = driver.title page_source = driver.page_source except: # quit the driver or it will never die! driver.quit() return({ 'success': False, 'result': 'Unable to load page, possible javascript alert issue' }) # handle odd bug where title is a 'webelement' object if not isinstance(title, str): title = None # We use the Chrome performance log get network traffic. Chrome performance log outputs a # number of independent 'message' events which are keyed to a 'requestId'. What we want # to send upstream is a dictionary keyed on the requested url so we do a lot of processing # here to stitch together a coherent log in the format expected by wbxr. # # There are two types of network events we are concerned with: normal http # requests (initiated by Network.requestWillBeSent) and websocket requests (initiated # by Network.webSocketCreated). # # For normal events, we add entries to the 'requests' dictionary which we key to the requested # url. The reason for this is a single requestId may correspond with many urls in # cases where a request results in redirects occuring. However, data from the # Network.loadingFinished event does not include the url, so we key that seperately # in the load_finish_data dict and then attach it later on. Note that if a request to # x.com results in redirects to y.com and z.com, all three will end up sharing # the same loadingFinished data. # # webSocket events are a special case in that they are not strictly HTTP events, but # they do two things we are concerned with: potentially linking a user to # a third-party domain and setting cookies. The url contacted is only exposed in the # first event, Network.webSocketCreated, so we must use the requestId to tie together # subsequent Network.webSocketWillSendHandshakeRequest and # Network.webSocketHandshakeResponseReceived events. We use the dictionary websocket_requests # to keep track of such events, and we then reprocess them to be keyed to the url in our # normal requests log. Note that to keep track of websocket request we use 'websocket' # for content type, and there may be a better way to handle this. # http requests are keyed to URL requests = {} # these events are keyed to requestID load_finish_data = {} websocket_requests = {} # to get page load time we will figure out when the first request and final load finished occured first_start_time = None last_end_time = None # for debuging duplicate_keys = [] # crunch through all the chrome logs here, the main event! for log_item in driver.get_log('performance'): for key, this_log_item in log_item.items(): # we are only interested in message events if key == 'message': # we have to read in this value to get json data log_item_data = json.loads(this_log_item) message_type = log_item_data['message']['method'] ################################ # normal http event processing # ################################ # we have a new http event, create new empty entry keyed to url # and keep track of start time info if message_type == 'Network.requestWillBeSent': this_request = log_item_data['message']['params']['request'] this_url = this_request['url'] # skip if not http(s) if not re.match('^https?://', this_url): continue # the presence of 'redirectResponse' means a prior request is redirected # so we update the status of the original request here and # then continue processing the new request if 'redirectResponse' in log_item_data['message']['params']: redirect_info = log_item_data['message']['params']['redirectResponse'] original_url = redirect_info['url'] # the request was received, mark it requests[original_url].update({'received': True}) # record status code and text requests[original_url].update({'status': redirect_info['status']}) requests[original_url].update({'status_text': redirect_info['statusText']}) # try to get response headers, fail gracefully as they are already None try: requests[this_url].update({'response_headers':this_response['headersText']}) except: pass try: requests[this_url].update({'content_type':this_response['headers']['Content-Type']}) except: pass # if a new request we initialize entry if this_url not in requests: requests[this_url] = {} # we use this to get the load_finish_data later on requests[this_url].update({'request_id': log_item_data['message']['params']['requestId']}) # we set received to false to start with requests[this_url].update({'received': False}) # initialze response values to None in case we don't get response requests[this_url].update({'end_time': None}) requests[this_url].update({'status': None}) requests[this_url].update({'status_text': None}) requests[this_url].update({'response_headers': None}) requests[this_url].update({'content_type': None}) requests[this_url].update({'body_size': None}) requests[this_url].update({'end_time': None}) requests[this_url].update({'user_agent': None}) requests[this_url].update({'referer': None}) # each request has a start_time, we use this to figure out the time it took to download this_start_time = log_item_data['message']['params']['timestamp'] requests[this_url].update({'start_time':this_start_time}) # update global start time to measure page load time if first_start_time == None or this_start_time < first_start_time: first_start_time = this_start_time # get the request headers requests[this_url].update({'request_headers':this_request['headers']}) # these can fail, if so, we ignore try: requests[this_url].update({'user_agent':this_request['headers']['User-Agent']}) except: pass try: requests[this_url].update({'referer':this_request['headers']['Referer']}) except: pass # this_url already exists, log else: duplicate_keys.append(this_url) continue # we have received a response to our request, update appropriately if message_type == 'Network.responseReceived': this_response = log_item_data['message']['params']['response'] this_url = this_response['url'] # skip if not http(s) if not re.match('^https?://', this_url): continue # the request was received, mark it requests[this_url].update({'received': True}) # record status code and text requests[this_url].update({'status': this_response['status']}) requests[this_url].update({'status_text': this_response['statusText']}) # try to get response headers, fail gracefully as they are already None try: requests[this_url].update({'response_headers':this_response['headersText']}) except: pass try: requests[this_url].update({'content_type':this_response['headers']['Content-Type']}) except: pass # load finish events are keyed to requestId and may apply to many requested urls # so we keep this in a seperate dictionary to be relinked when we're done if message_type == 'Network.loadingFinished': this_request_id = log_item_data['message']['params']['requestId'] this_end_time = log_item_data['message']['params']['timestamp'] # update global end time if last_end_time == None or this_end_time > last_end_time: last_end_time = this_end_time if this_request_id not in load_finish_data: load_finish_data[this_request_id] = {} # size is updated during loading and is shown in logs, but we only want the final size which is here load_finish_data[this_request_id].update({'body_size':log_item_data['message']['params']['encodedDataLength']}) # we use this to calculate the total time for all requests load_finish_data[this_request_id].update({'end_time':this_end_time}) ############################## # webSocket event processing # ############################## # we have a new websocket, create new empty entry keyed to requestId # this will be rekeyed to url # note we ignore timing data for websockets if message_type == 'Network.webSocketCreated': this_url = log_item_data['message']['params']['url'] this_request_id = log_item_data['message']['params']['requestId'] if this_request_id not in websocket_requests: websocket_requests[this_request_id] = {} websocket_requests[this_request_id].update({'url': this_url}) websocket_requests[this_request_id].update({'content_type': 'websocket'}) websocket_requests[this_request_id].update({'received': False}) websocket_requests[this_request_id].update({'end_time': None}) websocket_requests[this_request_id].update({'status': None}) websocket_requests[this_request_id].update({'status_text': None}) websocket_requests[this_request_id].update({'response_headers': None}) websocket_requests[this_request_id].update({'body_size': None}) websocket_requests[this_request_id].update({'end_time': None}) websocket_requests[this_request_id].update({'start_time': None}) websocket_requests[this_request_id].update({'user_agent': None}) websocket_requests[this_request_id].update({'referer': None}) # websocket request made, update relevant fields if message_type == 'Network.webSocketWillSendHandshakeRequest': this_request = log_item_data['message']['params']['request'] this_request_id = log_item_data['message']['params']['requestId'] websocket_requests[this_request_id].update({'request_headers': this_request['headers']}) websocket_requests[this_request_id].update({'user_agent': this_request['headers']['User-Agent']}) # websocket response received, update relevant fields if message_type == 'Network.webSocketHandshakeResponseReceived': this_response = log_item_data['message']['params']['response'] this_request_id = log_item_data['message']['params']['requestId'] websocket_requests[this_request_id].update({'received': True}) websocket_requests[this_request_id].update({'status': this_response['status']}) websocket_requests[this_request_id].update({'status_text': this_response['statusText']}) websocket_requests[this_request_id].update({'response_headers': this_response['headersText']}) # end log processing loop # append load finish info to requests for this_url in requests: this_request_id = requests[this_url]['request_id'] if this_request_id in load_finish_data: requests[this_url].update({'body_size': load_finish_data[this_request_id]['body_size']}) # load_time is start time minus end time, # multiplied by 1k to convert to miliseconds load_time = (load_finish_data[this_request_id]['end_time'] - requests[this_url]['start_time'])*1000 # we shouldn't be getting <=0, but make it null if this happens if load_time <= 0: requests[this_url].update({'load_time': load_time}) else: requests[this_url].update({'load_time': None}) else: requests[this_url].update({'body_size': None}) requests[this_url].update({'load_time': None}) # append websocket data to requests data for item in websocket_requests: requests[websocket_requests[item]['url']] = websocket_requests[item] # return all the links for later processing all_links = [] try: links = driver.find_elements_by_tag_name('a') for link in links: all_links.append([link.get_attribute('text'),link.get_attribute('href')]) except: pass # get the page meta description try: meta_desc = driver.find_element_by_xpath("//meta[@name='description']").get_attribute("content") except: meta_desc = None # get the language of the page try: lang = driver.find_element_by_xpath('/html').get_attribute('lang') except: lang = None # get all the cookies # the selenium get_cookies method does not return third-party cookies # so we open the cookie db directly from the chrome profile # note that in headless mode this does not work in chrome versions # prior to 64.0.3254.0 and no cookies will be returned cookies = [] try: conn = sqlite3.connect(driver.capabilities['chrome']['userDataDir']+'/Default/Cookies') c = conn.cursor() c.execute("SELECT name,is_secure,path,host_key,expires_utc,is_httponly,value FROM cookies") for cookie in c.fetchall(): cookies.append({ 'name': cookie[0], 'secure': cookie[1], 'path': cookie[2], 'domain': cookie[3], 'expiry': cookie[4], 'httponly': cookie[5], 'value': cookie[6] }) except: return({ 'success': False, 'result': 'Cookie database not loaded, if this message appears often something is fundamentally wrong and requires attention!' }) if self.headless == True: browser_version = driver.capabilities['version'] + ' [headless]' else: browser_version = driver.capabilities['version'] # other parts of webxray expect this data format, common to all browser drivers used return_dict = { 'browser_type': driver.capabilities['browserName'], 'browser_version': browser_version, 'browser_wait': browser_wait, 'start_url': url, 'final_url': final_url, 'title': title, 'meta_desc': meta_desc, 'lang': lang, 'load_time': int((last_end_time - first_start_time)*1000), 'processed_requests': requests, 'cookies': cookies, 'all_links': all_links, 'source': page_source } # quit the driver or it will never die! driver.quit() return ({ 'success': True, 'result': return_dict }) # get_webxray_scan_data def get_page_source_text_title_lang(self, url): """ Given a url, this function attempts to load the page, inject the Readability.js library, execute it, and extract only the text of a given page. Function returns the success value (True/False), page source, the extracted text, and the page title in case of success; in case of failure returns an error message. """ # set up sql_driver for logging errors driver = self.create_chromedriver() # browser hasn't started and error already printed to cli if driver == None: return({ 'success': False, 'result': 'Unable to launch Chrome instance' }) # starts the page load process, quits driver and returns nothing if we fail try: driver.get(url) except Exception as e: driver.quit() return({ 'success': False, 'result': 'Unable to load page: '+str(e).replace('\n', ' ') }) # if we can't get source something is wrong, return None try: page_source = driver.page_source except: driver.quit() return({ 'success': False, 'result': 'Unable to read page source' }) # if page title fails put in a null value try: page_title = driver.title except: page_title = None # get the language of the page try: page_lang = driver.find_element_by_xpath('/html').get_attribute('lang') except: page_lang = None # inject the locally downloaded copy of readability into the page # and extract the content # # NOTE: you must download readability on your own and place in the # appropriate directory readability_js = open(os.path.dirname(os.path.abspath(__file__))+'/resources/policyxray/readability.js', 'r', encoding='utf-8').read() try: readabilty_result = json.loads(driver.execute_script(""" %s var readabilityObj = new Readability("%s", document); return JSON.stringify(readabilityObj.parse(), null, 2); """ % (readability_js,url))) driver.quit() except: driver.quit() return({ 'success': False, 'result': 'Unable to inject Readability.js' }) # readability failure, return None if readabilty_result == None: return({ 'success': False, 'result': 'Empty Readability result' }) # readability has HTML formatting, remove it (and extra spaces), return None on failure try: page_text = re.sub('\s+', ' ', re.sub('&.+?;', ' ', re.sub('<.+?>', ' ', readabilty_result['content']))) except: return({ 'success': False, 'result': 'Unable to remove HTML from Readability result' }) # the page text is empty, return None if len(page_text) == 0: return({ 'success': False, 'result': 'Empty result after HTML removal' }) # looks good, return tuple return({ 'success': True, 'result': { 'page_source': page_source, 'page_text': page_text, 'page_title': page_title, 'page_lang': page_lang } }) # get_page_source_text_title_lang def get_absolute_page_links(self,url): """ Returns all links on the page, changes relative links to absolute links as well. """ # initialize browser instance driver = self.create_chromedriver() # browser hasn't started and error already printed to cli if driver == None: return({ 'success': False, 'result': 'Unable to launch Chrome instance' }) # allow one minute before we kill it driver.set_page_load_timeout(60) # starts the page load process, quits driver and returns nothing if we fail try: driver.get(url) except: driver.quit() return({ 'success': False, 'result': 'Unable to load page' }) # page has now been loaded, we process all the links all_links = set() # either there are no links or something has gone wrong try: links = driver.find_elements_by_tag_name('a') except: driver.quit() return({ 'success': False, 'result': 'Unable to extract links' }) # process the links for link in links: # to cope with: # selenium.common.exceptions.StaleElementReferenceException: Message: stale element reference: element is not attached to the page document try: this_link = link.get_attribute('href') this_link_text = re.sub('\s+', ' ', re.sub('[\\n|\\r|\\t]',' ',link.get_attribute('text').strip())) except: continue # sometimes can be empty if this_link == None: continue if len(this_link) == 0: continue # add in the absolute url from the link to our set all_links.add((this_link_text,self.utilities.get_absolute_url_from_page_link(url,this_link))) driver.quit() return({ 'success': True, 'result': all_links })