def process_dom0_compilation(self, f): entries = parse_log_entry(f) frame_id = entries['frame'] frame_id = self.check_frame_id_call_stack(frame_id) assert self.caller_stack[frame_id][-1][1] == "event_listener" handler = self.caller_stack[frame_id][-1] label = 'dom0_%s_listener' % (handler[3], ) dest_tuple = (entries['frame'], "event_listener", (handler[2], handler[3])) self.redirections[dest_tuple] = ((frame_id, "URL", entries['frame_url']), label)
def process_notification_click(self, f, timestamp): entries = parse_log_entry(f) url = entries['push_notification_target_url'] entries['push_notification_target_url'] = url[url.index('http'):] if 'http' in url else '' notification_target_url = entries['push_notification_target_url'] notification_img_url = entries['push_notification_image'] notification_body = entries['push_notification_body'] notification_title = entries['push_notification_title'] notification_icon_url = entries['push_notification_icon'] if 'push_notification_icon' in entries else '' notification_tag = entries['push_notification_tag'] if 'push_notification_tag' in entries else '' entries['timestamp'] = timestamp entries['log_id'] = self.log_id self.notification_logs.append({'timestamp':timestamp,'message':'Notification click: '+ ' && '.join(['',notification_title,notification_body,'',notification_tag, notification_img_url, notification_target_url, notification_icon_url])})
def stop_script_run(self, f): entries = parse_log_entry(f) frame_id = entries['frame'] if frame_id == self.ignore_script_run: self.ignore_script_run = None return frame_id = self.check_frame_id_call_stack(frame_id) try: assert self.caller_stack[frame_id][-1][1] == "compiled_script" except: return del self.caller_stack[frame_id][-1] if len(self.caller_stack[frame_id]) == 0: del self.caller_stack[frame_id]
def start_scheduled_action(self, f): entries = parse_log_entry(f) frame_id = entries['frame'] name = "code" if "code" in entries else "function" value = (entries["code"].strip() if "code" in entries else entries["scriptID"].strip() + " " + entries['callback_debug_name'].strip()) value = value.strip() related_frame_id = self.check_frame_id_call_stack(frame_id) if related_frame_id is None: self.caller_stack[frame_id] = [] related_frame_id = frame_id self.caller_stack[related_frame_id].append( (frame_id, "scheduled_action", name, value))
def stop_event_handler(self, f): entries = parse_log_entry(f) #target = ":" + ( # entries['scriptID'] + " " + entries['callback_debug_name']).strip() frame_id = entries['frame'] frame_id = self.check_frame_id_call_stack(frame_id) target = entries['current_target'] assert self.caller_stack[frame_id][-1][1] == "event_listener" assert self.caller_stack[frame_id][-1][2].startswith(target) del self.caller_stack[frame_id][-1] if len(self.caller_stack[frame_id]) == 0: del self.caller_stack[frame_id]
def process_js_navigation(self, f): entries = parse_log_entry(f) runner = self.get_current_runner(entries['frame'], entries['main_frame']) if not runner and entries['origin_frame'] and entries[ 'origin_frame'] != entries['frame']: runner = self.get_current_runner(entries['origin_frame']) if not runner: #print "Runner missing!!" #ipdb.set_trace() return dest = (entries['frame'], "URL", entries['url']) #if entries['url'] == debug_url: # ipdb.set_trace() if runner and dest not in self.redirections: self.redirections[dest] = (runner, 'JS Navigation')
def stop_scheduled_action(self, f): entries = parse_log_entry(f) frame_id = entries['frame'] value = (entries["code"].strip() if "code" in entries else entries["scriptID"].strip() + " " + entries['callback_debug_name'].strip()) value = value.strip() frame_id = self.check_frame_id_call_stack(frame_id) try: assert self.caller_stack[frame_id][-1][1] == "scheduled_action" assert self.caller_stack[frame_id][-1][3] == value except: return del self.caller_stack[frame_id][-1] if len(self.caller_stack[frame_id]) == 0: del self.caller_stack[frame_id]
def stop_animation_callback(self, f): entries = parse_log_entry(f) frame_id = entries['frame'] callback_id = entries['callback_id'] related_frame_id = self.check_frame_id_call_stack(frame_id) if related_frame_id is None: self.caller_stack[frame_id] = [] related_frame_id = frame_id try: assert self.caller_stack[related_frame_id][-1][ 1] == "request_animation_frame" assert self.caller_stack[frame_id][-1][2] == callback_id except: return del self.caller_stack[frame_id][-1] if len(self.caller_stack[frame_id]) == 0: del self.caller_stack[frame_id]
def process_load_frame(self, f, timestamp): entries = parse_log_entry(f) # Window Open frame_id = entries['frame'] main_frame_id = entries['main_frame'] if self.load_url is None and entries['load_url'].startswith('http'): self.load_url = entries['load_url'] dest = (frame_id, "URL", entries['load_url']) dest_2 = (frame_id, entries['load_url']) # Else, its a window open that probably happened because of the current runner if (self.pending_window_open_frame_id and self.check_frame_id_call_stack(self.pending_window_open_frame_id)): runner = self.get_current_runner(self.pending_window_open_frame_id) self.redirections[dest] = ( runner, 'Window Open') self.parent_frames[frame_id] = self.pending_window_open_frame_id self.child_frames[self.pending_window_open_frame_id].append(frame_id) self.pending_window_open_frame_id = None return # Else (if no runner), then its a totally unexplained window open # We can atlead log the load frame. if entries['frame_url'].strip('"') != "about:blank": self.load_frame_redirections[(frame_id, "URL", entries['load_url'])] = ( (frame_id, "URL", entries['frame_url']), "Load Frame") self.update_redirections(entries['local_frame_root'],frame_id,entries['load_url'],entries['frame_url'],'Load Frame', timestamp) elif entries['local_frame_root_url'].strip('"') != "about:blank": self.load_frame_redirections[(frame_id, "URL", entries['load_url'])] = ( (frame_id, "URL", entries['local_frame_root_url']), "Load Frame") self.update_redirections(entries['local_frame_root'],frame_id,entries['load_url'],entries['local_frame_root_url'],'Load Frame', timestamp) self.process_frame_based_entries(entries, 'load_url', "Load Frame", timestamp)
def process_notification(self, f, timestamp): entries = parse_log_entry(f) frame_url = entries['frame_url'] url = entries['push_notification_target_url'] entries['push_notification_target_url'] = url[url.index('http'):] if 'http' in url else '' notification_target_url = entries['push_notification_target_url'] notification_img_url = entries['push_notification_image'] notification_icon_url= entries['push_notification_icon'] if 'push_notification_icon' in entries else '' notification_body = entries['push_notification_body'] notification_title = entries['push_notification_title'] notification_tag = entries['push_notification_tag'] if 'push_notification_tag' in entries else '' entries['timestamp'] = timestamp entries['log_id'] = self.log_id self.notification_count +=1 entries['notification_count'] = self.notification_count dbo = db_operations.DBOperator() #dbo.insert_notification(entries) self.notification_logs.append({'timestamp':timestamp,'message':'Notification from: '+frame_url}) self.notification_logs.append({'timestamp':timestamp,'message':'Notification shown: '+ ' && '.join([str(self.notification_count),notification_title,notification_body,frame_url,notification_tag, notification_img_url, notification_target_url,notification_icon_url])})
def start_event_handler_part2(self, f, dom0=False): # TODO: Add script ID for cross checking entries = parse_log_entry(f) frame_id = entries['frame'] if frame_id not in self.pending_handle_event: return target, event = self.pending_handle_event[frame_id] related_frame_id = self.check_frame_id_call_stack(frame_id) target = target + ":" + (entries['scriptID'] + " " + entries['callback_debug_name']).strip() if related_frame_id is None: self.caller_stack[frame_id] = [] related_frame_id = frame_id dest = (frame_id, "event_listener", target, event) self.caller_stack[related_frame_id].append(dest) if dom0: label = 'dom0_%s_listener' self.redirections[dest] = (frame_id, "URL", entries['frame_url'], label) del self.pending_handle_event[frame_id]
def process_window_open(self, f): entries = parse_log_entry(f) self.pending_window_open_frame_id = entries['frame']
def process_requests(self, f, timestamp): entries = parse_log_entry(f) entries['timestamp'] = timestamp entries['log_id'] = self.log_id dbo = db_operations.DBOperator() dbo.insert_request(entries)
def process_notification_message(self, f, timestamp): entries = parse_log_entry(f) self.notification_logs.append({'timestamp':timestamp,'message': ' '.join(entries['notification_message'])})
def process_load_frame(self, f): entries = parse_log_entry(f) #if entries['load_url'] == debug_url: # ipdb.set_trace() # Window Open frame_id = entries['frame'] is_pending_unkown_node_insert = False if frame_id in self.pending_unknown_html_node_insert_frame_ids: is_pending_unkown_node_insert = True self.pending_unknown_html_node_insert_frame_ids.remove(frame_id) #if frame_id == "0x15d7c6f021e8": # ipdb.set_trace() main_frame_id = entries['main_frame'] if self.load_url is None and entries['load_url'].startswith('http'): self.load_url = entries['load_url'] # TODO: Add URL to pending_window_open and double check to make sure that is the one that is being opened. # If there was a href link already inserted for this, then that is likely the reason for this WindowOpen # hence, we can ignore the current_runner and set a redirection for link click instead. dest = (frame_id, "URL", entries['load_url']) if self.pending_window_open_frame_id: src = (self.pending_window_open_frame_id, "URL", entries['load_url']) if src in self.redirections and src != dest: self.redirections[dest] = (src, "Anchor link click") self.pending_window_open_frame_id = None return # Else, its a window open that probably happened because of the current runner if (self.pending_window_open_frame_id and self.check_frame_id_call_stack( self.pending_window_open_frame_id)): runner = self.get_current_runner(self.pending_window_open_frame_id) self.redirections[dest] = (runner, 'Window Open') self.parent_frames[frame_id] = self.pending_window_open_frame_id self.child_frames[self.pending_window_open_frame_id].append( frame_id) self.pending_window_open_frame_id = None return if is_pending_unkown_node_insert: #if frame_id == "0x3bdaa78421e8": # ipdb.set_trace() src = self.check_any_upstream_url_link(entries['load_url']) if src and src != dest: self.redirections[dest] = (src[0], src[1] + " unknown") #else: #print dest #print "Nope!" ########### #print "totally unxpected!" #ipdb.set_trace() # Else (if no runner), then its a totally unexplained window open # We can atlead log the load frame. if entries['frame_url'].strip('"') != "about:blank": self.load_frame_redirections[( frame_id, "URL", entries['load_url'])] = ((frame_id, "URL", entries['frame_url']), "Load Frame") elif entries['local_frame_root_url'].strip('"') != "about:blank": self.load_frame_redirections[( frame_id, "URL", entries['load_url'])] = ((frame_id, "URL", entries['local_frame_root_url']), "Load Frame") self.process_frame_based_entries(entries, 'load_url', "Frame Load")
def process_meta_refresh(self, f): entries = parse_log_entry(f) self.process_frame_based_entries(entries, 'refresh_url', "Meta Refresh")
def process_server_redirect(self, f): entries = parse_log_entry(f) self.update_redirections(entries['frame'], entries['request_url'], entries['redirect_url'], "Server Redirect")
def process_method_template(self, f, timestamp): entries = parse_log_entry(f) service_worker_file=None if 'args' in entries and 'Service_Worker_Register' in entries['args']: service_worker_file = entries['args'].get('Service_Worker_Register') self.notification_logs.append({'timestamp':timestamp,'message': 'Service Worker Registered :: '+ service_worker_file})
def collect_redirects(self, ): with open(self.log_path, 'rb') as f: line = f.readline() while line: #if "060217.434604" in line: # ipdb.set_trace() if "::DidReceiveMainResourceRedirect" in line: self.process_server_redirect(f) elif "::DidHandleHttpRefresh" in line: self.process_meta_refresh(f) elif "::WillNavigateFrame" in line: self.process_js_navigation(f) elif "::WindowOpen" in line: #print "WindowOpen" #ipdb.set_trace() self.process_window_open(f) elif "::WillLoadFrame" in line: self.process_load_frame(f) elif "::DidCompileScript" in line: self.start_script_run(f) elif "::DidRunCompiledScriptEnd" in line: self.stop_script_run(f) elif "::DidCallHandleEventBegin" in line: self.start_event_handler_part1(f) elif "::DidCallFunctionBegin" in line: self.start_event_handler_part2(f) elif "::DidCompileLazyEventListener" in line: self.start_event_handler_part2(f, dom0=True) # self.process_dom0_compilation(f) elif "::DidCallHandleEventEnd" in line: self.stop_event_handler(f) elif "::DidFireScheduledActionBegin" in line: self.start_scheduled_action(f) elif "::DidFireScheduledActionEnd" in line: self.stop_scheduled_action(f) elif "::ExecuteFrameRequestCallbackBegin" in line: self.start_animation_callback(f) elif "::ExecuteFrameRequestCallbackEnd" in line: self.stop_animation_callback(f) elif "::DidCallV8MethodCallback" in line: entries = parse_log_entry(f) if (entries['interface'] == 'History' and (entries['attribute'] == 'pushState' or entries['attribute'] == 'replaceState')): self.process_history_pushstate1(entries) elif "::DidCallV8MethodTemplate" in line: entries = parse_log_entry(f) if (entries['interface'] == 'History' and (entries['attribute'] == 'pushState' or entries['attribute'] == 'replaceState')): self.process_history_pushstate2(entries) if (entries['interface'] == 'EventTarget' and (entries['attribute'] == 'addEventListener')): self.process_addeventlistener(entries) if (entries['interface'] == 'Window' and (entries['attribute'] == 'setTimeout' or entries['attribute'] == 'setInterval')): self.process_settimeout(entries) if (entries['interface'] == 'Element' and (entries['attribute'] == 'setAttribute')): self.process_set_attribute_method(entries) elif "::RequestAnimationFrame" in line: entries = parse_log_entry(f) self.set_animation_callback(entries) elif "::DidCallV8SetAttributeEventListener" in line: entries = parse_log_entry(f) self.process_set_attribute_event_listener(entries) elif "::DidInsertDOMNode" in line: entries = parse_log_entry(f) if (entries['node'] == 'SCRIPT'): self.process_script_load(entries) elif (entries['node']) == 'HTML': self.process_unknown_html_node_insert(entries) # TODO: Add handling of Anchor element inserts elif "::DidCallV8SetAttribute" in line: entries = parse_log_entry(f) self.process_set_attribute(entries) elif "::DidCreateChildFrame" in line: entries = parse_log_entry(f) self.set_child_frame(entries) line = f.readline()
def process_js_navigation(self, f, timestamp): entries = parse_log_entry(f) self.update_redirections(entries['local_frame_root'], entries['frame'], entries['url'], entries['local_frame_root_url'], 'JS Navigation', timestamp)