def process_dom0_compilation(self, f):
entries = parse_log_entry(f)
frame_id = entries['frame']
frame_id = self.check_frame_id_call_stack(frame_id)
assert self.caller_stack[frame_id][-1][1] == "event_listener"
handler = self.caller_stack[frame_id][-1]
label = 'dom0_%s_listener' % (handler[3], )
dest_tuple = (entries['frame'], "event_listener", (handler[2],
handler[3]))
self.redirections[dest_tuple] = ((frame_id, "URL",
entries['frame_url']), label)
def process_notification_click(self, f, timestamp):
entries = parse_log_entry(f)
url = entries['push_notification_target_url']
entries['push_notification_target_url'] = url[url.index('http'):] if 'http' in url else ''
notification_target_url = entries['push_notification_target_url']
notification_img_url = entries['push_notification_image']
notification_body = entries['push_notification_body']
notification_title = entries['push_notification_title']
notification_icon_url = entries['push_notification_icon'] if 'push_notification_icon' in entries else ''
notification_tag = entries['push_notification_tag'] if 'push_notification_tag' in entries else ''
entries['timestamp'] = timestamp
entries['log_id'] = self.log_id
self.notification_logs.append({'timestamp':timestamp,'message':'Notification click: '+
' && '.join(['',notification_title,notification_body,'',notification_tag, notification_img_url, notification_target_url, notification_icon_url])})
def stop_script_run(self, f):
entries = parse_log_entry(f)
frame_id = entries['frame']
if frame_id == self.ignore_script_run:
self.ignore_script_run = None
return
frame_id = self.check_frame_id_call_stack(frame_id)
try:
assert self.caller_stack[frame_id][-1][1] == "compiled_script"
except:
return
del self.caller_stack[frame_id][-1]
if len(self.caller_stack[frame_id]) == 0:
del self.caller_stack[frame_id]
def start_scheduled_action(self, f):
entries = parse_log_entry(f)
frame_id = entries['frame']
name = "code" if "code" in entries else "function"
value = (entries["code"].strip()
if "code" in entries else entries["scriptID"].strip() + " " +
entries['callback_debug_name'].strip())
value = value.strip()
related_frame_id = self.check_frame_id_call_stack(frame_id)
if related_frame_id is None:
self.caller_stack[frame_id] = []
related_frame_id = frame_id
self.caller_stack[related_frame_id].append(
(frame_id, "scheduled_action", name, value))
def stop_event_handler(self, f):
entries = parse_log_entry(f)
#target = ":" + (
# entries['scriptID'] + " " + entries['callback_debug_name']).strip()
frame_id = entries['frame']
frame_id = self.check_frame_id_call_stack(frame_id)
target = entries['current_target']
assert self.caller_stack[frame_id][-1][1] == "event_listener"
assert self.caller_stack[frame_id][-1][2].startswith(target)
del self.caller_stack[frame_id][-1]
if len(self.caller_stack[frame_id]) == 0:
del self.caller_stack[frame_id]
def process_js_navigation(self, f):
entries = parse_log_entry(f)
runner = self.get_current_runner(entries['frame'],
entries['main_frame'])
if not runner and entries['origin_frame'] and entries[
'origin_frame'] != entries['frame']:
runner = self.get_current_runner(entries['origin_frame'])
if not runner:
#print "Runner missing!!"
#ipdb.set_trace()
return
dest = (entries['frame'], "URL", entries['url'])
#if entries['url'] == debug_url:
# ipdb.set_trace()
if runner and dest not in self.redirections:
self.redirections[dest] = (runner, 'JS Navigation')
def stop_scheduled_action(self, f):
entries = parse_log_entry(f)
frame_id = entries['frame']
value = (entries["code"].strip()
if "code" in entries else entries["scriptID"].strip() + " " +
entries['callback_debug_name'].strip())
value = value.strip()
frame_id = self.check_frame_id_call_stack(frame_id)
try:
assert self.caller_stack[frame_id][-1][1] == "scheduled_action"
assert self.caller_stack[frame_id][-1][3] == value
except:
return
del self.caller_stack[frame_id][-1]
if len(self.caller_stack[frame_id]) == 0:
del self.caller_stack[frame_id]
def stop_animation_callback(self, f):
entries = parse_log_entry(f)
frame_id = entries['frame']
callback_id = entries['callback_id']
related_frame_id = self.check_frame_id_call_stack(frame_id)
if related_frame_id is None:
self.caller_stack[frame_id] = []
related_frame_id = frame_id
try:
assert self.caller_stack[related_frame_id][-1][
1] == "request_animation_frame"
assert self.caller_stack[frame_id][-1][2] == callback_id
except:
return
del self.caller_stack[frame_id][-1]
if len(self.caller_stack[frame_id]) == 0:
del self.caller_stack[frame_id]
def process_load_frame(self, f, timestamp):
entries = parse_log_entry(f)
# Window Open
frame_id = entries['frame']
main_frame_id = entries['main_frame']
if self.load_url is None and entries['load_url'].startswith('http'):
self.load_url = entries['load_url']
dest = (frame_id, "URL", entries['load_url'])
dest_2 = (frame_id, entries['load_url'])
# Else, its a window open that probably happened because of the current runner
if (self.pending_window_open_frame_id and
self.check_frame_id_call_stack(self.pending_window_open_frame_id)):
runner = self.get_current_runner(self.pending_window_open_frame_id)
self.redirections[dest] = (
runner, 'Window Open')
self.parent_frames[frame_id] = self.pending_window_open_frame_id
self.child_frames[self.pending_window_open_frame_id].append(frame_id)
self.pending_window_open_frame_id = None
return
# Else (if no runner), then its a totally unexplained window open
# We can atlead log the load frame.
if entries['frame_url'].strip('"') != "about:blank":
self.load_frame_redirections[(frame_id, "URL", entries['load_url'])] = (
(frame_id, "URL", entries['frame_url']), "Load Frame")
self.update_redirections(entries['local_frame_root'],frame_id,entries['load_url'],entries['frame_url'],'Load Frame', timestamp)
elif entries['local_frame_root_url'].strip('"') != "about:blank":
self.load_frame_redirections[(frame_id, "URL", entries['load_url'])] = (
(frame_id, "URL", entries['local_frame_root_url']), "Load Frame")
self.update_redirections(entries['local_frame_root'],frame_id,entries['load_url'],entries['local_frame_root_url'],'Load Frame', timestamp)
self.process_frame_based_entries(entries, 'load_url', "Load Frame", timestamp)
def process_notification(self, f, timestamp):
entries = parse_log_entry(f)
frame_url = entries['frame_url']
url = entries['push_notification_target_url']
entries['push_notification_target_url'] = url[url.index('http'):] if 'http' in url else ''
notification_target_url = entries['push_notification_target_url']
notification_img_url = entries['push_notification_image']
notification_icon_url= entries['push_notification_icon'] if 'push_notification_icon' in entries else ''
notification_body = entries['push_notification_body']
notification_title = entries['push_notification_title']
notification_tag = entries['push_notification_tag'] if 'push_notification_tag' in entries else ''
entries['timestamp'] = timestamp
entries['log_id'] = self.log_id
self.notification_count +=1
entries['notification_count'] = self.notification_count
dbo = db_operations.DBOperator()
#dbo.insert_notification(entries)
self.notification_logs.append({'timestamp':timestamp,'message':'Notification from: '+frame_url})
self.notification_logs.append({'timestamp':timestamp,'message':'Notification shown: '+
' && '.join([str(self.notification_count),notification_title,notification_body,frame_url,notification_tag, notification_img_url, notification_target_url,notification_icon_url])})
def start_event_handler_part2(self, f, dom0=False):
# TODO: Add script ID for cross checking
entries = parse_log_entry(f)
frame_id = entries['frame']
if frame_id not in self.pending_handle_event:
return
target, event = self.pending_handle_event[frame_id]
related_frame_id = self.check_frame_id_call_stack(frame_id)
target = target + ":" + (entries['scriptID'] + " " +
entries['callback_debug_name']).strip()
if related_frame_id is None:
self.caller_stack[frame_id] = []
related_frame_id = frame_id
dest = (frame_id, "event_listener", target, event)
self.caller_stack[related_frame_id].append(dest)
if dom0:
label = 'dom0_%s_listener'
self.redirections[dest] = (frame_id, "URL", entries['frame_url'],
label)
del self.pending_handle_event[frame_id]
def process_window_open(self, f):
entries = parse_log_entry(f)
self.pending_window_open_frame_id = entries['frame']
def process_requests(self, f, timestamp):
entries = parse_log_entry(f)
entries['timestamp'] = timestamp
entries['log_id'] = self.log_id
dbo = db_operations.DBOperator()
dbo.insert_request(entries)
def process_notification_message(self, f, timestamp):
entries = parse_log_entry(f)
self.notification_logs.append({'timestamp':timestamp,'message': ' '.join(entries['notification_message'])})
def process_load_frame(self, f):
entries = parse_log_entry(f)
#if entries['load_url'] == debug_url:
# ipdb.set_trace()
# Window Open
frame_id = entries['frame']
is_pending_unkown_node_insert = False
if frame_id in self.pending_unknown_html_node_insert_frame_ids:
is_pending_unkown_node_insert = True
self.pending_unknown_html_node_insert_frame_ids.remove(frame_id)
#if frame_id == "0x15d7c6f021e8":
# ipdb.set_trace()
main_frame_id = entries['main_frame']
if self.load_url is None and entries['load_url'].startswith('http'):
self.load_url = entries['load_url']
# TODO: Add URL to pending_window_open and double check to make sure that is the one that is being opened.
# If there was a href link already inserted for this, then that is likely the reason for this WindowOpen
# hence, we can ignore the current_runner and set a redirection for link click instead.
dest = (frame_id, "URL", entries['load_url'])
if self.pending_window_open_frame_id:
src = (self.pending_window_open_frame_id, "URL",
entries['load_url'])
if src in self.redirections and src != dest:
self.redirections[dest] = (src, "Anchor link click")
self.pending_window_open_frame_id = None
return
# Else, its a window open that probably happened because of the current runner
if (self.pending_window_open_frame_id
and self.check_frame_id_call_stack(
self.pending_window_open_frame_id)):
runner = self.get_current_runner(self.pending_window_open_frame_id)
self.redirections[dest] = (runner, 'Window Open')
self.parent_frames[frame_id] = self.pending_window_open_frame_id
self.child_frames[self.pending_window_open_frame_id].append(
frame_id)
self.pending_window_open_frame_id = None
return
if is_pending_unkown_node_insert:
#if frame_id == "0x3bdaa78421e8":
# ipdb.set_trace()
src = self.check_any_upstream_url_link(entries['load_url'])
if src and src != dest:
self.redirections[dest] = (src[0], src[1] + " unknown")
#else:
#print dest
#print "Nope!"
###########
#print "totally unxpected!"
#ipdb.set_trace()
# Else (if no runner), then its a totally unexplained window open
# We can atlead log the load frame.
if entries['frame_url'].strip('"') != "about:blank":
self.load_frame_redirections[(
frame_id, "URL",
entries['load_url'])] = ((frame_id, "URL",
entries['frame_url']), "Load Frame")
elif entries['local_frame_root_url'].strip('"') != "about:blank":
self.load_frame_redirections[(
frame_id, "URL",
entries['load_url'])] = ((frame_id, "URL",
entries['local_frame_root_url']),
"Load Frame")
self.process_frame_based_entries(entries, 'load_url', "Frame Load")
def process_meta_refresh(self, f):
entries = parse_log_entry(f)
self.process_frame_based_entries(entries, 'refresh_url',
"Meta Refresh")
def process_server_redirect(self, f):
entries = parse_log_entry(f)
self.update_redirections(entries['frame'], entries['request_url'],
entries['redirect_url'], "Server Redirect")
def process_method_template(self, f, timestamp):
entries = parse_log_entry(f)
service_worker_file=None
if 'args' in entries and 'Service_Worker_Register' in entries['args']:
service_worker_file = entries['args'].get('Service_Worker_Register')
self.notification_logs.append({'timestamp':timestamp,'message': 'Service Worker Registered :: '+ service_worker_file})
def collect_redirects(self, ):
with open(self.log_path, 'rb') as f:
line = f.readline()
while line:
#if "060217.434604" in line:
# ipdb.set_trace()
if "::DidReceiveMainResourceRedirect" in line:
self.process_server_redirect(f)
elif "::DidHandleHttpRefresh" in line:
self.process_meta_refresh(f)
elif "::WillNavigateFrame" in line:
self.process_js_navigation(f)
elif "::WindowOpen" in line:
#print "WindowOpen"
#ipdb.set_trace()
self.process_window_open(f)
elif "::WillLoadFrame" in line:
self.process_load_frame(f)
elif "::DidCompileScript" in line:
self.start_script_run(f)
elif "::DidRunCompiledScriptEnd" in line:
self.stop_script_run(f)
elif "::DidCallHandleEventBegin" in line:
self.start_event_handler_part1(f)
elif "::DidCallFunctionBegin" in line:
self.start_event_handler_part2(f)
elif "::DidCompileLazyEventListener" in line:
self.start_event_handler_part2(f, dom0=True)
# self.process_dom0_compilation(f)
elif "::DidCallHandleEventEnd" in line:
self.stop_event_handler(f)
elif "::DidFireScheduledActionBegin" in line:
self.start_scheduled_action(f)
elif "::DidFireScheduledActionEnd" in line:
self.stop_scheduled_action(f)
elif "::ExecuteFrameRequestCallbackBegin" in line:
self.start_animation_callback(f)
elif "::ExecuteFrameRequestCallbackEnd" in line:
self.stop_animation_callback(f)
elif "::DidCallV8MethodCallback" in line:
entries = parse_log_entry(f)
if (entries['interface'] == 'History'
and (entries['attribute'] == 'pushState'
or entries['attribute'] == 'replaceState')):
self.process_history_pushstate1(entries)
elif "::DidCallV8MethodTemplate" in line:
entries = parse_log_entry(f)
if (entries['interface'] == 'History'
and (entries['attribute'] == 'pushState'
or entries['attribute'] == 'replaceState')):
self.process_history_pushstate2(entries)
if (entries['interface'] == 'EventTarget'
and (entries['attribute'] == 'addEventListener')):
self.process_addeventlistener(entries)
if (entries['interface'] == 'Window'
and (entries['attribute'] == 'setTimeout'
or entries['attribute'] == 'setInterval')):
self.process_settimeout(entries)
if (entries['interface'] == 'Element'
and (entries['attribute'] == 'setAttribute')):
self.process_set_attribute_method(entries)
elif "::RequestAnimationFrame" in line:
entries = parse_log_entry(f)
self.set_animation_callback(entries)
elif "::DidCallV8SetAttributeEventListener" in line:
entries = parse_log_entry(f)
self.process_set_attribute_event_listener(entries)
elif "::DidInsertDOMNode" in line:
entries = parse_log_entry(f)
if (entries['node'] == 'SCRIPT'):
self.process_script_load(entries)
elif (entries['node']) == 'HTML':
self.process_unknown_html_node_insert(entries)
# TODO: Add handling of Anchor element inserts
elif "::DidCallV8SetAttribute" in line:
entries = parse_log_entry(f)
self.process_set_attribute(entries)
elif "::DidCreateChildFrame" in line:
entries = parse_log_entry(f)
self.set_child_frame(entries)
line = f.readline()
def process_js_navigation(self, f, timestamp):
entries = parse_log_entry(f)
self.update_redirections(entries['local_frame_root'],
entries['frame'], entries['url'], entries['local_frame_root_url'],
'JS Navigation', timestamp)