def _FullFetch(url, json_output, prefetch):
"""Do a full fetch with optional prefetching."""
if not url.startswith('http') and not url.startswith('file'):
url = 'http://' + url
logging.warning('Cold fetch')
cold_data = _LogRequests(url)
assert cold_data, 'Cold fetch failed to produce data. Check your phone.'
if prefetch:
assert not OPTIONS.local
logging.warning('Generating prefetch')
prefetch_html = _GetPrefetchHtml(_ProcessJsonTrace(cold_data), name=url)
tmp = tempfile.NamedTemporaryFile()
tmp.write(prefetch_html)
tmp.flush()
# We hope that the tmpfile name is unique enough for the device.
target = os.path.join('/sdcard/Download', os.path.basename(tmp.name))
device = device_setup.GetFirstDevice()
device.adb.Push(tmp.name, target)
logging.warning('Pushed prefetch %s to device at %s' % (tmp.name, target))
_LoadPage(device, 'file://' + target)
time.sleep(OPTIONS.prefetch_delay_seconds)
logging.warning('Warm fetch')
warm_data = _LogRequests(url, clear_cache_override=False)
with open(json_output, 'w') as f:
json.dump(warm_data, f)
logging.warning('Wrote ' + json_output)
with open(json_output + '.cold', 'w') as f:
json.dump(cold_data, f)
logging.warning('Wrote ' + json_output + '.cold')
else:
with open(json_output, 'w') as f:
json.dump(cold_data, f)
logging.warning('Wrote ' + json_output)
def _LogRequests(url, clear_cache_override=None):
"""Logs requests for a web page.
Args:
url: url to log as string.
clear_cache_override: if not None, set clear_cache different from OPTIONS.
Returns:
JSON dict of logged information (ie, a dict that describes JSON).
"""
if OPTIONS.local:
chrome_ctl = controller.LocalChromeController()
chrome_ctl.SetHeadless(OPTIONS.headless)
else:
chrome_ctl = controller.RemoteChromeController(
device_setup.GetFirstDevice())
clear_cache = (clear_cache_override if clear_cache_override is not None
else OPTIONS.clear_cache)
if OPTIONS.emulate_device:
chrome_ctl.SetDeviceEmulation(OPTIONS.emulate_device)
if OPTIONS.emulate_network:
chrome_ctl.SetNetworkEmulation(OPTIONS.emulate_network)
with chrome_ctl.Open() as connection:
if clear_cache:
connection.ClearCache()
trace = loading_trace.LoadingTrace.RecordUrlNavigation(
url, connection, chrome_ctl.ChromeMetadata())
return trace.ToJsonDict()
def DoPrefetchSetup(arg_str):
OPTIONS.ParseArgs(arg_str, description='Sets up prefetch',
extra=['request_json', 'target_html', ('--upload', False)])
graph_view = _ProcessTraceFile(OPTIONS.request_json)
with open(OPTIONS.target_html, 'w') as html:
html.write(_GetPrefetchHtml(
graph_view, name=os.path.basename(OPTIONS.request_json)))
if OPTIONS.upload:
device = device_setup.GetFirstDevice()
destination = os.path.join('/sdcard/Download',
os.path.basename(OPTIONS.target_html))
device.adb.Push(OPTIONS.target_html, destination)
logging.warning(
'Pushed %s to device at %s' % (OPTIONS.target_html, destination))
def _LogRequests(url, clear_cache_override=None):
"""Logs requests for a web page.
Args:
url: url to log as string.
clear_cache_override: if not None, set clear_cache different from OPTIONS.
Returns:
JSON dict of logged information (ie, a dict that describes JSON).
"""
xvfb_process = None
if OPTIONS.local:
chrome_ctl = controller.LocalChromeController()
if OPTIONS.headless:
xvfb_process = xvfb_helper.LaunchXvfb()
chrome_ctl.SetChromeEnvOverride(xvfb_helper.GetChromeEnvironment())
else:
chrome_ctl = controller.RemoteChromeController(
device_setup.GetFirstDevice())
clear_cache = (clear_cache_override if clear_cache_override is not None
else OPTIONS.clear_cache)
if OPTIONS.emulate_device:
chrome_ctl.SetDeviceEmulation(OPTIONS.emulate_device)
if OPTIONS.emulate_network:
chrome_ctl.SetNetworkEmulation(OPTIONS.emulate_network)
try:
with chrome_ctl.Open() as connection:
if clear_cache:
connection.ClearCache()
trace = loading_trace.LoadingTrace.RecordUrlNavigation(
url,
connection,
chrome_ctl.ChromeMetadata(),
categories=clovis_constants.DEFAULT_CATEGORIES)
except controller.ChromeControllerError as e:
e.Dump(sys.stderr)
raise
if xvfb_process:
xvfb_process.terminate()
return trace.ToJsonDict()
def _LogRequests(url, clear_cache_override=None):
"""Logs requests for a web page.
Args:
url: url to log as string.
clear_cache_override: if not None, set clear_cache different from OPTIONS.
Returns:
JSON dict of logged information (ie, a dict that describes JSON).
"""
device = device_setup.GetFirstDevice() if not OPTIONS.local else None
clear_cache = (clear_cache_override if clear_cache_override is not None
else OPTIONS.clear_cache)
with device_setup.DeviceConnection(device) as connection:
additional_metadata = {}
if OPTIONS.local:
additional_metadata = chrome_setup.SetUpEmulationAndReturnMetadata(
connection, OPTIONS.emulate_device, OPTIONS.emulate_network)
trace = trace_recorder.MonitorUrl(connection,
url,
clear_cache=clear_cache)
trace.metadata.update(additional_metadata)
return trace.ToJsonDict()