def main():
input_file = 'africa.txt'
#_change_resolv_conf()
_path = '/home/jnejati/PLTSpeed/domains_list'
webDnsSetup.clear_folder(_path)
with open('/home/jnejati/PLTSpeed/res/' + input_file) as _sites:
for _site in _sites:
_site = _site.strip()
logging.info('Navigating to: ' + _site)
s1 = urlparse(_site)
dnsHandler = webDnsSetup.setup_dns_recorder()
time.sleep(5)
try:
node = '/home/jnejati/.nvm/versions/node/v6.9.5/bin/node'
_node_cmd = [
node, 'chrome_launcher.js', _site, '_trace_file',
'_summary_file', '_screenshot_file'
]
subprocess.call(_node_cmd, timeout=110)
except subprocess.TimeoutExpired:
print("Timeout: ", _site, run_no)
dnsHandler.kill()
time.sleep(15)
os.rename(os.path.join(_path, 'domains_list.txt'),
os.path.join(_path, (s1.netloc + '.txt')))
time.sleep(15)
def main():
start = timeit.default_timer()
input_file = 'mixed_live_100runs.txt'
config_file = '/Desktop/Mallesh/PLTSpeed/confs/netProfiles.json'
perf_args = '-etask-clock,context-switches,branches,branch-misses,cache-misses,cache-references,cycles:u,cycles:k,page-faults,sched:sched_switch,sched:sched_stat_runtime,sched:sched_wakeup,instructions:u,instructions:k,dTLB-load-misses,dTLB-loads,dTLB-store-misses,dTLB-stores,iTLB-load-misses,iTLB-loads,L1-dcache-load-misses,L1-dcache-loads,L1-dcache-stores,L1-icache-load-misses,LLC-load-misses,LLC-loads,LLC-store-misses,LLC-stores'
with open(config_file, 'r') as f:
net_profile = json.load(f)[0]
_path = os.path.join(
'/Desktop/Mallesh/PLTSpeed',
net_profile['device_type'] + '_' + net_profile['name'])
webDnsSetup.clear_folder(_path)
with open('/Desktop/Mallesh/PLTSpeed/res/' + input_file) as _sites:
for _site in _sites:
os.system('pkill chrome')
os.system('pkill google-chrome-stable')
time.sleep(5)
os.system(
'DISPLAY=:7 sudo google-chrome-stable --remote-debugging-port=9222 --disable-background-networking --enable-benchmarking --enable-net-benchmarking --start-maximized --ignore-certificate-errors --user-data-dir=$TMPDIR/chrome-profiling --no-default-browser-check &'
)
time.sleep(15)
_site = _site.strip()
logging.info('Navigating to: ' + _site)
s1 = urlparse(_site)
_site_data_folder = os.path.join(_path, s1.netloc)
if not os.path.isdir(_site_data_folder):
os.mkdir(_site_data_folder)
for run_no in range(5):
_run_data_folder = os.path.join(_site_data_folder,
'run_' + str(run_no))
if not os.path.isdir(_run_data_folder):
os.mkdir(_run_data_folder)
_subfolders = [
'trace', 'screenshot', 'analysis', 'summary',
'tcpdump', 'perf'
]
for folder in _subfolders:
os.mkdir(os.path.join(_run_data_folder, folder))
logging.info('Current profile: ' + net_profile['device_type'] +
' - ' + net_profile['name'] + ' run_no: ' +
str(run_no) + ' site: ' + _site)
os.system('pkill tcpdump')
time.sleep(5)
_tcpdump_folder = os.path.join(_run_data_folder, 'tcpdump')
_tcpdump_file = os.path.join(_tcpdump_folder,
str(run_no) + '_' + s1.netloc)
_tcpdump_cmd = [
'tcpdump', '-i', 'enp0s31f6', '-s', '0', '-U', '-w',
_tcpdump_file, 'not', 'port', '22'
]
_tcpdump_proc = subprocess.Popen(_tcpdump_cmd)
_tcpdump_pid = str(_tcpdump_proc.pid)
_perf_folder = os.path.join(_run_data_folder, 'perf')
_perf_file = os.path.join(_perf_folder,
str(run_no) + '_' + s1.netloc)
_perf_cmd = [
'perf', 'stat', '-x,', perf_args, '--output', _perf_file,
'timeout', '--signal=SIGINT', '50'
]
_trace_folder = os.path.join(_run_data_folder, 'trace')
_screenshot_folder = os.path.join(_run_data_folder,
'screenshot')
_summary_folder = os.path.join(_run_data_folder, 'summary')
_trace_file = os.path.join(_trace_folder,
str(run_no) + '_' + s1.netloc)
_screenshot_file = os.path.join(_screenshot_folder,
str(run_no) + '_' + s1.netloc)
_summary_file = os.path.join(_summary_folder,
str(run_no) + '_' + s1.netloc)
logging.info(_trace_file, _screenshot_file, _summary_file)
time.sleep(5)
try:
_node_cmd = [
'node', 'chrome_launcher.js', _site, _trace_file,
_summary_file, _screenshot_file, _tcpdump_pid
]
_cmd = _perf_cmd + _node_cmd
subprocess.call(_cmd, timeout=60)
except subprocess.TimeoutExpired:
print("Timeout: ", _site, run_no)
with open(os.path.join(_site_data_folder, 'log.txt'),
'w+') as _log:
_log.write("Timed out: " + _site + ' ' + str(run_no) +
'\n')
time.sleep(15)
time.sleep(2)
stop = timeit.default_timer()
logging.info(100 * '-' + '\nTotal time: ' + str(stop - start))
def main(input_file, runs, launchChrome=False):
start = timeit.default_timer()
_domains_dir = 'domains_list/'
archive_file = '/home/savargas/DevRegions/data/02_26_18_Africa_sites/testbed/recordings/recording1/archive.wprgo'
# archive_file = "/home/savargas/DevRegions/data/02_03_18_US_sites/testbed/recordings/recording1/archive.wprgo"
config_file = 'confs/netProfiles.2g.json'
_change_resolv_conf('127.0.0.1')
with open(config_file, 'r') as f:
default_net_profile = json.load(f)[0]
_path = os.path.join(default_net_profile['device_type'] + '_' +
default_net_profile['name'])
webDnsSetup.clear_folder(_path)
with open(input_file) as _sites:
# _sites = [x for x in _sites]
for _site in _sites:
_site = _site.strip()
print('Navigating to: ' + _site)
s1 = urlparse(_site)
print(s1)
_site_data_folder = os.path.join(_path, s1.netloc)
if not os.path.isdir(_site_data_folder):
os.mkdir(_site_data_folder)
os.mkdir(os.path.join(_site_data_folder, 'dns'))
with open(os.path.join(_domains_dir, s1.netloc + '.txt'),
newline='') as f:
_domains = csv.reader(f, delimiter=',')
_domains = [x for x in _domains][0]
_domains = [x.rstrip('.') for x in _domains if x[:-1]]
_domains.sort()
### ping Delays
netp = webDnsSetup.ping_delays(_domains, default_net_profile)
netns = network_emulator.NetworkEmulator('desktop', _domains, netp,
default_net_profile)
print('Setting up namespaces ...')
netns.setup_namespace()
print('Setting up link profiles ...')
netns.set_profile()
### DNS delays
time.sleep(5)
dnsHandler = webDnsSetup.setup_dns(_domains)
webDnsSetup.setup_replay(_domains, archive_file)
time.sleep(30)
for run_no in range(runs):
_run_data_folder = os.path.join(_site_data_folder,
'run_' + str(run_no))
if not os.path.isdir(_run_data_folder):
os.mkdir(_run_data_folder)
_subfolders = [
'trace', 'screenshot', 'analysis', 'summary'
]
for folder in _subfolders:
os.mkdir(os.path.join(_run_data_folder, folder))
print('Current profile: ' +
default_net_profile['device_type'] + ' - ' +
default_net_profile['name'] + ' run_no: ' + str(run_no) +
' site: ' + _site)
time.sleep(15)
_trace_folder = os.path.join(_run_data_folder, 'trace')
_screenshot_folder = os.path.join(_run_data_folder,
'screenshot')
_summary_folder = os.path.join(_run_data_folder, 'summary')
_trace_file = os.path.join(_trace_folder,
str(run_no) + '_' + s1.netloc)
_screenshot_file = os.path.join(_screenshot_folder,
str(run_no) + '_' + s1.netloc)
_summary_file = os.path.join(_summary_folder,
str(run_no) + '_' + s1.netloc)
_launch_chrome = str(launchChrome)
logging.info(_trace_file, _screenshot_file, _summary_file)
time.sleep(5)
try:
node = 'node'
_node_cmd = [
node, 'chrome_launcher.js', _site, _trace_file,
_summary_file, _screenshot_file, _launch_chrome
]
_cmd = _node_cmd
# print("Node Command", _cmd)
subprocess.call(_cmd, timeout=110)
except subprocess.TimeoutExpired:
print("Timeout: ", _site, run_no)
with open(os.path.join(_site_data_folder, 'log.txt'),
'w+') as _log:
_log.write("Timed out: " + _site + ' ' + str(run_no) +
'\n')
finally:
os.system('pkill node')
os.system('pkill wpr')
dnsHandler.terminate()
time.sleep(5)
webDnsSetup.clear_ip_tables()
stop = timeit.default_timer()
logging.info(100 * '-' + '\nTotal time: ' + str(stop - start))
_change_resolv_conf('8.8.8.8')
def main():
start = timeit.default_timer()
input_file = 'mixed_live_100runs.txt'
archive_dir = '/home/jnejati/PLTSpeed/record/archive/'
config_file = '/home/jnejati/PLTSpeed/confs/netProfiles.json'
perf_args = '-etask-clock,context-switches,branches,branch-misses,cache-misses,cache-references,cycles:u,cycles:k,page-faults,sched:sched_switch,sched:sched_stat_runtime,sched:sched_wakeup,instructions:u,instructions:k,dTLB-load-misses,dTLB-loads,dTLB-store-misses,dTLB-stores,iTLB-load-misses,iTLB-loads,L1-dcache-load-misses,L1-dcache-loads,L1-dcache-stores,L1-icache-load-misses,LLC-load-misses,LLC-loads,LLC-store-misses,LLC-stores'
#_change_resolv_conf()
with open(config_file, 'r') as f:
net_profile = json.load(f)[0]
_path = os.path.join(
'/home/jnejati/PLTSpeed',
net_profile['device_type'] + '_' + net_profile['name'])
webDnsSetup.clear_folder(_path)
#_google_chrome_stable = '/usr/bin/google-chrome-stable'
#_remote_debugging_cmd = [_google_chrome_stable, '--remote-debugging-port=9222', '--start-maximized', '--ignore-certificate-errors', '--user-data-dir=$TMPDIR/chrome-profiling', '--no-default-browser-check']
#netns = network_emulator.NetworkEmulator(net_profile, dirs)
#netns.set_profile(net_profile['conn_type'])
with open('/home/jnejati/PLTSpeed/res/' + input_file) as _sites:
for _site in _sites:
#_chrome_process = subprocess.Popen(_remote_debugging_cmd)
os.system('pkill chrome')
os.system('pkill google-chrome-stable')
#os.system('service dbus restart')
time.sleep(5)
os.system(
'google-chrome-stable --remote-debugging-port=9222 --enable-benchmarking --enable-net-benchmarking --start-maximized --ignore-certificate-errors --user-data-dir=$TMPDIR/chrome-profiling --no-default-browser-check &'
)
time.sleep(40)
_site = _site.strip()
logging.info('Navigating to: ' + _site)
s1 = urlparse(_site)
#s1 = urlparse('http://' + _site.split('localhost.localdomain/')[-1])
_site_data_folder = os.path.join(_path, s1.netloc)
if not os.path.isdir(_site_data_folder):
os.mkdir(_site_data_folder)
#os.mkdir(os.path.join(_site_data_folder, 'dns'))
#_d_ip_dict = webDnsSetup.setup_ip_subdomain(s1.netloc, archive_dir)
for run_no in range(1):
#os.system('pkill chrome')
#os.system('pkill google-chrome-stable')
#time.sleep(5)
#os.system('DISPLAY=:7 sudo google-chrome-stable --remote-debugging-port=9222 --start-maximized --ignore-certificate-errors --user-data-dir=$TMPDIR/chrome-profiling --no-default-browser-check &')
#time.sleep(5)
_run_data_folder = os.path.join(_site_data_folder,
'run_' + str(run_no))
if not os.path.isdir(_run_data_folder):
os.mkdir(_run_data_folder)
_subfolders = [
'trace', 'screenshot', 'analysis', 'summary',
'tcpdump', 'perf'
]
for folder in _subfolders:
os.mkdir(os.path.join(_run_data_folder, folder))
logging.info('Current profile: ' + net_profile['device_type'] +
' - ' + net_profile['name'] + ' run_no: ' +
str(run_no) + ' site: ' + _site)
#netns = network_emulator.NetworkEmulator(net_profile, dirs)
#netns.set_profile(net_profile['conn_type'])
os.system('pkill tcpdump')
#os.system('pkill node')
time.sleep(5)
_tcpdump_folder = os.path.join(_run_data_folder, 'tcpdump')
_tcpdump_file = os.path.join(_tcpdump_folder,
str(run_no) + '_' + s1.netloc)
_tcpdump_cmd = [
'tcpdump', '-i', 'enp1s0f0', '-s', '0', '-U', '-w',
_tcpdump_file, 'not', 'port', '22'
]
_tcpdump_proc = subprocess.Popen(_tcpdump_cmd)
_tcpdump_pid = str(_tcpdump_proc.pid)
_perf_folder = os.path.join(_run_data_folder, 'perf')
_perf_file = os.path.join(_perf_folder,
str(run_no) + '_' + s1.netloc)
_perf_cmd = [
'perf', 'stat', '-x,', perf_args, '--output', _perf_file,
'timeout', '--signal=SIGINT', '210'
]
_trace_folder = os.path.join(_run_data_folder, 'trace')
_screenshot_folder = os.path.join(_run_data_folder,
'screenshot')
_summary_folder = os.path.join(_run_data_folder, 'summary')
_trace_file = os.path.join(_trace_folder,
str(run_no) + '_' + s1.netloc)
_screenshot_file = os.path.join(_screenshot_folder,
str(run_no) + '_' + s1.netloc)
_summary_file = os.path.join(_summary_folder,
str(run_no) + '_' + s1.netloc)
logging.info(_trace_file, _screenshot_file, _summary_file)
time.sleep(5)
try:
_node_cmd = [
'node', 'chrome_launcher.js', _site, _trace_file,
_summary_file, _screenshot_file, _tcpdump_pid
]
_cmd = _perf_cmd + _node_cmd
subprocess.call(_cmd, timeout=220)
except subprocess.TimeoutExpired:
print("Timeout: ", _site, run_no)
with open(os.path.join(_site_data_folder, 'log.txt'),
'w+') as _log:
_log.write("Timed out: " + _site + ' ' + str(run_no) +
'\n')
time.sleep(15)
#pickle.dump(_d_ip_dict, open(os.path.join(_site_data_folder, 'dns/dnsBackup.txt'), 'wb'))
time.sleep(2)
stop = timeit.default_timer()
#_chrome_process.kill()
logging.info(100 * '-' + '\nTotal time: ' + str(stop - start))