-
Notifications
You must be signed in to change notification settings - Fork 1
/
submit.py
104 lines (86 loc) · 3.07 KB
/
submit.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
#!/usr/bin/env python2.7
#
# This script submits requests to webpagetest.org in order to compare
# performance of web browsers in various conditions.
# Use config.toml to define the parameters.
# alexa-top-10000-global.txt is where the top domains are listed
# The output of this script will be several csv files, one for each location
# we have defined in the config. Once the checks are completed,
# use process_results.py or some other script to fetch the results and process
# them.
#
import toml as pytoml
import httplib
import urllib
import json
import os.path
import sys
from datetime import datetime, date, time
def parse_config():
with open('config.toml', 'rb') as fin:
obj = pytoml.load(fin)
if not obj['key']:
raise Exception('Key must be defined in config file')
if not 'start' in obj:
obj['start'] = 0
if not 'end' in obj:
obj['end'] = 10000
return obj
def parse_domains():
tests = []
with open('top100.txt', 'r') as fin:
tests = [x.strip('\n') for x in fin.readlines()]
return tests
def log_response_id(files, location, index, domain, test_id, owner_key):
f = None
if not location in files:
filename = location+"_"+datetime.now().strftime("%m-%d-%H%M")+".csv"
if not os.path.exists(filename):
f = open(filename, 'w')
else:
f = open(filename, 'a+')
f.seek(0, 2) # go to end
files[location] = f
else:
f = files[location]
f.write(str(index) + ", "+ domain + ", " + test_id + ", " + owner_key + "\n")
def main():
config = parse_config()
print config
tests = parse_domains()
results = {}
files = {}
for i in range(config['start'], config['end']):
for location in config['locations']:
params = urllib.urlencode({'k': config['key'], 'location': location, 'url': tests[i], 'f': 'json', 'runs': 10, 'pingback': config['pingback']})
f = urllib.urlopen("http://www.webpagetest.org/runtest.php", params)
response = json.loads(f.read())
log_response_id(files, location, i, tests[i], response['data']['testId'], response['data']['ownerKey'])
print response
for key in files:
files[key].close()
def submit_one():
config = parse_config()
# domain, location, output
domain = sys.argv[1]
location = sys.argv[2]
output = sys.argv[3]
files = {}
if not os.path.exists(output):
f = open(output, 'w')
else:
f = open(output, 'a+')
f.seek(0, 2) # go to end
files[location] = f
params = urllib.urlencode({'k': config['key'], 'location': location, 'url': domain, 'f': 'json', 'runs': 10, 'pingback': config['pingback']})
f = urllib.urlopen("http://www.webpagetest.org/runtest.php", params)
response = json.loads(f.read())
log_response_id(files, location, 0, domain, response['data']['testId'], response['data']['ownerKey'])
print response
for key in files:
files[key].close()
if __name__ == '__main__':
if len(sys.argv) == 1:
main()
else:
submit_one()