def run(self):
logging.debug('runner running')
tests = self._build_testlist()
preargs = self._build_preargs()
logging.debug('tests to run: %s' % (tests,))
logging.debug('args to prepend: %s' % (preargs,))
# Ensure our output directory exists
outdir = stoneridge.get_config('run', 'out')
profile = stoneridge.get_config('run', 'profile')
installroot = stoneridge.get_config('stoneridge', 'root')
for test in tests:
logging.debug('test: %s' % (test,))
outfile = os.path.join(outdir, '%s.out' % (test,))
logging.debug('outfile: %s' % (outfile,))
if test.endswith('.js'):
escaped_outfile = outfile.replace('\\', '\\\\')
args = preargs + [
'-e', 'const _SR_OUT_FILE = "%s";' % (escaped_outfile,),
'-f', os.path.join(installroot, 'srdata.js'),
'-f', os.path.join(installroot, 'head.js'),
'-f', os.path.join(self.testroot, test),
'-e', 'do_stoneridge(); quit(0);'
]
logging.debug('xpcshell args: %s' % (args,))
runner = stoneridge.run_xpcshell
else:
args = [
'-sr', os.path.join(self.testroot, test),
'-sroutput', outfile,
'-profile', profile,
# -srwidth, <some width value>,
# -srheight, <some height value>,
# -srtimeout, <some timeout value per page>,
# -srdelay, <some delay value between pages>,
# -srmozafterpaint
]
runner = stoneridge.run_firefox
if self.unittest:
logging.debug('Not running processes: in unit test mode')
else:
process_out_file = '%s.process.out' % (test,)
process_out_file = os.path.join(outdir, process_out_file)
logging.debug('process output at %s' % (process_out_file,))
timed_out = False
with file(process_out_file, 'wb') as f:
try:
res = runner(args, f)
except stoneridge.TestProcessTimeout:
logging.exception('test process timed out!')
timed_out = True
res = None
if res or timed_out:
logging.error('TEST FAILED: %s' % (test,))
else:
logging.debug('test succeeded')
def __init__(self):
self.workdir = stoneridge.get_config('run', 'work')
logging.debug('work directory: %s' % (self.workdir, ))
self.bindir = stoneridge.get_config('run', 'bin')
logging.debug('bin directory: %s' % (self.bindir, ))
downloaddir = stoneridge.get_config('run', 'download')
download_suffix = stoneridge.get_config('machine', 'download_suffix')
self.firefoxpkg = os.path.join(downloaddir,
'firefox.%s' % (download_suffix, ))
logging.debug('firefox package: %s' % (self.firefoxpkg, ))
self.testzip = os.path.join(downloaddir, 'tests.zip')
logging.debug('test zip file: %s' % (self.testzip, ))
def __init__(self):
self.workdir = stoneridge.get_config('run', 'work')
logging.debug('work directory: %s' % (self.workdir,))
self.bindir = stoneridge.get_config('run', 'bin')
logging.debug('bin directory: %s' % (self.bindir,))
downloaddir = stoneridge.get_config('run', 'download')
download_suffix = stoneridge.get_config('machine', 'download_suffix')
self.firefoxpkg = os.path.join(downloaddir,
'firefox.%s' % (download_suffix,))
logging.debug('firefox package: %s' % (self.firefoxpkg,))
self.testzip = os.path.join(downloaddir, 'tests.zip')
logging.debug('test zip file: %s' % (self.testzip,))
def setup(self):
self.host = stoneridge.get_config('report', 'host')
self.project = stoneridge.get_config('report', 'project')
self.key = stoneridge.get_config('report', 'key')
self.secret = stoneridge.get_config('report', 'secret')
self.archives = stoneridge.get_config('stoneridge', 'archives')
self.unittest = stoneridge.get_config_bool('stoneridge', 'unittest')
logging.debug('report host: %s' % (self.host,))
logging.debug('project: %s' % (self.project,))
logging.debug('oauth key: %s' % (self.key,))
logging.debug('oauth secret: %s' % (self.secret,))
logging.debug('archives: %s' % (self.archives,))
logging.debug('unittest: %s' % (self.unittest,))
def setup(self):
self.host = stoneridge.get_config('report', 'host')
self.project = stoneridge.get_config('report', 'project')
self.key = stoneridge.get_config('report', 'key')
self.secret = stoneridge.get_config('report', 'secret')
self.archives = stoneridge.get_config('stoneridge', 'archives')
self.unittest = stoneridge.get_config_bool('stoneridge', 'unittest')
logging.debug('report host: %s' % (self.host, ))
logging.debug('project: %s' % (self.project, ))
logging.debug('oauth key: %s' % (self.key, ))
logging.debug('oauth secret: %s' % (self.secret, ))
logging.debug('archives: %s' % (self.archives, ))
logging.debug('unittest: %s' % (self.unittest, ))
def __init__(self, restore):
self.restore = restore
self.peer = ('127.0.0.1', 63250)
os_name = stoneridge.get_config('machine', 'os')
if os_name == 'windows':
self.is_windows = True
else:
self.is_windows = False
self.netconfig = stoneridge.get_config('run', 'netconfig')
self.unittest = stoneridge.get_config_bool('stoneridge', 'unittest')
logging.debug('restore: %s' % (restore,))
logging.debug('peer: %s' % (self.peer,))
logging.debug('is windows: %s' % (self.is_windows,))
logging.debug('netconfig: %s' % (self.netconfig,))
logging.debug('unittest: %s' % (self.unittest,))
def __init__(self, restore):
self.restore = restore
self.peer = ('127.0.0.1', 63250)
os_name = stoneridge.get_config('machine', 'os')
if os_name == 'windows':
self.is_windows = True
else:
self.is_windows = False
self.netconfig = stoneridge.get_config('run', 'netconfig')
self.unittest = stoneridge.get_config_bool('stoneridge', 'unittest')
logging.debug('restore: %s' % (restore, ))
logging.debug('peer: %s' % (self.peer, ))
logging.debug('is windows: %s' % (self.is_windows, ))
logging.debug('netconfig: %s' % (self.netconfig, ))
logging.debug('unittest: %s' % (self.unittest, ))
def run(self):
logging.debug('info gatherer running')
bindir = stoneridge.get_config('run', 'bin')
info_file = os.path.join(bindir, 'application.ini')
logging.debug('parsing ini file at %s' % (info_file, ))
cp = ConfigParser.SafeConfigParser()
cp.read([info_file])
build_info = {}
build_info['name'] = cp.get('App', 'Name')
build_info['version'] = cp.get('App', 'Version')
build_info['revision'] = cp.get('App', 'SourceStamp')
build_info['branch'] = stoneridge.get_config('run', 'netconfig')
# Due to the way the graph server works, we need to create a unique
# build id for each build/os/netconfig combination. We also want to
# keep the unmodified build ID around for posterity.
build_info['original_buildid'] = cp.get('App', 'BuildID')
# Build ID is limited to 16 characters in the receiving database, and
# our suffix is 2 characters, so we truncate the original to 14
# characters before adding our suffix. It should already be only 14
# characters, but we do this Just In Case.
buildid_base = build_info['original_buildid'][:14]
build_info['id'] = buildid_base + stoneridge.get_buildid_suffix()
machine_info = {}
machine_info['name'] = platform.node()
machine_info['os'] = stoneridge.get_config('machine', 'os')
machine_info['osversion'] = stoneridge.get_os_version()
machine_info['platform'] = platform.machine()
info = {
'test_machine': machine_info,
'test_build': build_info,
'testrun': {},
'date': stoneridge.get_config_int('run', 'tstamp')
}
logging.debug('gathered info: %s' % (info, ))
outdir = stoneridge.get_config('run', 'out')
if not os.path.exists(outdir):
logging.debug('making outdir %s' % (outdir, ))
os.mkdir(outdir)
with file(os.path.join(outdir, 'info.json'), 'wb') as f:
logging.debug('dumping json to file')
json.dump(info, f)
def do_POST(self):
"""Handle getting uploaded results from the clients
"""
logging.debug('Handling request from %s' % (self.client_address,))
logging.debug('Path: %s' % (self.path,))
rootdir = stoneridge.get_config('server', 'uploads')
now = int(time.time())
idx = 0
post = cgi.FieldStorage(fp=self.rfile, headers=self.headers,
environ={'REQUEST_METHOD':'POST',
'CONTENT_TYPE':self.headers['Content-Type']})
for k in post.keys():
v = post[k].value
logging.debug('Processing post variable %s' % (k,))
logging.debug('%s value = %s' % (k, v))
if len(v) < 10:
# Stupid check to filter out junk data that gets submitted from
# time to time, until we can figure out why said junk data gets
# submitted
logging.debug('Skipping invalid json value')
continue
pfx = '%s_%s_' % (now, idx)
idx += 1
fd, name = tempfile.mkstemp(dir=rootdir, prefix=pfx, suffix='.json')
os.write(fd, v)
os.fchmod(fd, 0644)
os.close(fd)
self.send_response(200)
def __init__(self, stop):
self.stop = stop
self.macaddr = stoneridge.get_config('machine', 'macaddr')
self.netconfig = stoneridge.get_config('run', 'netconfig')
self.host = stoneridge.get_config('tcpdump', self.netconfig)
self.outdir = stoneridge.get_config('run', 'out')
self.stdout = os.path.join(self.outdir, 'tcpdump.out')
self.pcap = os.path.join(self.outdir, 'traffic.pcap')
logging.debug('stop: %s' % (stop,))
logging.debug('macaddr: %s' % (self.macaddr,))
logging.debug('netconfig: %s' % (self.netconfig,))
logging.debug('host: %s' % (self.host,))
logging.debug('output directory: %s' % (self.outdir,))
logging.debug('tcpdump output: %s' % (self.stdout,))
logging.debug('packet capture: %s' % (self.pcap,))
def __init__(self, stop):
self.stop = stop
self.macaddr = stoneridge.get_config('machine', 'macaddr')
self.netconfig = stoneridge.get_config('run', 'netconfig')
self.host = stoneridge.get_config('tcpdump', self.netconfig)
self.outdir = stoneridge.get_config('run', 'out')
self.stdout = os.path.join(self.outdir, 'tcpdump.out')
self.pcap = os.path.join(self.outdir, 'traffic.pcap')
logging.debug('stop: %s' % (stop, ))
logging.debug('macaddr: %s' % (self.macaddr, ))
logging.debug('netconfig: %s' % (self.netconfig, ))
logging.debug('host: %s' % (self.host, ))
logging.debug('output directory: %s' % (self.outdir, ))
logging.debug('tcpdump output: %s' % (self.stdout, ))
logging.debug('packet capture: %s' % (self.pcap, ))
def run(self):
logging.debug('info gatherer running')
bindir = stoneridge.get_config('run', 'bin')
info_file = os.path.join(bindir, 'application.ini')
logging.debug('parsing ini file at %s' % (info_file,))
cp = ConfigParser.SafeConfigParser()
cp.read([info_file])
build_info = {}
build_info['name'] = cp.get('App', 'Name')
build_info['version'] = cp.get('App', 'Version')
build_info['revision'] = cp.get('App', 'SourceStamp')
build_info['branch'] = stoneridge.get_config('run', 'netconfig')
# Due to the way the graph server works, we need to create a unique
# build id for each build/os/netconfig combination. We also want to
# keep the unmodified build ID around for posterity.
build_info['original_buildid'] = cp.get('App', 'BuildID')
# Build ID is limited to 16 characters in the receiving database, and
# our suffix is 2 characters, so we truncate the original to 14
# characters before adding our suffix. It should already be only 14
# characters, but we do this Just In Case.
buildid_base = build_info['original_buildid'][:14]
build_info['id'] = buildid_base + stoneridge.get_buildid_suffix()
machine_info = {}
machine_info['name'] = platform.node()
machine_info['os'] = stoneridge.get_config('machine', 'os')
machine_info['osversion'] = stoneridge.get_os_version()
machine_info['platform'] = platform.machine()
info = {'test_machine': machine_info,
'test_build': build_info,
'testrun': {},
'date': stoneridge.get_config_int('run', 'tstamp')}
logging.debug('gathered info: %s' % (info,))
outdir = stoneridge.get_config('run', 'out')
if not os.path.exists(outdir):
logging.debug('making outdir %s' % (outdir,))
os.mkdir(outdir)
with file(os.path.join(outdir, 'info.json'), 'wb') as f:
logging.debug('dumping json to file')
json.dump(info, f)
def daemon():
global conn
dbfile = stoneridge.get_config('mqproxy', 'db')
conn = sqlite3.connect(dbfile)
stoneridge.StreamLogger.bottle_inject()
port = stoneridge.get_config_int('mqproxy', 'port')
bottle.run(host='0.0.0.0', port=port)
def setup(self):
self.srconffile = stoneridge.get_config_file()
self.unittest = stoneridge.get_config_bool('stoneridge', 'unittest')
self.workroot = stoneridge.get_config('stoneridge', 'work')
logging.debug('srconffile: %s' % (self.srconffile, ))
logging.debug('unittest: %s' % (self.unittest, ))
self.runconfig = None # Needs to be here so reset doesn't barf
self.reset()
def setup(self):
self.srconffile = stoneridge.get_config_file()
self.unittest = stoneridge.get_config_bool('stoneridge', 'unittest')
self.workroot = stoneridge.get_config('stoneridge', 'work')
logging.debug('srconffile: %s' % (self.srconffile,))
logging.debug('unittest: %s' % (self.unittest,))
self.runconfig = None # Needs to be here so reset doesn't barf
self.reset()
def daemon():
global conn
dbfile = stoneridge.get_config('mqproxy', 'db')
conn = sqlite3.connect(dbfile)
stoneridge.StreamLogger.bottle_inject()
port = stoneridge.get_config_int('mqproxy', 'port')
bottle.run(host='0.0.0.0', port=port)
def unpack_firefox(self):
# MAC, Y U NO USE REGULAR ARCHIVE?!
installroot = stoneridge.get_config('stoneridge', 'root')
installdmg = os.path.join(installroot, 'installdmg.sh')
logging.debug('mac using installdmg at %s' % (installdmg,))
out = subprocess.check_output(['/bin/bash', installdmg,
self.firefoxpkg],
cwd=self.workdir,
stderr=subprocess.STDOUT)
logging.debug(out)
def send_email(check):
"""Send an email to me so I know when the dns update failed.
"""
myos = stoneridge.get_config('machine', 'os')
netconfig = stoneridge.get_config('run', 'netconfig')
srid = stoneridge.get_config('run', 'srid')
logging.debug('sending email: os=%s, netconfig=%s, srid=%s, check=%s' %
(myos, netconfig, srid, check))
to = '*****@*****.**'
subject = 'DNS Update Failed'
msg = '''The DNS Update failed for the following run:
OS: %s
Netconfig: %s
SRID: %s
Check failed: %s
''' % (myos, netconfig, srid, check)
stoneridge.mail(to, subject, msg)
def unpack_firefox(self):
# MAC, Y U NO USE REGULAR ARCHIVE?!
installroot = stoneridge.get_config('stoneridge', 'root')
installdmg = os.path.join(installroot, 'installdmg.sh')
logging.debug('mac using installdmg at %s' % (installdmg, ))
out = subprocess.check_output(
['/bin/bash', installdmg, self.firefoxpkg],
cwd=self.workdir,
stderr=subprocess.STDOUT)
logging.debug(out)
def setup(self):
self.queues = {
'broadband': stoneridge.QueueWriter(
stoneridge.NETCONFIG_QUEUES['broadband']),
'umts': stoneridge.QueueWriter(
stoneridge.NETCONFIG_QUEUES['umts']),
'gsm': stoneridge.QueueWriter(
stoneridge.NETCONFIG_QUEUES['gsm'])
}
self.logdir = stoneridge.get_config('stoneridge', 'logs')
self.config = stoneridge.get_config_file()
def main():
parser = stoneridge.TestRunArgumentParser()
parser.parse_args()
netconfig = stoneridge.get_config('run', 'netconfig')
server = stoneridge.get_config('dns', netconfig)
myos = stoneridge.get_config('machine', 'os')
logging.debug('netconfig: %s' % (netconfig, ))
logging.debug('server: %s' % (server, ))
logging.debug('os: %s' % (myos, ))
if myos == 'windows':
countarg = '-n'
else:
countarg = '-c'
logging.debug('countarg: %s' % (countarg, ))
logging.debug('Give me a ping, Vasili. One ping only, please.')
p = stoneridge.Process(['ping', countarg, '1', server])
p.communicate()
p.wait()
def main():
parser = stoneridge.TestRunArgumentParser()
parser.parse_args()
netconfig = stoneridge.get_config('run', 'netconfig')
server = stoneridge.get_config('dns', netconfig)
myos = stoneridge.get_config('machine', 'os')
logging.debug('netconfig: %s' % (netconfig,))
logging.debug('server: %s' % (server,))
logging.debug('os: %s' % (myos,))
if myos == 'windows':
countarg = '-n'
else:
countarg = '-c'
logging.debug('countarg: %s' % (countarg,))
logging.debug('Give me a ping, Vasili. One ping only, please.')
p = stoneridge.Process(['ping', countarg, '1', server])
p.communicate()
p.wait()
def main():
parser = stoneridge.DaemonArgumentParser()
parser.add_argument('--nochange', dest='nochange', action='store_true')
args = parser.parse_args()
global nochange
nochange = args.nochange
global rundir
rundir = stoneridge.get_config('stoneridge', 'run')
parser.start_daemon(daemon)
def main():
parser = stoneridge.DaemonArgumentParser()
parser.add_argument('--nochange', dest='nochange', action='store_true')
args = parser.parse_args()
global nochange
nochange = args.nochange
global rundir
rundir = stoneridge.get_config('stoneridge', 'run')
parser.start_daemon(daemon)
def run(self):
logging.debug('uploader running')
outdir = stoneridge.get_config('run', 'out')
pattern = os.path.join(outdir, 'upload_*.json')
files = glob.glob(pattern)
if not files:
# No results, but we still want to upload metadata for centralized
# debugging information.
logging.warning('no results to upload!')
results = {}
for filename in files:
fname = os.path.basename(filename)
with file(filename) as f:
results[fname] = json.load(f)
metadata_file = stoneridge.get_config('run', 'metadata')
if os.path.exists(metadata_file):
with file(metadata_file, 'rb') as f:
contents = f.read()
metadata = base64.b64encode(contents)
else:
# Missing metadata, but we can still report results
logging.warning('missing metadata, continuing anyway')
metadata = base64.b64encode('')
srid = stoneridge.get_config('run', 'srid')
netconfig = stoneridge.get_config('run', 'netconfig')
ldap = stoneridge.get_config('run', 'ldap')
operating_system = stoneridge.get_config('machine', 'os')
self.queue.enqueue(srid=srid, results=results, metadata=metadata,
netconfig=netconfig,
operating_system=operating_system,
ldap=ldap)
def main():
parser = stoneridge.ArgumentParser()
parser.parse_args()
root = stoneridge.get_config('enqueuer', 'root')
username = stoneridge.get_config('enqueuer', 'username')
password = stoneridge.get_config('enqueuer', 'password')
try:
res = requests.get(root + '/list_unhandled', auth=(username, password))
except:
# For some reason, we sometimes get a requests failure here, even
# though everything seems to be working fine. Ignore that, and try
# again later.
logging.exception('Error listing unhandled pushes')
return
try:
queue = json.loads(res.text)
except:
# Yet another failure mode. Yay.
logging.exception('Error demarshalling result %s' % (res.text,))
return
for entry in queue:
try:
requests.post(root + '/mark_handled', data={'id': entry['pushid']},
auth=(username, password))
except:
# If we fail to mark this as handled, wait until the next try so we
# don't run the same thing more than once. It's not the end of the
# world ot have to wait...
logging.exception('Error marking entry %s as handled' % (entry,))
return
logging.debug('Equeuing entry %s' % (entry,))
stoneridge.enqueue(nightly=False, ldap=entry['ldap'], sha=entry['sha'],
netconfigs=entry['netconfigs'],
operating_systems=entry['operating_systems'],
srid=entry['srid'])
def __init__(self, nightly, srid, operating_systems, netconfigs,
ldap, sha, attempt):
self.host = stoneridge.get_config('cloner', 'host')
self.nightly = nightly
self.outroot = stoneridge.get_config('cloner', 'output')
self.srid = srid
self.outdir = os.path.join(self.outroot, srid)
self.keep = stoneridge.get_config_int('cloner', 'keep', default=50)
self.max_attempts = stoneridge.get_config_int('cloner', 'attempts')
self.operating_systems = operating_systems
self.netconfigs = netconfigs
self.ldap = ldap
self.sha = sha
self.attempt = attempt
if not os.path.exists(self.outroot):
os.mkdir(self.outroot)
root = stoneridge.get_config('cloner', 'root')
if nightly:
self.path = '/'.join([root, 'nightly', 'latest-mozilla-central'])
else:
self.path = '/'.join([root, 'try-builds', '%s-%s' % (ldap, sha)])
logging.debug('host: %s' % (self.host,))
logging.debug('path: %s' % (self.path,))
logging.debug('nightly: %s' % (self.nightly,))
logging.debug('srid: %s' % (self.srid,))
logging.debug('output root: %s' % (self.outroot,))
logging.debug('output directory: %s' % (self.outdir,))
logging.debug('keep history: %s' % (self.keep,))
logging.debug('max attempts: %s' % (self.max_attempts,))
logging.debug('operating systems: %s' % (self.operating_systems,))
logging.debug('netconfigs: %s' % (self.netconfigs,))
logging.debug('ldap: %s' % (self.ldap,))
logging.debug('sha: %s' % (self.sha,))
logging.debug('attempt: %s' % (self.attempt,))
self.prefix = ''
def main():
parser = stoneridge.ArgumentParser()
parser.parse_args()
osname = stoneridge.get_config('machine', 'os')
queue = stoneridge.CLIENT_QUEUES[osname]
while True:
try:
worker = StoneRidgeWorker(queue)
worker.run()
except:
logging.exception('Worker failed')
def __init__(self, nightly, srid, operating_systems, netconfigs, ldap, sha,
attempt):
self.host = stoneridge.get_config('cloner', 'host')
self.nightly = nightly
self.outroot = stoneridge.get_config('cloner', 'output')
self.srid = srid
self.outdir = os.path.join(self.outroot, srid)
self.keep = stoneridge.get_config_int('cloner', 'keep', default=50)
self.max_attempts = stoneridge.get_config_int('cloner', 'attempts')
self.operating_systems = operating_systems
self.netconfigs = netconfigs
self.ldap = ldap
self.sha = sha
self.attempt = attempt
if not os.path.exists(self.outroot):
os.mkdir(self.outroot)
root = stoneridge.get_config('cloner', 'root')
if nightly:
self.path = '/'.join([root, 'nightly', 'latest-mozilla-central'])
else:
self.path = '/'.join([root, 'try-builds', '%s-%s' % (ldap, sha)])
logging.debug('host: %s' % (self.host, ))
logging.debug('path: %s' % (self.path, ))
logging.debug('nightly: %s' % (self.nightly, ))
logging.debug('srid: %s' % (self.srid, ))
logging.debug('output root: %s' % (self.outroot, ))
logging.debug('output directory: %s' % (self.outdir, ))
logging.debug('keep history: %s' % (self.keep, ))
logging.debug('max attempts: %s' % (self.max_attempts, ))
logging.debug('operating systems: %s' % (self.operating_systems, ))
logging.debug('netconfigs: %s' % (self.netconfigs, ))
logging.debug('ldap: %s' % (self.ldap, ))
logging.debug('sha: %s' % (self.sha, ))
logging.debug('attempt: %s' % (self.attempt, ))
self.prefix = ''
def main():
parser = stoneridge.ArgumentParser()
parser.parse_args()
osname = stoneridge.get_config('machine', 'os')
queue = stoneridge.CLIENT_QUEUES[osname]
while True:
try:
worker = StoneRidgeWorker(queue)
worker.run()
except:
logging.exception('Worker failed')
def run(self):
logging.debug('dns updater running')
if self.restore:
self._reset_dns()
return
logging.debug('Searching for dns server for netconfig %s' %
(self.netconfig,))
dns_server = stoneridge.get_config('dns', self.netconfig)
if dns_server is None:
logging.error('Error finding dns server')
return
self._set_dns(dns_server)
def run(self):
logging.debug('dns updater running')
if self.restore:
self._reset_dns()
return
logging.debug('Searching for dns server for netconfig %s' %
(self.netconfig, ))
dns_server = stoneridge.get_config('dns', self.netconfig)
if dns_server is None:
logging.error('Error finding dns server')
return
self._set_dns(dns_server)
def __init__(self, tests=None, heads=None):
"""tests - a subset of the tests to run
heads - js files that provide extra functionality
"""
# These we just copy over and worry about them later
self.tests = tests
self.heads = heads if heads else []
logging.debug('requested tests: %s' % (tests,))
logging.debug('heads: %s' % (heads,))
self.testroot = stoneridge.get_config('stoneridge', 'testroot')
self.unittest = stoneridge.get_config_bool('stoneridge', 'unittest')
logging.debug('testroot: %s' % (self.testroot,))
logging.debug('unittest: %s' % (self.unittest,))
def __init__(self):
self.host = stoneridge.get_config('cloner', 'host')
self.path = stoneridge.get_config('cloner', 'path')
self.outroot = stoneridge.get_config('server', 'downloads')
self.tstamp = time.strftime('%Y%m%d%H%M%S', time.gmtime())
self.outdir = os.path.join(self.outroot, self.tstamp)
self.latest = os.path.join(self.outroot, 'latest')
self.keep = stoneridge.get_config('server', 'keep', default=5)
if not os.path.exists(self.outroot):
os.mkdir(self.outroot)
logfile = os.path.join(self.outroot, 'cloner_%s.log' % (self.tstamp,))
logging.basicConfig(filename=logfile, level=logging.DEBUG,
format=stoneridge.log_fmt)
logging.debug('host: %s' % (self.host,))
logging.debug('path: %s' % (self.path,))
logging.debug('output root: %s' % (self.outroot,))
logging.debug('timestamp: %s' % (self.tstamp,))
logging.debug('output directory: %s' % (self.outdir,))
logging.debug('latest directory: %s' % (self.latest,))
logging.debug('keep history: %s' % (self.keep,))
self.prefix = ''
def __init__(self, tests=None, heads=None):
"""tests - a subset of the tests to run
heads - js files that provide extra functionality
"""
# These we just copy over and worry about them later
self.tests = tests
self.heads = heads if heads else []
logging.debug('requested tests: %s' % (tests, ))
logging.debug('heads: %s' % (heads, ))
self.testroot = stoneridge.get_config('stoneridge', 'testroot')
self.unittest = stoneridge.get_config_bool('stoneridge', 'unittest')
logging.debug('testroot: %s' % (self.testroot, ))
logging.debug('unittest: %s' % (self.unittest, ))
def __new__(self, *args, **kwargs):
# The caller shouldn't care what platform its running on, so we
# override __new__ to create the class that will unpack properly no
# matter what
os_name = stoneridge.get_config('machine', 'os')
if os_name == 'windows':
logging.debug('creating windows unpacker')
return object.__new__(WindowsUnpacker)
elif os_name == 'linux':
logging.debug('creating linux unpacker')
return object.__new__(LinuxUnpacker)
elif os_name == 'mac':
logging.debug('creating mac unpacker')
return object.__new__(MacUnpacker)
logging.critical('could not figure out what unpacker to create')
raise ValueError('Invalid system type: %s' % (os_name, ))
def __new__(self, *args, **kwargs):
# The caller shouldn't care what platform its running on, so we
# override __new__ to create the class that will unpack properly no
# matter what
os_name = stoneridge.get_config('machine', 'os')
if os_name == 'windows':
logging.debug('creating windows unpacker')
return object.__new__(WindowsUnpacker)
elif os_name == 'linux':
logging.debug('creating linux unpacker')
return object.__new__(LinuxUnpacker)
elif os_name == 'mac':
logging.debug('creating mac unpacker')
return object.__new__(MacUnpacker)
logging.critical('could not figure out what unpacker to create')
raise ValueError('Invalid system type: %s' % (os_name,))
def run(self):
logging.debug('archiver running')
outdir = stoneridge.get_config('run', 'out')
archivedir = stoneridge.get_config('stoneridge', 'archives')
srid = stoneridge.get_config('run', 'srid')
netconfig = stoneridge.get_config('run', 'netconfig')
infofile = stoneridge.get_config('run', 'info')
metadata = stoneridge.get_config('run', 'metadata')
with file(infofile, 'rb') as f:
info = json.load(f)
logging.debug('loaded info %s' % (info,))
arcname = 'stoneridge_%s_%s_%s_%s_%s' % (
info['date'],
info['test_machine']['name'],
info['test_build']['revision'],
srid,
netconfig)
logging.debug('archive name %s.zip' % (arcname,))
filename = os.path.join(archivedir, '%s.zip' % (arcname,))
if not os.path.exists(archivedir):
logging.debug('making archive directory %s' % (archivedir,))
os.mkdir(archivedir)
logging.debug('opening zip file for writing')
zfile = zipfile.ZipFile(filename, mode='w')
# Put all the files under a directory in the zip named for the zip
# file itself, for easy separation when unzipping multiple archives
# in the same place
logging.debug('adding files to zip')
for dirpath, dirs, files in os.walk(outdir):
dirname = dirpath.replace(outdir, arcname, 1)
logging.debug('directory %s -> %s' % (dirpath, dirname))
# Add the directories to the zip
for d in dirs:
logging.debug('subdirectory %s' % (d,))
zfile.write(os.path.join(dirpath, d),
arcname=os.path.join(dirname, d))
# Add the files to the zip
for f in files:
logging.debug('file %s' % (f,))
zfile.write(os.path.join(dirpath, f),
arcname=os.path.join(dirname, f))
logging.debug('closing zip file')
zfile.close()
# Make a copy where the uploader will find it
shutil.copyfile(filename, metadata)
def __init__(self):
self.server = stoneridge.get_config("download", "server")
self.downloadroot = stoneridge.get_config("download", "root")
self.download_platform = stoneridge.get_config("machine", "download_platform")
self.download_suffix = stoneridge.get_config("machine", "download_suffix")
self.srid = stoneridge.get_config("run", "srid")
self.downloaddir = stoneridge.get_config("run", "download")
logging.debug("server = %s" % (self.server,))
logging.debug("download root = %s" % (self.downloadroot,))
logging.debug("platform = %s" % (self.download_platform,))
logging.debug("suffix = %s" % (self.download_suffix,))
logging.debug("srid = %s" % (self.srid,))
logging.debug("downloaddir = %s" % (self.downloaddir,))
def run(self):
logging.debug('archiver running')
outdir = stoneridge.get_config('run', 'out')
archivedir = stoneridge.get_config('stoneridge', 'archives')
srid = stoneridge.get_config('run', 'srid')
netconfig = stoneridge.get_config('run', 'netconfig')
infofile = stoneridge.get_config('run', 'info')
metadata = stoneridge.get_config('run', 'metadata')
with file(infofile, 'rb') as f:
info = json.load(f)
logging.debug('loaded info %s' % (info, ))
arcname = 'stoneridge_%s_%s_%s_%s_%s' % (
info['date'], info['test_machine']['name'],
info['test_build']['revision'], srid, netconfig)
logging.debug('archive name %s.zip' % (arcname, ))
filename = os.path.join(archivedir, '%s.zip' % (arcname, ))
if not os.path.exists(archivedir):
logging.debug('making archive directory %s' % (archivedir, ))
os.mkdir(archivedir)
logging.debug('opening zip file for writing')
zfile = zipfile.ZipFile(filename, mode='w')
# Put all the files under a directory in the zip named for the zip
# file itself, for easy separation when unzipping multiple archives
# in the same place
logging.debug('adding files to zip')
for dirpath, dirs, files in os.walk(outdir):
dirname = dirpath.replace(outdir, arcname, 1)
logging.debug('directory %s -> %s' % (dirpath, dirname))
# Add the directories to the zip
for d in dirs:
logging.debug('subdirectory %s' % (d, ))
zfile.write(os.path.join(dirpath, d),
arcname=os.path.join(dirname, d))
# Add the files to the zip
for f in files:
logging.debug('file %s' % (f, ))
zfile.write(os.path.join(dirpath, f),
arcname=os.path.join(dirname, f))
logging.debug('closing zip file')
zfile.close()
# Make a copy where the uploader will find it
shutil.copyfile(filename, metadata)
def translate_path(self, path):
"""Override the base translate_path to get something with a configurable
root
"""
logging.debug('Translating path %s' % (path,))
rootdir = stoneridge.get_config('server', 'downloads')
path = path.split('?', 1)[0]
logging.debug('Without query string: %s' % (path,))
path = path.split('#', 1)[0]
logging.debug('Without fragment: %s' % (path,))
path = posixpath.normpath(urllib.unquote(path))
logging.debug('Normalized path: %s' % (path,))
words = [w for w in path.split('/') if w]
logging.debug('Non-empty path components: %s' % (words,))
path = rootdir
logging.debug('Root of translated path: %s' % (path,))
for w in words:
path = os.path.join(path, w)
logging.debug('Translated path: %s' % (path,))
return path
def __init__(self):
self.server = stoneridge.get_config('download', 'server')
self.downloadroot = stoneridge.get_config('download', 'root')
self.download_platform = stoneridge.get_config('machine',
'download_platform')
self.download_suffix = stoneridge.get_config('machine',
'download_suffix')
self.srid = stoneridge.get_config('run', 'srid')
self.downloaddir = stoneridge.get_config('run', 'download')
logging.debug('server = %s' % (self.server,))
logging.debug('download root = %s' % (self.downloadroot,))
logging.debug('platform = %s' % (self.download_platform,))
logging.debug('suffix = %s' % (self.download_suffix,))
logging.debug('srid = %s' % (self.srid,))
logging.debug('downloaddir = %s' % (self.downloaddir,))
def _build_testlist(self):
"""Return a list of test file names, all relative to the test root.
This weeds out any tests that may be missing from the directory.
"""
if not self.tests:
logging.debug('searching for all tests in %s' % (self.testroot, ))
if stoneridge.get_config('test', 'enabled'):
tests = []
if os.path.exists(os.path.join(self.testroot, 'fake.js')):
tests.append('fake.js')
else:
jstests = [
os.path.basename(f)
for f in glob.glob(os.path.join(self.testroot, '*.js'))
]
try:
jstests.remove('fake.js')
except ValueError:
# Don't care if fake.js isn't in the list
pass
pagetests = [
os.path.basename(f)
for f in glob.glob(os.path.join(self.testroot, '*.page'))
]
tests = jstests + pagetests
logging.debug('tests found %s' % (tests, ))
return tests
tests = []
for candidate in self.tests:
logging.debug('candidate test %s' % (candidate, ))
if not candidate.endswith('.js'):
logging.error('invalid test filename %s' % (candidate, ))
elif not os.path.exists(os.path.join(self.testroot, candidate)):
logging.error('missing test %s' % (candidate, ))
else:
logging.debug('valid test file %s' % (candidate, ))
tests.append(candidate)
logging.debug('tests selected %s' % (tests, ))
return tests
def _build_testlist(self):
"""Return a list of test file names, all relative to the test root.
This weeds out any tests that may be missing from the directory.
"""
if not self.tests:
logging.debug('searching for all tests in %s' %
(self.testroot,))
if stoneridge.get_config('test', 'enabled'):
tests = []
if os.path.exists(os.path.join(self.testroot, 'fake.js')):
tests.append('fake.js')
else:
jstests = [os.path.basename(f) for f in
glob.glob(os.path.join(self.testroot, '*.js'))]
try:
jstests.remove('fake.js')
except ValueError:
# Don't care if fake.js isn't in the list
pass
pagetests = [os.path.basename(f) for f in
glob.glob(os.path.join(self.testroot, '*.page'))]
tests = jstests + pagetests
logging.debug('tests found %s' % (tests,))
return tests
tests = []
for candidate in self.tests:
logging.debug('candidate test %s' % (candidate,))
if not candidate.endswith('.js'):
logging.error('invalid test filename %s' % (candidate,))
elif not os.path.exists(os.path.join(self.testroot, candidate)):
logging.error('missing test %s' % (candidate,))
else:
logging.debug('valid test file %s' % (candidate,))
tests.append(candidate)
logging.debug('tests selected %s' % (tests,))
return tests
def __init__(self):
self.rootdir = stoneridge.get_config('server', 'uploads')
self.pattern = os.path.join(self.rootdir, '*.json')
self.host = stoneridge.get_config('report', 'host')
self.project = stoneridge.get_config('report', 'project')
self.key = stoneridge.get_config('report', 'key')
self.secret = stoneridge.get_config('report', 'secret')
self.archives = stoneridge.get_config('report', 'archives')
logfile = os.path.join(self.archives, '%s.log' % (int(time.time()),))
logging.basicConfig(filename=logfile, level=logging.DEBUG,
format=stoneridge.log_fmt)
logging.debug('root directory: %s' % (self.rootdir,))
logging.debug('pattern: %s' % (self.pattern,))
logging.debug('host: %s' % (self.host,))
logging.debug('project: %s' % (self.project,))
logging.debug('oauth key: %s' % (self.key,))
logging.debug('oauth secret: %s' % (self.secret,))
logging.debug('archives: %s' % (self.archives,))
def run(self):
logging.debug('uploader running')
outdir = stoneridge.get_config('run', 'out')
pattern = os.path.join(outdir, 'upload_*.json')
files = glob.glob(pattern)
if not files:
# No results, but we still want to upload metadata for centralized
# debugging information.
logging.warning('no results to upload!')
results = {}
for filename in files:
fname = os.path.basename(filename)
with file(filename) as f:
results[fname] = json.load(f)
metadata_file = stoneridge.get_config('run', 'metadata')
if os.path.exists(metadata_file):
with file(metadata_file, 'rb') as f:
contents = f.read()
metadata = base64.b64encode(contents)
else:
# Missing metadata, but we can still report results
logging.warning('missing metadata, continuing anyway')
metadata = base64.b64encode('')
srid = stoneridge.get_config('run', 'srid')
netconfig = stoneridge.get_config('run', 'netconfig')
ldap = stoneridge.get_config('run', 'ldap')
operating_system = stoneridge.get_config('machine', 'os')
self.queue.enqueue(srid=srid,
results=results,
metadata=metadata,
netconfig=netconfig,
operating_system=operating_system,
ldap=ldap)
def run(self):
logging.debug('unpacker running')
# Get our firefox
logging.debug('unpacking firefox')
self.unpack_firefox()
# Unzip the stuff we need from the tests zipfile
unzipdir = os.path.join(self.workdir, 'tests')
logging.debug('creating unzip dir %s' % (unzipdir, ))
os.mkdir(unzipdir)
z = zipfile.ZipFile(self.testzip, 'r')
members = [f for f in z.namelist() if f.startswith('bin')]
logging.debug('unzipping %s' % (str(members), ))
z.extractall(unzipdir, members)
# Put the xpcshell binary where it belongs
unzipbin = os.path.join(unzipdir, 'bin')
xpcshell_bin = stoneridge.get_config('machine', 'xpcshell')
xpcshell = os.path.join(unzipbin, xpcshell_bin)
logging.debug('xpcshell: %s' % (xpcshell, ))
# Apparently xpcshell stopped being executable in the tests zip at some
# point, so we need to fix that before copying
logging.debug('setting permissions on xpcshell')
os.chmod(xpcshell, 0755)
logging.debug('copy xpcshell %s -> %s' % (xpcshell, self.bindir))
shutil.copy(xpcshell, self.bindir)
# Put our components into place
logging.debug('copying components')
self._copy_tree(unzipbin, 'components')
# Put the plugins in place, in case we need them
logging.debug('copying plugins')
self._copy_tree(unzipbin, 'plugins')
# Put the pageloader components into place
srroot = stoneridge.get_config('stoneridge', 'root')
pageloader = os.path.join(srroot, 'pageloader')
self._copy_tree(pageloader, 'components')
self._copy_tree(pageloader, 'chrome')
# Now we need to put srdata.js into the appropriate place for it to be
# picked up by the pageloader
chrome = os.path.join(self.bindir, 'chrome')
srdatasrc = os.path.join(srroot, 'srdata.js')
srdatadst = os.path.join(chrome, 'srdata.js')
if os.path.exists(srdatadst):
os.unlink(srdatadst)
logging.debug('copy srdata.js %s -> %s' % (srdatasrc, srdatadst))
shutil.copyfile(srdatasrc, srdatadst)
# Finally, we need to update chrome.manifest with the appropriate bits
# from our local pageloader
plmanifest = os.path.join(pageloader, 'chrome.manifest')
fxmanifest = os.path.join(self.bindir, 'chrome.manifest')
logging.debug('append %s to %s' % (plmanifest, fxmanifest))
if os.path.exists(fxmanifest):
with file(fxmanifest, 'rb') as f:
lines = f.readlines()
else:
lines = []
with file(plmanifest, 'rb') as f:
lines.extend(f.readlines())
with file(fxmanifest, 'wb') as f:
f.writelines(lines)
def __init__(self):
self.url = stoneridge.get_config('mqproxy', 'url')
self.setup()
def run(self):
logging.debug('runner running')
tests = self._build_testlist()
preargs = self._build_preargs()
logging.debug('tests to run: %s' % (tests, ))
logging.debug('args to prepend: %s' % (preargs, ))
# Ensure our output directory exists
outdir = stoneridge.get_config('run', 'out')
profile = stoneridge.get_config('run', 'profile')
installroot = stoneridge.get_config('stoneridge', 'root')
for test in tests:
logging.debug('test: %s' % (test, ))
outfile = os.path.join(outdir, '%s.out' % (test, ))
logging.debug('outfile: %s' % (outfile, ))
if test.endswith('.js'):
escaped_outfile = outfile.replace('\\', '\\\\')
args = preargs + [
'-e',
'const _SR_OUT_FILE = "%s";' % (escaped_outfile, ), '-f',
os.path.join(installroot, 'srdata.js'), '-f',
os.path.join(installroot, 'head.js'), '-f',
os.path.join(self.testroot, test), '-e',
'do_stoneridge(); quit(0);'
]
logging.debug('xpcshell args: %s' % (args, ))
runner = stoneridge.run_xpcshell
else:
args = [
'-sr',
os.path.join(self.testroot, test),
'-sroutput',
outfile,
'-profile',
profile,
# -srwidth, <some width value>,
# -srheight, <some height value>,
# -srtimeout, <some timeout value per page>,
# -srdelay, <some delay value between pages>,
# -srmozafterpaint
]
runner = stoneridge.run_firefox
if self.unittest:
logging.debug('Not running processes: in unit test mode')
else:
process_out_file = '%s.process.out' % (test, )
process_out_file = os.path.join(outdir, process_out_file)
logging.debug('process output at %s' % (process_out_file, ))
timed_out = False
with file(process_out_file, 'wb') as f:
try:
res = runner(args, f)
except stoneridge.TestProcessTimeout:
logging.exception('test process timed out!')
timed_out = True
res = None
if res or timed_out:
logging.error('TEST FAILED: %s' % (test, ))
else:
logging.debug('test succeeded')
def run(self):
logging.debug('collator running')
outdir = stoneridge.get_config('run', 'out')
outfiles = glob.glob(os.path.join(outdir, '*.js.out'))
outfiles.extend(glob.glob(os.path.join(outdir, '*.page.out')))
infofile = stoneridge.get_config('run', 'info')
logging.debug('found outfiles %s' % (outfiles,))
logging.debug('loading info from %s' % (infofile,))
with file(infofile, 'rb') as f:
info = json.load(f)
logging.debug('loaded info: %s' % (info,))
for ofile in outfiles:
logging.debug('processing %s' % (ofile,))
# Make a new copy of the base info
results = copy.deepcopy(info)
del results['date']
results['testrun'] = {'date': info['date'],
'suite': None,
'options': {}}
results['results'] = collections.defaultdict(list)
results['results_aux'] = collections.defaultdict(list)
logging.debug('initial testrun: %s' % (results['testrun'],))
# Figure out the test-specific data
fname = os.path.basename(ofile)
suite = fname.split('.')[0]
results['testrun']['suite'] = suite
ldap = stoneridge.get_config('run', 'ldap')
if ldap is None:
results['testrun']['options']['ldap'] = 'nightly'
else:
results['testrun']['options']['ldap'] = ldap
logging.debug('suite: %s' % (suite,))
# Read the raw data
logging.debug('reading raw data')
with file(ofile, 'rb') as f:
testinfo = json.load(f)
logging.debug('raw testinfo: %s' % (testinfo,))
# Stick the raw data into the json to be uploaded
logging.debug('processing raw data')
for k, vlist in testinfo.items():
logging.debug('k: %s, vlist: %s' % (k, vlist))
for v in vlist:
logging.debug('v: %s' % (v,))
if k == 'total':
logging.debug('appending total %s' % (v['total'],))
# The graph server calculates totals for us, we just
# keep our calculations around for verification in case
results['results_aux']['totals'].append(v['total'])
else:
logging.debug('appending %s total %s' %
(k, v['total']))
results['results'][k].append(v['total'])
for s in ('start', 'stop'):
key = '%s_%s' % (k, s)
logging.debug('appending %s %s stamp %s' %
(k, s, v[s]))
results['results_aux'][key].append(v[s])
# Turn our defaultdicts into regular dicts for jsonification
results['results'] = dict(results['results'])
results['results_aux'] = dict(results['results_aux'])
logging.debug('results: %s' % (results['results'],))
logging.debug('aux results: %s' % (results['results_aux'],))
# Write our json results for uploading
upload_filename = 'upload_%s.json' % (suite,)
logging.debug('upload filename: %s' % (upload_filename,))
upload_file = os.path.join(outdir, upload_filename)
with file(upload_file, 'wb') as f:
logging.debug('jsonifying %s' % (results,))
json.dump(results, f)
def __init__(self):
self.workdir = stoneridge.get_config('stoneridge', 'work')
self.keep = stoneridge.get_config_int('cleaner', 'keep')
def __init__(self):
self.workdir = stoneridge.get_config('stoneridge', 'work')
self.keep = stoneridge.get_config_int('cleaner', 'keep')
def __init__(self):
self.url = stoneridge.get_config('upload', 'url')
logging.debug('upload url: %s' % (self.url,))
def handle(self, srid, netconfig, tstamp, ldap):
# Create the directory where data we want to save from this run will go
srwork = os.path.join(self.workroot, srid, netconfig)
if os.path.exists(srwork):
srwork = '%s_%s' % (srwork, tstamp)
os.makedirs(srwork)
srout = os.path.join(srwork, 'out')
os.mkdir(srout)
# Have a logger just for this run
self.logdir = os.path.join(srout, 'logs')
os.makedirs(self.logdir)
logging.debug('Running test with logs in %s' % (self.logdir, ))
logfile = os.path.join(self.logdir, '00_worker.log')
handler = logging.FileHandler(logfile)
formatter = logging.Formatter(fmt=stoneridge.LOG_FMT)
handler.setFormatter(formatter)
self.logger = logging.getLogger('%s_%s_%s' % (srid, netconfig, tstamp))
self.logger.setLevel(logging.DEBUG)
self.logger.addHandler(handler)
# Create the rest of the working space for this run
srdownload = os.path.join(srwork, 'download')
os.mkdir(srdownload)
firefox_path = stoneridge.get_config('machine', 'firefox_path')
srbindir = os.path.join(srwork, firefox_path)
metadata = os.path.join(srout, 'metadata.zip')
info = os.path.join(srout, 'info.json')
profile = os.path.join(srout, 'profile')
os.mkdir(profile)
self.srnetconfig = netconfig
self.uploaded = False
self.archive_on_failure = True
self.procno = 1
self.childlog = None
self.need_dns_reset = False
self.runconfig = os.path.join(srout, 'run.ini')
with file(self.runconfig, 'w') as f:
f.write('[run]\n')
f.write('netconfig = %s\n' % (netconfig, ))
f.write('work = %s\n' % (srwork, ))
f.write('download = %s\n' % (srdownload, ))
f.write('bin = %s\n' % (srbindir, ))
f.write('out = %s\n' % (srout, ))
f.write('profile = %s\n' % (profile, ))
f.write('metadata = %s\n' % (metadata, ))
f.write('info = %s\n' % (info, ))
f.write('tstamp = %s\n' % (tstamp, ))
f.write('srid = %s\n' % (srid, ))
if ldap:
f.write('ldap = %s\n' % (ldap, ))
self.logger.debug('srnetconfig: %s' % (self.srnetconfig, ))
self.logger.debug('uploaded: %s' % (self.uploaded, ))
self.logger.debug('archive on failure: %s' %
(self.archive_on_failure, ))
self.logger.debug('procno: %s' % (self.procno, ))
self.logger.debug('childlog: %s' % (self.childlog, ))
self.logger.debug('logdir: %s' % (self.logdir, ))
self.logger.debug('runconfig: %s' % (self.runconfig, ))
self.logger.debug('ldap: %s' % (ldap, ))
self.logger.debug('profile: %s' % (profile, ))
try:
self.run_test()
except StoneRidgeException as e:
self.logger.exception(e)
self.reset()
def __init__(self):
self.pcaps = {}
self.macaddr = stoneridge.get_config('machine', 'macaddr')
self.tcpdump = stoneridge.get_config('tcpdump', 'exe')
self.interface = stoneridge.get_config('tcpdump', 'interface')
def setup(self):
self.dbfile = stoneridge.get_config('mqproxy', 'db')
self.conn = sqlite3.connect(self.dbfile)
