def run(self): logging.debug('runner running') tests = self._build_testlist() preargs = self._build_preargs() logging.debug('tests to run: %s' % (tests,)) logging.debug('args to prepend: %s' % (preargs,)) # Ensure our output directory exists outdir = stoneridge.get_config('run', 'out') profile = stoneridge.get_config('run', 'profile') installroot = stoneridge.get_config('stoneridge', 'root') for test in tests: logging.debug('test: %s' % (test,)) outfile = os.path.join(outdir, '%s.out' % (test,)) logging.debug('outfile: %s' % (outfile,)) if test.endswith('.js'): escaped_outfile = outfile.replace('\\', '\\\\') args = preargs + [ '-e', 'const _SR_OUT_FILE = "%s";' % (escaped_outfile,), '-f', os.path.join(installroot, 'srdata.js'), '-f', os.path.join(installroot, 'head.js'), '-f', os.path.join(self.testroot, test), '-e', 'do_stoneridge(); quit(0);' ] logging.debug('xpcshell args: %s' % (args,)) runner = stoneridge.run_xpcshell else: args = [ '-sr', os.path.join(self.testroot, test), '-sroutput', outfile, '-profile', profile, # -srwidth, <some width value>, # -srheight, <some height value>, # -srtimeout, <some timeout value per page>, # -srdelay, <some delay value between pages>, # -srmozafterpaint ] runner = stoneridge.run_firefox if self.unittest: logging.debug('Not running processes: in unit test mode') else: process_out_file = '%s.process.out' % (test,) process_out_file = os.path.join(outdir, process_out_file) logging.debug('process output at %s' % (process_out_file,)) timed_out = False with file(process_out_file, 'wb') as f: try: res = runner(args, f) except stoneridge.TestProcessTimeout: logging.exception('test process timed out!') timed_out = True res = None if res or timed_out: logging.error('TEST FAILED: %s' % (test,)) else: logging.debug('test succeeded')
def __init__(self): self.workdir = stoneridge.get_config('run', 'work') logging.debug('work directory: %s' % (self.workdir, )) self.bindir = stoneridge.get_config('run', 'bin') logging.debug('bin directory: %s' % (self.bindir, )) downloaddir = stoneridge.get_config('run', 'download') download_suffix = stoneridge.get_config('machine', 'download_suffix') self.firefoxpkg = os.path.join(downloaddir, 'firefox.%s' % (download_suffix, )) logging.debug('firefox package: %s' % (self.firefoxpkg, )) self.testzip = os.path.join(downloaddir, 'tests.zip') logging.debug('test zip file: %s' % (self.testzip, ))
def __init__(self): self.workdir = stoneridge.get_config('run', 'work') logging.debug('work directory: %s' % (self.workdir,)) self.bindir = stoneridge.get_config('run', 'bin') logging.debug('bin directory: %s' % (self.bindir,)) downloaddir = stoneridge.get_config('run', 'download') download_suffix = stoneridge.get_config('machine', 'download_suffix') self.firefoxpkg = os.path.join(downloaddir, 'firefox.%s' % (download_suffix,)) logging.debug('firefox package: %s' % (self.firefoxpkg,)) self.testzip = os.path.join(downloaddir, 'tests.zip') logging.debug('test zip file: %s' % (self.testzip,))
def setup(self): self.host = stoneridge.get_config('report', 'host') self.project = stoneridge.get_config('report', 'project') self.key = stoneridge.get_config('report', 'key') self.secret = stoneridge.get_config('report', 'secret') self.archives = stoneridge.get_config('stoneridge', 'archives') self.unittest = stoneridge.get_config_bool('stoneridge', 'unittest') logging.debug('report host: %s' % (self.host,)) logging.debug('project: %s' % (self.project,)) logging.debug('oauth key: %s' % (self.key,)) logging.debug('oauth secret: %s' % (self.secret,)) logging.debug('archives: %s' % (self.archives,)) logging.debug('unittest: %s' % (self.unittest,))
def setup(self): self.host = stoneridge.get_config('report', 'host') self.project = stoneridge.get_config('report', 'project') self.key = stoneridge.get_config('report', 'key') self.secret = stoneridge.get_config('report', 'secret') self.archives = stoneridge.get_config('stoneridge', 'archives') self.unittest = stoneridge.get_config_bool('stoneridge', 'unittest') logging.debug('report host: %s' % (self.host, )) logging.debug('project: %s' % (self.project, )) logging.debug('oauth key: %s' % (self.key, )) logging.debug('oauth secret: %s' % (self.secret, )) logging.debug('archives: %s' % (self.archives, )) logging.debug('unittest: %s' % (self.unittest, ))
def __init__(self, restore): self.restore = restore self.peer = ('127.0.0.1', 63250) os_name = stoneridge.get_config('machine', 'os') if os_name == 'windows': self.is_windows = True else: self.is_windows = False self.netconfig = stoneridge.get_config('run', 'netconfig') self.unittest = stoneridge.get_config_bool('stoneridge', 'unittest') logging.debug('restore: %s' % (restore,)) logging.debug('peer: %s' % (self.peer,)) logging.debug('is windows: %s' % (self.is_windows,)) logging.debug('netconfig: %s' % (self.netconfig,)) logging.debug('unittest: %s' % (self.unittest,))
def __init__(self, restore): self.restore = restore self.peer = ('127.0.0.1', 63250) os_name = stoneridge.get_config('machine', 'os') if os_name == 'windows': self.is_windows = True else: self.is_windows = False self.netconfig = stoneridge.get_config('run', 'netconfig') self.unittest = stoneridge.get_config_bool('stoneridge', 'unittest') logging.debug('restore: %s' % (restore, )) logging.debug('peer: %s' % (self.peer, )) logging.debug('is windows: %s' % (self.is_windows, )) logging.debug('netconfig: %s' % (self.netconfig, )) logging.debug('unittest: %s' % (self.unittest, ))
def run(self): logging.debug('info gatherer running') bindir = stoneridge.get_config('run', 'bin') info_file = os.path.join(bindir, 'application.ini') logging.debug('parsing ini file at %s' % (info_file, )) cp = ConfigParser.SafeConfigParser() cp.read([info_file]) build_info = {} build_info['name'] = cp.get('App', 'Name') build_info['version'] = cp.get('App', 'Version') build_info['revision'] = cp.get('App', 'SourceStamp') build_info['branch'] = stoneridge.get_config('run', 'netconfig') # Due to the way the graph server works, we need to create a unique # build id for each build/os/netconfig combination. We also want to # keep the unmodified build ID around for posterity. build_info['original_buildid'] = cp.get('App', 'BuildID') # Build ID is limited to 16 characters in the receiving database, and # our suffix is 2 characters, so we truncate the original to 14 # characters before adding our suffix. It should already be only 14 # characters, but we do this Just In Case. buildid_base = build_info['original_buildid'][:14] build_info['id'] = buildid_base + stoneridge.get_buildid_suffix() machine_info = {} machine_info['name'] = platform.node() machine_info['os'] = stoneridge.get_config('machine', 'os') machine_info['osversion'] = stoneridge.get_os_version() machine_info['platform'] = platform.machine() info = { 'test_machine': machine_info, 'test_build': build_info, 'testrun': {}, 'date': stoneridge.get_config_int('run', 'tstamp') } logging.debug('gathered info: %s' % (info, )) outdir = stoneridge.get_config('run', 'out') if not os.path.exists(outdir): logging.debug('making outdir %s' % (outdir, )) os.mkdir(outdir) with file(os.path.join(outdir, 'info.json'), 'wb') as f: logging.debug('dumping json to file') json.dump(info, f)
def do_POST(self): """Handle getting uploaded results from the clients """ logging.debug('Handling request from %s' % (self.client_address,)) logging.debug('Path: %s' % (self.path,)) rootdir = stoneridge.get_config('server', 'uploads') now = int(time.time()) idx = 0 post = cgi.FieldStorage(fp=self.rfile, headers=self.headers, environ={'REQUEST_METHOD':'POST', 'CONTENT_TYPE':self.headers['Content-Type']}) for k in post.keys(): v = post[k].value logging.debug('Processing post variable %s' % (k,)) logging.debug('%s value = %s' % (k, v)) if len(v) < 10: # Stupid check to filter out junk data that gets submitted from # time to time, until we can figure out why said junk data gets # submitted logging.debug('Skipping invalid json value') continue pfx = '%s_%s_' % (now, idx) idx += 1 fd, name = tempfile.mkstemp(dir=rootdir, prefix=pfx, suffix='.json') os.write(fd, v) os.fchmod(fd, 0644) os.close(fd) self.send_response(200)
def __init__(self, stop): self.stop = stop self.macaddr = stoneridge.get_config('machine', 'macaddr') self.netconfig = stoneridge.get_config('run', 'netconfig') self.host = stoneridge.get_config('tcpdump', self.netconfig) self.outdir = stoneridge.get_config('run', 'out') self.stdout = os.path.join(self.outdir, 'tcpdump.out') self.pcap = os.path.join(self.outdir, 'traffic.pcap') logging.debug('stop: %s' % (stop,)) logging.debug('macaddr: %s' % (self.macaddr,)) logging.debug('netconfig: %s' % (self.netconfig,)) logging.debug('host: %s' % (self.host,)) logging.debug('output directory: %s' % (self.outdir,)) logging.debug('tcpdump output: %s' % (self.stdout,)) logging.debug('packet capture: %s' % (self.pcap,))
def __init__(self, stop): self.stop = stop self.macaddr = stoneridge.get_config('machine', 'macaddr') self.netconfig = stoneridge.get_config('run', 'netconfig') self.host = stoneridge.get_config('tcpdump', self.netconfig) self.outdir = stoneridge.get_config('run', 'out') self.stdout = os.path.join(self.outdir, 'tcpdump.out') self.pcap = os.path.join(self.outdir, 'traffic.pcap') logging.debug('stop: %s' % (stop, )) logging.debug('macaddr: %s' % (self.macaddr, )) logging.debug('netconfig: %s' % (self.netconfig, )) logging.debug('host: %s' % (self.host, )) logging.debug('output directory: %s' % (self.outdir, )) logging.debug('tcpdump output: %s' % (self.stdout, )) logging.debug('packet capture: %s' % (self.pcap, ))
def run(self): logging.debug('info gatherer running') bindir = stoneridge.get_config('run', 'bin') info_file = os.path.join(bindir, 'application.ini') logging.debug('parsing ini file at %s' % (info_file,)) cp = ConfigParser.SafeConfigParser() cp.read([info_file]) build_info = {} build_info['name'] = cp.get('App', 'Name') build_info['version'] = cp.get('App', 'Version') build_info['revision'] = cp.get('App', 'SourceStamp') build_info['branch'] = stoneridge.get_config('run', 'netconfig') # Due to the way the graph server works, we need to create a unique # build id for each build/os/netconfig combination. We also want to # keep the unmodified build ID around for posterity. build_info['original_buildid'] = cp.get('App', 'BuildID') # Build ID is limited to 16 characters in the receiving database, and # our suffix is 2 characters, so we truncate the original to 14 # characters before adding our suffix. It should already be only 14 # characters, but we do this Just In Case. buildid_base = build_info['original_buildid'][:14] build_info['id'] = buildid_base + stoneridge.get_buildid_suffix() machine_info = {} machine_info['name'] = platform.node() machine_info['os'] = stoneridge.get_config('machine', 'os') machine_info['osversion'] = stoneridge.get_os_version() machine_info['platform'] = platform.machine() info = {'test_machine': machine_info, 'test_build': build_info, 'testrun': {}, 'date': stoneridge.get_config_int('run', 'tstamp')} logging.debug('gathered info: %s' % (info,)) outdir = stoneridge.get_config('run', 'out') if not os.path.exists(outdir): logging.debug('making outdir %s' % (outdir,)) os.mkdir(outdir) with file(os.path.join(outdir, 'info.json'), 'wb') as f: logging.debug('dumping json to file') json.dump(info, f)
def daemon(): global conn dbfile = stoneridge.get_config('mqproxy', 'db') conn = sqlite3.connect(dbfile) stoneridge.StreamLogger.bottle_inject() port = stoneridge.get_config_int('mqproxy', 'port') bottle.run(host='0.0.0.0', port=port)
def setup(self): self.srconffile = stoneridge.get_config_file() self.unittest = stoneridge.get_config_bool('stoneridge', 'unittest') self.workroot = stoneridge.get_config('stoneridge', 'work') logging.debug('srconffile: %s' % (self.srconffile, )) logging.debug('unittest: %s' % (self.unittest, )) self.runconfig = None # Needs to be here so reset doesn't barf self.reset()
def setup(self): self.srconffile = stoneridge.get_config_file() self.unittest = stoneridge.get_config_bool('stoneridge', 'unittest') self.workroot = stoneridge.get_config('stoneridge', 'work') logging.debug('srconffile: %s' % (self.srconffile,)) logging.debug('unittest: %s' % (self.unittest,)) self.runconfig = None # Needs to be here so reset doesn't barf self.reset()
def unpack_firefox(self): # MAC, Y U NO USE REGULAR ARCHIVE?! installroot = stoneridge.get_config('stoneridge', 'root') installdmg = os.path.join(installroot, 'installdmg.sh') logging.debug('mac using installdmg at %s' % (installdmg,)) out = subprocess.check_output(['/bin/bash', installdmg, self.firefoxpkg], cwd=self.workdir, stderr=subprocess.STDOUT) logging.debug(out)
def send_email(check): """Send an email to me so I know when the dns update failed. """ myos = stoneridge.get_config('machine', 'os') netconfig = stoneridge.get_config('run', 'netconfig') srid = stoneridge.get_config('run', 'srid') logging.debug('sending email: os=%s, netconfig=%s, srid=%s, check=%s' % (myos, netconfig, srid, check)) to = '*****@*****.**' subject = 'DNS Update Failed' msg = '''The DNS Update failed for the following run: OS: %s Netconfig: %s SRID: %s Check failed: %s ''' % (myos, netconfig, srid, check) stoneridge.mail(to, subject, msg)
def unpack_firefox(self): # MAC, Y U NO USE REGULAR ARCHIVE?! installroot = stoneridge.get_config('stoneridge', 'root') installdmg = os.path.join(installroot, 'installdmg.sh') logging.debug('mac using installdmg at %s' % (installdmg, )) out = subprocess.check_output( ['/bin/bash', installdmg, self.firefoxpkg], cwd=self.workdir, stderr=subprocess.STDOUT) logging.debug(out)
def setup(self): self.queues = { 'broadband': stoneridge.QueueWriter( stoneridge.NETCONFIG_QUEUES['broadband']), 'umts': stoneridge.QueueWriter( stoneridge.NETCONFIG_QUEUES['umts']), 'gsm': stoneridge.QueueWriter( stoneridge.NETCONFIG_QUEUES['gsm']) } self.logdir = stoneridge.get_config('stoneridge', 'logs') self.config = stoneridge.get_config_file()
def main(): parser = stoneridge.TestRunArgumentParser() parser.parse_args() netconfig = stoneridge.get_config('run', 'netconfig') server = stoneridge.get_config('dns', netconfig) myos = stoneridge.get_config('machine', 'os') logging.debug('netconfig: %s' % (netconfig, )) logging.debug('server: %s' % (server, )) logging.debug('os: %s' % (myos, )) if myos == 'windows': countarg = '-n' else: countarg = '-c' logging.debug('countarg: %s' % (countarg, )) logging.debug('Give me a ping, Vasili. One ping only, please.') p = stoneridge.Process(['ping', countarg, '1', server]) p.communicate() p.wait()
def main(): parser = stoneridge.TestRunArgumentParser() parser.parse_args() netconfig = stoneridge.get_config('run', 'netconfig') server = stoneridge.get_config('dns', netconfig) myos = stoneridge.get_config('machine', 'os') logging.debug('netconfig: %s' % (netconfig,)) logging.debug('server: %s' % (server,)) logging.debug('os: %s' % (myos,)) if myos == 'windows': countarg = '-n' else: countarg = '-c' logging.debug('countarg: %s' % (countarg,)) logging.debug('Give me a ping, Vasili. One ping only, please.') p = stoneridge.Process(['ping', countarg, '1', server]) p.communicate() p.wait()
def main(): parser = stoneridge.DaemonArgumentParser() parser.add_argument('--nochange', dest='nochange', action='store_true') args = parser.parse_args() global nochange nochange = args.nochange global rundir rundir = stoneridge.get_config('stoneridge', 'run') parser.start_daemon(daemon)
def run(self): logging.debug('uploader running') outdir = stoneridge.get_config('run', 'out') pattern = os.path.join(outdir, 'upload_*.json') files = glob.glob(pattern) if not files: # No results, but we still want to upload metadata for centralized # debugging information. logging.warning('no results to upload!') results = {} for filename in files: fname = os.path.basename(filename) with file(filename) as f: results[fname] = json.load(f) metadata_file = stoneridge.get_config('run', 'metadata') if os.path.exists(metadata_file): with file(metadata_file, 'rb') as f: contents = f.read() metadata = base64.b64encode(contents) else: # Missing metadata, but we can still report results logging.warning('missing metadata, continuing anyway') metadata = base64.b64encode('') srid = stoneridge.get_config('run', 'srid') netconfig = stoneridge.get_config('run', 'netconfig') ldap = stoneridge.get_config('run', 'ldap') operating_system = stoneridge.get_config('machine', 'os') self.queue.enqueue(srid=srid, results=results, metadata=metadata, netconfig=netconfig, operating_system=operating_system, ldap=ldap)
def main(): parser = stoneridge.ArgumentParser() parser.parse_args() root = stoneridge.get_config('enqueuer', 'root') username = stoneridge.get_config('enqueuer', 'username') password = stoneridge.get_config('enqueuer', 'password') try: res = requests.get(root + '/list_unhandled', auth=(username, password)) except: # For some reason, we sometimes get a requests failure here, even # though everything seems to be working fine. Ignore that, and try # again later. logging.exception('Error listing unhandled pushes') return try: queue = json.loads(res.text) except: # Yet another failure mode. Yay. logging.exception('Error demarshalling result %s' % (res.text,)) return for entry in queue: try: requests.post(root + '/mark_handled', data={'id': entry['pushid']}, auth=(username, password)) except: # If we fail to mark this as handled, wait until the next try so we # don't run the same thing more than once. It's not the end of the # world ot have to wait... logging.exception('Error marking entry %s as handled' % (entry,)) return logging.debug('Equeuing entry %s' % (entry,)) stoneridge.enqueue(nightly=False, ldap=entry['ldap'], sha=entry['sha'], netconfigs=entry['netconfigs'], operating_systems=entry['operating_systems'], srid=entry['srid'])
def __init__(self, nightly, srid, operating_systems, netconfigs, ldap, sha, attempt): self.host = stoneridge.get_config('cloner', 'host') self.nightly = nightly self.outroot = stoneridge.get_config('cloner', 'output') self.srid = srid self.outdir = os.path.join(self.outroot, srid) self.keep = stoneridge.get_config_int('cloner', 'keep', default=50) self.max_attempts = stoneridge.get_config_int('cloner', 'attempts') self.operating_systems = operating_systems self.netconfigs = netconfigs self.ldap = ldap self.sha = sha self.attempt = attempt if not os.path.exists(self.outroot): os.mkdir(self.outroot) root = stoneridge.get_config('cloner', 'root') if nightly: self.path = '/'.join([root, 'nightly', 'latest-mozilla-central']) else: self.path = '/'.join([root, 'try-builds', '%s-%s' % (ldap, sha)]) logging.debug('host: %s' % (self.host,)) logging.debug('path: %s' % (self.path,)) logging.debug('nightly: %s' % (self.nightly,)) logging.debug('srid: %s' % (self.srid,)) logging.debug('output root: %s' % (self.outroot,)) logging.debug('output directory: %s' % (self.outdir,)) logging.debug('keep history: %s' % (self.keep,)) logging.debug('max attempts: %s' % (self.max_attempts,)) logging.debug('operating systems: %s' % (self.operating_systems,)) logging.debug('netconfigs: %s' % (self.netconfigs,)) logging.debug('ldap: %s' % (self.ldap,)) logging.debug('sha: %s' % (self.sha,)) logging.debug('attempt: %s' % (self.attempt,)) self.prefix = ''
def main(): parser = stoneridge.ArgumentParser() parser.parse_args() osname = stoneridge.get_config('machine', 'os') queue = stoneridge.CLIENT_QUEUES[osname] while True: try: worker = StoneRidgeWorker(queue) worker.run() except: logging.exception('Worker failed')
def __init__(self, nightly, srid, operating_systems, netconfigs, ldap, sha, attempt): self.host = stoneridge.get_config('cloner', 'host') self.nightly = nightly self.outroot = stoneridge.get_config('cloner', 'output') self.srid = srid self.outdir = os.path.join(self.outroot, srid) self.keep = stoneridge.get_config_int('cloner', 'keep', default=50) self.max_attempts = stoneridge.get_config_int('cloner', 'attempts') self.operating_systems = operating_systems self.netconfigs = netconfigs self.ldap = ldap self.sha = sha self.attempt = attempt if not os.path.exists(self.outroot): os.mkdir(self.outroot) root = stoneridge.get_config('cloner', 'root') if nightly: self.path = '/'.join([root, 'nightly', 'latest-mozilla-central']) else: self.path = '/'.join([root, 'try-builds', '%s-%s' % (ldap, sha)]) logging.debug('host: %s' % (self.host, )) logging.debug('path: %s' % (self.path, )) logging.debug('nightly: %s' % (self.nightly, )) logging.debug('srid: %s' % (self.srid, )) logging.debug('output root: %s' % (self.outroot, )) logging.debug('output directory: %s' % (self.outdir, )) logging.debug('keep history: %s' % (self.keep, )) logging.debug('max attempts: %s' % (self.max_attempts, )) logging.debug('operating systems: %s' % (self.operating_systems, )) logging.debug('netconfigs: %s' % (self.netconfigs, )) logging.debug('ldap: %s' % (self.ldap, )) logging.debug('sha: %s' % (self.sha, )) logging.debug('attempt: %s' % (self.attempt, )) self.prefix = ''
def run(self): logging.debug('dns updater running') if self.restore: self._reset_dns() return logging.debug('Searching for dns server for netconfig %s' % (self.netconfig,)) dns_server = stoneridge.get_config('dns', self.netconfig) if dns_server is None: logging.error('Error finding dns server') return self._set_dns(dns_server)
def run(self): logging.debug('dns updater running') if self.restore: self._reset_dns() return logging.debug('Searching for dns server for netconfig %s' % (self.netconfig, )) dns_server = stoneridge.get_config('dns', self.netconfig) if dns_server is None: logging.error('Error finding dns server') return self._set_dns(dns_server)
def __init__(self, tests=None, heads=None): """tests - a subset of the tests to run heads - js files that provide extra functionality """ # These we just copy over and worry about them later self.tests = tests self.heads = heads if heads else [] logging.debug('requested tests: %s' % (tests,)) logging.debug('heads: %s' % (heads,)) self.testroot = stoneridge.get_config('stoneridge', 'testroot') self.unittest = stoneridge.get_config_bool('stoneridge', 'unittest') logging.debug('testroot: %s' % (self.testroot,)) logging.debug('unittest: %s' % (self.unittest,))
def __init__(self): self.host = stoneridge.get_config('cloner', 'host') self.path = stoneridge.get_config('cloner', 'path') self.outroot = stoneridge.get_config('server', 'downloads') self.tstamp = time.strftime('%Y%m%d%H%M%S', time.gmtime()) self.outdir = os.path.join(self.outroot, self.tstamp) self.latest = os.path.join(self.outroot, 'latest') self.keep = stoneridge.get_config('server', 'keep', default=5) if not os.path.exists(self.outroot): os.mkdir(self.outroot) logfile = os.path.join(self.outroot, 'cloner_%s.log' % (self.tstamp,)) logging.basicConfig(filename=logfile, level=logging.DEBUG, format=stoneridge.log_fmt) logging.debug('host: %s' % (self.host,)) logging.debug('path: %s' % (self.path,)) logging.debug('output root: %s' % (self.outroot,)) logging.debug('timestamp: %s' % (self.tstamp,)) logging.debug('output directory: %s' % (self.outdir,)) logging.debug('latest directory: %s' % (self.latest,)) logging.debug('keep history: %s' % (self.keep,)) self.prefix = ''
def __init__(self, tests=None, heads=None): """tests - a subset of the tests to run heads - js files that provide extra functionality """ # These we just copy over and worry about them later self.tests = tests self.heads = heads if heads else [] logging.debug('requested tests: %s' % (tests, )) logging.debug('heads: %s' % (heads, )) self.testroot = stoneridge.get_config('stoneridge', 'testroot') self.unittest = stoneridge.get_config_bool('stoneridge', 'unittest') logging.debug('testroot: %s' % (self.testroot, )) logging.debug('unittest: %s' % (self.unittest, ))
def __new__(self, *args, **kwargs): # The caller shouldn't care what platform its running on, so we # override __new__ to create the class that will unpack properly no # matter what os_name = stoneridge.get_config('machine', 'os') if os_name == 'windows': logging.debug('creating windows unpacker') return object.__new__(WindowsUnpacker) elif os_name == 'linux': logging.debug('creating linux unpacker') return object.__new__(LinuxUnpacker) elif os_name == 'mac': logging.debug('creating mac unpacker') return object.__new__(MacUnpacker) logging.critical('could not figure out what unpacker to create') raise ValueError('Invalid system type: %s' % (os_name, ))
def __new__(self, *args, **kwargs): # The caller shouldn't care what platform its running on, so we # override __new__ to create the class that will unpack properly no # matter what os_name = stoneridge.get_config('machine', 'os') if os_name == 'windows': logging.debug('creating windows unpacker') return object.__new__(WindowsUnpacker) elif os_name == 'linux': logging.debug('creating linux unpacker') return object.__new__(LinuxUnpacker) elif os_name == 'mac': logging.debug('creating mac unpacker') return object.__new__(MacUnpacker) logging.critical('could not figure out what unpacker to create') raise ValueError('Invalid system type: %s' % (os_name,))
def run(self): logging.debug('archiver running') outdir = stoneridge.get_config('run', 'out') archivedir = stoneridge.get_config('stoneridge', 'archives') srid = stoneridge.get_config('run', 'srid') netconfig = stoneridge.get_config('run', 'netconfig') infofile = stoneridge.get_config('run', 'info') metadata = stoneridge.get_config('run', 'metadata') with file(infofile, 'rb') as f: info = json.load(f) logging.debug('loaded info %s' % (info,)) arcname = 'stoneridge_%s_%s_%s_%s_%s' % ( info['date'], info['test_machine']['name'], info['test_build']['revision'], srid, netconfig) logging.debug('archive name %s.zip' % (arcname,)) filename = os.path.join(archivedir, '%s.zip' % (arcname,)) if not os.path.exists(archivedir): logging.debug('making archive directory %s' % (archivedir,)) os.mkdir(archivedir) logging.debug('opening zip file for writing') zfile = zipfile.ZipFile(filename, mode='w') # Put all the files under a directory in the zip named for the zip # file itself, for easy separation when unzipping multiple archives # in the same place logging.debug('adding files to zip') for dirpath, dirs, files in os.walk(outdir): dirname = dirpath.replace(outdir, arcname, 1) logging.debug('directory %s -> %s' % (dirpath, dirname)) # Add the directories to the zip for d in dirs: logging.debug('subdirectory %s' % (d,)) zfile.write(os.path.join(dirpath, d), arcname=os.path.join(dirname, d)) # Add the files to the zip for f in files: logging.debug('file %s' % (f,)) zfile.write(os.path.join(dirpath, f), arcname=os.path.join(dirname, f)) logging.debug('closing zip file') zfile.close() # Make a copy where the uploader will find it shutil.copyfile(filename, metadata)
def __init__(self): self.server = stoneridge.get_config("download", "server") self.downloadroot = stoneridge.get_config("download", "root") self.download_platform = stoneridge.get_config("machine", "download_platform") self.download_suffix = stoneridge.get_config("machine", "download_suffix") self.srid = stoneridge.get_config("run", "srid") self.downloaddir = stoneridge.get_config("run", "download") logging.debug("server = %s" % (self.server,)) logging.debug("download root = %s" % (self.downloadroot,)) logging.debug("platform = %s" % (self.download_platform,)) logging.debug("suffix = %s" % (self.download_suffix,)) logging.debug("srid = %s" % (self.srid,)) logging.debug("downloaddir = %s" % (self.downloaddir,))
def run(self): logging.debug('archiver running') outdir = stoneridge.get_config('run', 'out') archivedir = stoneridge.get_config('stoneridge', 'archives') srid = stoneridge.get_config('run', 'srid') netconfig = stoneridge.get_config('run', 'netconfig') infofile = stoneridge.get_config('run', 'info') metadata = stoneridge.get_config('run', 'metadata') with file(infofile, 'rb') as f: info = json.load(f) logging.debug('loaded info %s' % (info, )) arcname = 'stoneridge_%s_%s_%s_%s_%s' % ( info['date'], info['test_machine']['name'], info['test_build']['revision'], srid, netconfig) logging.debug('archive name %s.zip' % (arcname, )) filename = os.path.join(archivedir, '%s.zip' % (arcname, )) if not os.path.exists(archivedir): logging.debug('making archive directory %s' % (archivedir, )) os.mkdir(archivedir) logging.debug('opening zip file for writing') zfile = zipfile.ZipFile(filename, mode='w') # Put all the files under a directory in the zip named for the zip # file itself, for easy separation when unzipping multiple archives # in the same place logging.debug('adding files to zip') for dirpath, dirs, files in os.walk(outdir): dirname = dirpath.replace(outdir, arcname, 1) logging.debug('directory %s -> %s' % (dirpath, dirname)) # Add the directories to the zip for d in dirs: logging.debug('subdirectory %s' % (d, )) zfile.write(os.path.join(dirpath, d), arcname=os.path.join(dirname, d)) # Add the files to the zip for f in files: logging.debug('file %s' % (f, )) zfile.write(os.path.join(dirpath, f), arcname=os.path.join(dirname, f)) logging.debug('closing zip file') zfile.close() # Make a copy where the uploader will find it shutil.copyfile(filename, metadata)
def translate_path(self, path): """Override the base translate_path to get something with a configurable root """ logging.debug('Translating path %s' % (path,)) rootdir = stoneridge.get_config('server', 'downloads') path = path.split('?', 1)[0] logging.debug('Without query string: %s' % (path,)) path = path.split('#', 1)[0] logging.debug('Without fragment: %s' % (path,)) path = posixpath.normpath(urllib.unquote(path)) logging.debug('Normalized path: %s' % (path,)) words = [w for w in path.split('/') if w] logging.debug('Non-empty path components: %s' % (words,)) path = rootdir logging.debug('Root of translated path: %s' % (path,)) for w in words: path = os.path.join(path, w) logging.debug('Translated path: %s' % (path,)) return path
def __init__(self): self.server = stoneridge.get_config('download', 'server') self.downloadroot = stoneridge.get_config('download', 'root') self.download_platform = stoneridge.get_config('machine', 'download_platform') self.download_suffix = stoneridge.get_config('machine', 'download_suffix') self.srid = stoneridge.get_config('run', 'srid') self.downloaddir = stoneridge.get_config('run', 'download') logging.debug('server = %s' % (self.server,)) logging.debug('download root = %s' % (self.downloadroot,)) logging.debug('platform = %s' % (self.download_platform,)) logging.debug('suffix = %s' % (self.download_suffix,)) logging.debug('srid = %s' % (self.srid,)) logging.debug('downloaddir = %s' % (self.downloaddir,))
def _build_testlist(self): """Return a list of test file names, all relative to the test root. This weeds out any tests that may be missing from the directory. """ if not self.tests: logging.debug('searching for all tests in %s' % (self.testroot, )) if stoneridge.get_config('test', 'enabled'): tests = [] if os.path.exists(os.path.join(self.testroot, 'fake.js')): tests.append('fake.js') else: jstests = [ os.path.basename(f) for f in glob.glob(os.path.join(self.testroot, '*.js')) ] try: jstests.remove('fake.js') except ValueError: # Don't care if fake.js isn't in the list pass pagetests = [ os.path.basename(f) for f in glob.glob(os.path.join(self.testroot, '*.page')) ] tests = jstests + pagetests logging.debug('tests found %s' % (tests, )) return tests tests = [] for candidate in self.tests: logging.debug('candidate test %s' % (candidate, )) if not candidate.endswith('.js'): logging.error('invalid test filename %s' % (candidate, )) elif not os.path.exists(os.path.join(self.testroot, candidate)): logging.error('missing test %s' % (candidate, )) else: logging.debug('valid test file %s' % (candidate, )) tests.append(candidate) logging.debug('tests selected %s' % (tests, )) return tests
def _build_testlist(self): """Return a list of test file names, all relative to the test root. This weeds out any tests that may be missing from the directory. """ if not self.tests: logging.debug('searching for all tests in %s' % (self.testroot,)) if stoneridge.get_config('test', 'enabled'): tests = [] if os.path.exists(os.path.join(self.testroot, 'fake.js')): tests.append('fake.js') else: jstests = [os.path.basename(f) for f in glob.glob(os.path.join(self.testroot, '*.js'))] try: jstests.remove('fake.js') except ValueError: # Don't care if fake.js isn't in the list pass pagetests = [os.path.basename(f) for f in glob.glob(os.path.join(self.testroot, '*.page'))] tests = jstests + pagetests logging.debug('tests found %s' % (tests,)) return tests tests = [] for candidate in self.tests: logging.debug('candidate test %s' % (candidate,)) if not candidate.endswith('.js'): logging.error('invalid test filename %s' % (candidate,)) elif not os.path.exists(os.path.join(self.testroot, candidate)): logging.error('missing test %s' % (candidate,)) else: logging.debug('valid test file %s' % (candidate,)) tests.append(candidate) logging.debug('tests selected %s' % (tests,)) return tests
def __init__(self): self.rootdir = stoneridge.get_config('server', 'uploads') self.pattern = os.path.join(self.rootdir, '*.json') self.host = stoneridge.get_config('report', 'host') self.project = stoneridge.get_config('report', 'project') self.key = stoneridge.get_config('report', 'key') self.secret = stoneridge.get_config('report', 'secret') self.archives = stoneridge.get_config('report', 'archives') logfile = os.path.join(self.archives, '%s.log' % (int(time.time()),)) logging.basicConfig(filename=logfile, level=logging.DEBUG, format=stoneridge.log_fmt) logging.debug('root directory: %s' % (self.rootdir,)) logging.debug('pattern: %s' % (self.pattern,)) logging.debug('host: %s' % (self.host,)) logging.debug('project: %s' % (self.project,)) logging.debug('oauth key: %s' % (self.key,)) logging.debug('oauth secret: %s' % (self.secret,)) logging.debug('archives: %s' % (self.archives,))
def run(self): logging.debug('unpacker running') # Get our firefox logging.debug('unpacking firefox') self.unpack_firefox() # Unzip the stuff we need from the tests zipfile unzipdir = os.path.join(self.workdir, 'tests') logging.debug('creating unzip dir %s' % (unzipdir, )) os.mkdir(unzipdir) z = zipfile.ZipFile(self.testzip, 'r') members = [f for f in z.namelist() if f.startswith('bin')] logging.debug('unzipping %s' % (str(members), )) z.extractall(unzipdir, members) # Put the xpcshell binary where it belongs unzipbin = os.path.join(unzipdir, 'bin') xpcshell_bin = stoneridge.get_config('machine', 'xpcshell') xpcshell = os.path.join(unzipbin, xpcshell_bin) logging.debug('xpcshell: %s' % (xpcshell, )) # Apparently xpcshell stopped being executable in the tests zip at some # point, so we need to fix that before copying logging.debug('setting permissions on xpcshell') os.chmod(xpcshell, 0755) logging.debug('copy xpcshell %s -> %s' % (xpcshell, self.bindir)) shutil.copy(xpcshell, self.bindir) # Put our components into place logging.debug('copying components') self._copy_tree(unzipbin, 'components') # Put the plugins in place, in case we need them logging.debug('copying plugins') self._copy_tree(unzipbin, 'plugins') # Put the pageloader components into place srroot = stoneridge.get_config('stoneridge', 'root') pageloader = os.path.join(srroot, 'pageloader') self._copy_tree(pageloader, 'components') self._copy_tree(pageloader, 'chrome') # Now we need to put srdata.js into the appropriate place for it to be # picked up by the pageloader chrome = os.path.join(self.bindir, 'chrome') srdatasrc = os.path.join(srroot, 'srdata.js') srdatadst = os.path.join(chrome, 'srdata.js') if os.path.exists(srdatadst): os.unlink(srdatadst) logging.debug('copy srdata.js %s -> %s' % (srdatasrc, srdatadst)) shutil.copyfile(srdatasrc, srdatadst) # Finally, we need to update chrome.manifest with the appropriate bits # from our local pageloader plmanifest = os.path.join(pageloader, 'chrome.manifest') fxmanifest = os.path.join(self.bindir, 'chrome.manifest') logging.debug('append %s to %s' % (plmanifest, fxmanifest)) if os.path.exists(fxmanifest): with file(fxmanifest, 'rb') as f: lines = f.readlines() else: lines = [] with file(plmanifest, 'rb') as f: lines.extend(f.readlines()) with file(fxmanifest, 'wb') as f: f.writelines(lines)
def __init__(self): self.url = stoneridge.get_config('mqproxy', 'url') self.setup()
def run(self): logging.debug('runner running') tests = self._build_testlist() preargs = self._build_preargs() logging.debug('tests to run: %s' % (tests, )) logging.debug('args to prepend: %s' % (preargs, )) # Ensure our output directory exists outdir = stoneridge.get_config('run', 'out') profile = stoneridge.get_config('run', 'profile') installroot = stoneridge.get_config('stoneridge', 'root') for test in tests: logging.debug('test: %s' % (test, )) outfile = os.path.join(outdir, '%s.out' % (test, )) logging.debug('outfile: %s' % (outfile, )) if test.endswith('.js'): escaped_outfile = outfile.replace('\\', '\\\\') args = preargs + [ '-e', 'const _SR_OUT_FILE = "%s";' % (escaped_outfile, ), '-f', os.path.join(installroot, 'srdata.js'), '-f', os.path.join(installroot, 'head.js'), '-f', os.path.join(self.testroot, test), '-e', 'do_stoneridge(); quit(0);' ] logging.debug('xpcshell args: %s' % (args, )) runner = stoneridge.run_xpcshell else: args = [ '-sr', os.path.join(self.testroot, test), '-sroutput', outfile, '-profile', profile, # -srwidth, <some width value>, # -srheight, <some height value>, # -srtimeout, <some timeout value per page>, # -srdelay, <some delay value between pages>, # -srmozafterpaint ] runner = stoneridge.run_firefox if self.unittest: logging.debug('Not running processes: in unit test mode') else: process_out_file = '%s.process.out' % (test, ) process_out_file = os.path.join(outdir, process_out_file) logging.debug('process output at %s' % (process_out_file, )) timed_out = False with file(process_out_file, 'wb') as f: try: res = runner(args, f) except stoneridge.TestProcessTimeout: logging.exception('test process timed out!') timed_out = True res = None if res or timed_out: logging.error('TEST FAILED: %s' % (test, )) else: logging.debug('test succeeded')
def run(self): logging.debug('collator running') outdir = stoneridge.get_config('run', 'out') outfiles = glob.glob(os.path.join(outdir, '*.js.out')) outfiles.extend(glob.glob(os.path.join(outdir, '*.page.out'))) infofile = stoneridge.get_config('run', 'info') logging.debug('found outfiles %s' % (outfiles,)) logging.debug('loading info from %s' % (infofile,)) with file(infofile, 'rb') as f: info = json.load(f) logging.debug('loaded info: %s' % (info,)) for ofile in outfiles: logging.debug('processing %s' % (ofile,)) # Make a new copy of the base info results = copy.deepcopy(info) del results['date'] results['testrun'] = {'date': info['date'], 'suite': None, 'options': {}} results['results'] = collections.defaultdict(list) results['results_aux'] = collections.defaultdict(list) logging.debug('initial testrun: %s' % (results['testrun'],)) # Figure out the test-specific data fname = os.path.basename(ofile) suite = fname.split('.')[0] results['testrun']['suite'] = suite ldap = stoneridge.get_config('run', 'ldap') if ldap is None: results['testrun']['options']['ldap'] = 'nightly' else: results['testrun']['options']['ldap'] = ldap logging.debug('suite: %s' % (suite,)) # Read the raw data logging.debug('reading raw data') with file(ofile, 'rb') as f: testinfo = json.load(f) logging.debug('raw testinfo: %s' % (testinfo,)) # Stick the raw data into the json to be uploaded logging.debug('processing raw data') for k, vlist in testinfo.items(): logging.debug('k: %s, vlist: %s' % (k, vlist)) for v in vlist: logging.debug('v: %s' % (v,)) if k == 'total': logging.debug('appending total %s' % (v['total'],)) # The graph server calculates totals for us, we just # keep our calculations around for verification in case results['results_aux']['totals'].append(v['total']) else: logging.debug('appending %s total %s' % (k, v['total'])) results['results'][k].append(v['total']) for s in ('start', 'stop'): key = '%s_%s' % (k, s) logging.debug('appending %s %s stamp %s' % (k, s, v[s])) results['results_aux'][key].append(v[s]) # Turn our defaultdicts into regular dicts for jsonification results['results'] = dict(results['results']) results['results_aux'] = dict(results['results_aux']) logging.debug('results: %s' % (results['results'],)) logging.debug('aux results: %s' % (results['results_aux'],)) # Write our json results for uploading upload_filename = 'upload_%s.json' % (suite,) logging.debug('upload filename: %s' % (upload_filename,)) upload_file = os.path.join(outdir, upload_filename) with file(upload_file, 'wb') as f: logging.debug('jsonifying %s' % (results,)) json.dump(results, f)
def __init__(self): self.workdir = stoneridge.get_config('stoneridge', 'work') self.keep = stoneridge.get_config_int('cleaner', 'keep')
def __init__(self): self.url = stoneridge.get_config('upload', 'url') logging.debug('upload url: %s' % (self.url,))
def handle(self, srid, netconfig, tstamp, ldap): # Create the directory where data we want to save from this run will go srwork = os.path.join(self.workroot, srid, netconfig) if os.path.exists(srwork): srwork = '%s_%s' % (srwork, tstamp) os.makedirs(srwork) srout = os.path.join(srwork, 'out') os.mkdir(srout) # Have a logger just for this run self.logdir = os.path.join(srout, 'logs') os.makedirs(self.logdir) logging.debug('Running test with logs in %s' % (self.logdir, )) logfile = os.path.join(self.logdir, '00_worker.log') handler = logging.FileHandler(logfile) formatter = logging.Formatter(fmt=stoneridge.LOG_FMT) handler.setFormatter(formatter) self.logger = logging.getLogger('%s_%s_%s' % (srid, netconfig, tstamp)) self.logger.setLevel(logging.DEBUG) self.logger.addHandler(handler) # Create the rest of the working space for this run srdownload = os.path.join(srwork, 'download') os.mkdir(srdownload) firefox_path = stoneridge.get_config('machine', 'firefox_path') srbindir = os.path.join(srwork, firefox_path) metadata = os.path.join(srout, 'metadata.zip') info = os.path.join(srout, 'info.json') profile = os.path.join(srout, 'profile') os.mkdir(profile) self.srnetconfig = netconfig self.uploaded = False self.archive_on_failure = True self.procno = 1 self.childlog = None self.need_dns_reset = False self.runconfig = os.path.join(srout, 'run.ini') with file(self.runconfig, 'w') as f: f.write('[run]\n') f.write('netconfig = %s\n' % (netconfig, )) f.write('work = %s\n' % (srwork, )) f.write('download = %s\n' % (srdownload, )) f.write('bin = %s\n' % (srbindir, )) f.write('out = %s\n' % (srout, )) f.write('profile = %s\n' % (profile, )) f.write('metadata = %s\n' % (metadata, )) f.write('info = %s\n' % (info, )) f.write('tstamp = %s\n' % (tstamp, )) f.write('srid = %s\n' % (srid, )) if ldap: f.write('ldap = %s\n' % (ldap, )) self.logger.debug('srnetconfig: %s' % (self.srnetconfig, )) self.logger.debug('uploaded: %s' % (self.uploaded, )) self.logger.debug('archive on failure: %s' % (self.archive_on_failure, )) self.logger.debug('procno: %s' % (self.procno, )) self.logger.debug('childlog: %s' % (self.childlog, )) self.logger.debug('logdir: %s' % (self.logdir, )) self.logger.debug('runconfig: %s' % (self.runconfig, )) self.logger.debug('ldap: %s' % (ldap, )) self.logger.debug('profile: %s' % (profile, )) try: self.run_test() except StoneRidgeException as e: self.logger.exception(e) self.reset()
def __init__(self): self.pcaps = {} self.macaddr = stoneridge.get_config('machine', 'macaddr') self.tcpdump = stoneridge.get_config('tcpdump', 'exe') self.interface = stoneridge.get_config('tcpdump', 'interface')
def setup(self): self.dbfile = stoneridge.get_config('mqproxy', 'db') self.conn = sqlite3.connect(self.dbfile)