def phone_disconnected(self, msg_body):
"""Indicate that a phone has become unreachable or experienced a
error from which we might be able to recover."""
if self.has_error():
return
self.loggerdeco.info('Phone disconnected: %s.' % msg_body)
if msg_body and self.mailer:
self.loggerdeco.info('Sending notification...')
try:
self.mailer.send(
'Phone %s disconnected' % self.phone_cfg['phoneid'],
'''Hello, this is Autophone. Phone %s appears to be disconnected:
%s
I'll keep trying to ping it periodically in case it reappears.
''' % (self.phone_cfg['phoneid'], msg_body))
self.loggerdeco.info('Sent.')
except socket.error:
self.loggerdeco.exception('Failed to send disconnected-phone '
'notification.')
self.status_update(
phonetest.PhoneTestMessage(
self.phone_cfg['phoneid'],
phonetest.PhoneTestMessage.DISCONNECTED))
def run(self):
sys.stdout = file(self.outfile, 'a', 0)
sys.stderr = sys.stdout
self.filehandler = MultiprocessingTimedRotatingFileHandler(
self.logfile, when='midnight', backupCount=7)
fileformatstring = ('%(asctime)s|%(levelname)s' '|%(message)s')
self.fileformatter = logging.Formatter(fileformatstring)
self.filehandler.setFormatter(self.fileformatter)
self.logger.addHandler(self.filehandler)
self.loggerdeco.info('PhoneWorker starting up.')
DroidSUT.loglevel = self.user_cfg.get('debug', 3)
for t in self.tests:
t.status_cb = self.status_update
self.status_update(
phonetest.PhoneTestMessage(self.phone_cfg['phoneid'], self.status))
if self.status != phonetest.PhoneTestMessage.DISABLED:
if not self.check_sdcard():
self.recover_phone()
if self.has_error():
self.loggerdeco.error('Initial SD card check failed.')
self.main_loop()
def handle_cmd(self, request):
if not request:
self.loggerdeco.debug('handle_cmd: No request')
pass
elif request[0] == 'stop':
self.loggerdeco.info('Stopping at user\'s request...')
self._stop = True
elif request[0] == 'job':
# This is just a notification that breaks us from waiting on the
# command queue; it's not essential, since jobs are stored in
# a db, but it allows the worker to react quickly to a request if
# it isn't doing anything else.
self.loggerdeco.debug('Received job command request...')
pass
elif request[0] == 'reboot':
self.loggerdeco.info('Rebooting at user\'s request...')
self.reboot()
elif request[0] == 'disable':
self.disable_phone('Disabled at user\'s request', False)
elif request[0] == 'enable':
self.loggerdeco.info('Enabling phone at user\'s request...')
if self.has_error():
self.status_update(
phonetest.PhoneTestMessage(self.phone_cfg['phoneid'],
phonetest.PhoneTestMessage.IDLE,
self.current_build))
self.last_ping = None
elif request[0] == 'debug':
self.loggerdeco.info(
'Setting debug level %d at user\'s request...' % request[1])
self.user_cfg['debug'] = request[1]
DroidSUT.debug = self.user_cfg['debug']
# update any existing DroidSUT objects
if self._dm:
self._dm.loglevel = self.user_cfg['debug']
for t in self.tests:
t.set_dm_debug(self.user_cfg['debug'])
elif request[0] == 'ping':
self.loggerdeco.info('Pinging at user\'s request...')
self.ping()
else:
self.loggerdeco.debug('handle_cmd: Unknown request %s' %
request[0])
def handle_timeout(self):
if (self.status != phonetest.PhoneTestMessage.DISABLED and
(not self.last_ping or
(datetime.datetime.now() - self.last_ping > datetime.timedelta(
seconds=self.user_cfg[PHONE_PING_INTERVAL])))):
self.last_ping = datetime.datetime.now()
if self.ping():
if self.status == phonetest.PhoneTestMessage.DISCONNECTED:
self.recover_phone()
if not self.has_error():
self.status_update(
phonetest.PhoneTestMessage(
self.phone_cfg['phoneid'],
phonetest.PhoneTestMessage.IDLE,
self.current_build))
else:
self.loggerdeco.info('Ping unanswered.')
# No point in trying to recover, since we couldn't
# even perform a simple action.
if not self.has_error():
self.phone_disconnected('No response to ping.')
def check_sdcard(self):
self.loggerdeco.info('Checking SD card.')
success = True
try:
dev_root = self.dm.getDeviceRoot()
if dev_root:
d = posixpath.join(dev_root, 'autophonetest')
self.dm.removeDir(d)
self.dm.mkDir(d)
if self.dm.dirExists(d):
with tempfile.NamedTemporaryFile() as tmp:
tmp.write('autophone test\n')
tmp.flush()
self.dm.pushFile(tmp.name,
posixpath.join(d, 'sdcard_check'))
self.dm.removeDir(d)
else:
self.loggerdeco.error('Failed to create directory under '
'device root!')
success = False
else:
self.loggerdeco.error('Invalid device root.')
success = False
except DMError:
self.loggerdeco.exception('Exception while checking SD card!')
success = False
if not success:
# FIXME: Should this be called under more circumstances than just
# checking the SD card?
self.clear_test_base_paths()
return False
# reset status if there had previous been an error.
# FIXME: should send email that phone is back up.
self.status_update(
phonetest.PhoneTestMessage(self.phone_cfg['phoneid'],
phonetest.PhoneTestMessage.IDLE))
return True
def disable_phone(self, errmsg, send_email=True):
"""Completely disable phone. No further attempts to recover it will
be performed unless initiated by the user."""
self.loggerdeco.info('Disabling phone: %s.' % errmsg)
if errmsg and send_email and self.mailer:
self.loggerdeco.info('Sending notification...')
try:
self.mailer.send(
'Phone %s disabled' % self.phone_cfg['phoneid'],
'''Hello, this is Autophone. Phone %s has been disabled:
%s
I gave up on it. Sorry about that. You can manually re-enable it with
the "enable" command.
''' % (self.phone_cfg['phoneid'], errmsg))
self.loggerdeco.info('Sent.')
except socket.error:
self.loggerdeco.exception('Failed to send disabled-phone '
'notification.')
self.status_update(
phonetest.PhoneTestMessage(self.phone_cfg['phoneid'],
phonetest.PhoneTestMessage.DISABLED,
msg=errmsg))
def handle_job(self, job):
phoneid = self.phone_cfg['phoneid']
abi = self.phone_cfg['abi']
build_url = job['build_url']
self.loggerdeco.debug('handle_job: job: %s, abi: %s' % (job, abi))
incompatible_job = False
if abi == 'x86':
if 'x86' not in build_url:
incompatible_job = True
elif abi == 'armeabi-v6':
if 'armv6' not in build_url:
incompatible_job = True
else:
if 'x86' in build_url or 'armv6' in build_url:
incompatible_job = True
if incompatible_job:
self.loggerdeco.debug('Ignoring incompatible job %s '
'for phone abi %s' % (build_url, abi))
self.jobs.job_completed(job['id'])
return
# Determine if we will test this build and if we need
# to enable unittests.
skip_build = True
enable_unittests = False
for test in self.tests:
test_devices_repos = test.test_devices_repos
if not test_devices_repos:
# We know we will test this build, but not yet
# if any of the other tests enable_unittests.
skip_build = False
elif not phoneid in test_devices_repos:
# This device will not run this test.
pass
else:
for repo in test_devices_repos[phoneid]:
if repo in build_url:
skip_build = False
enable_unittests = test.enable_unittests
break
if not skip_build:
break
if skip_build:
self.loggerdeco.debug('Ignoring job %s ' % build_url)
self.jobs.job_completed(job['id'])
return
self.loggerdeco.info('Checking job %s.' % build_url)
client = buildserver.BuildCacheClient(port=self.build_cache_port)
self.loggerdeco.info('Fetching build...')
cache_response = client.get(build_url,
enable_unittests=enable_unittests)
client.close()
if not cache_response['success']:
self.loggerdeco.warning('Errors occured getting build %s: %s' %
(build_url, cache_response['error']))
return
self.loggerdeco.info('Starting job %s.' % build_url)
starttime = datetime.datetime.now()
if self.run_tests(cache_response['metadata']):
self.loggerdeco.info('Job completed.')
self.jobs.job_completed(job['id'])
self.status_update(
phonetest.PhoneTestMessage(self.phone_cfg['phoneid'],
phonetest.PhoneTestMessage.IDLE,
self.current_build))
else:
self.loggerdeco.error('Job failed.')
stoptime = datetime.datetime.now()
self.loggerdeco.info('Job elapsed time: %s' % (stoptime - starttime))
def run_tests(self, build_metadata):
if not self.has_error():
self.loggerdeco.info('Rebooting...')
self.reboot()
# may have gotten an error trying to reboot, so test again
if self.has_error():
self.loggerdeco.info('Phone is in error state; not running test.')
return False
repo = build_metadata['tree']
build_date = datetime.datetime.fromtimestamp(
float(build_metadata['blddate']))
self.status_update(
phonetest.PhoneTestMessage(self.phone_cfg['phoneid'],
phonetest.PhoneTestMessage.INSTALLING,
build_metadata['blddate']))
self.loggerdeco.info('Installing build %s.' % build_date)
success = False
for attempt in range(self.user_cfg[PHONE_RETRY_LIMIT]):
try:
pathOnDevice = posixpath.join(self.dm.getDeviceRoot(),
'build.apk')
self.dm.pushFile(
os.path.join(build_metadata['cache_build_dir'],
'build.apk'), pathOnDevice)
self.dm.installApp(pathOnDevice)
self.dm.removeFile(pathOnDevice)
success = True
except DMError:
exc = 'Exception installing fennec attempt %d!\n\n%s' % (
attempt, traceback.format_exc())
self.loggerdeco.exception(
'Exception installing fennec attempt %d!' % attempt)
time.sleep(self.user_cfg[PHONE_RETRY_WAIT])
if not success:
self.phone_disconnected(exc)
return False
self.current_build = build_metadata['blddate']
self.loggerdeco.info('Running tests...')
for t in self.tests:
if self.has_error():
break
try:
repos = t.test_devices_repos[self.phone_cfg['phoneid']]
if repos and repo not in repos:
self.loggerdeco.debug('run_tests: ignoring build %s '
'repo %s not in '
'defined repos: %s' %
(build_date, repo, repos))
continue
except KeyError:
pass
t.current_build = build_metadata['blddate']
try:
t.runjob(build_metadata, self)
except DMError:
exc = 'Uncaught device error while running test!\n\n%s' % \
traceback.format_exc()
self.loggerdeco.exception('Uncaught device error while '
'running test!')
self.phone_disconnected(exc)
return False
return True
def reboot(self):
self.status_update(
phonetest.PhoneTestMessage(self.phone_cfg['phoneid'],
phonetest.PhoneTestMessage.REBOOTING))
self.recover_phone()