class StartHandler(tornado.web.RequestHandler):
def post(self):
global active_test
cfg, cfg_data = None, None
try:
cfg_data = self.get_body_argument("config")
cfg = yaml.load(cfg_data)
except Exception:
self.set_status(400)
self.write('Error parsing config: %s. \nTraceback: %s' %
(cfg_data, traceback.format_exc()))
return
logger.debug('Received config: %s. Starting test', cfg)
try:
self.core = Core(cfg)
self.core.configure()
self.perform_test()
active_test = self.core
self.write(json.dumps(self.core.get_current_test_info()))
except Exception:
logger.warning('Failed to start the test', exc_info=True)
self.set_status(500)
self.write('Failed to start the test: %s' % traceback.format_exc())
@gen.coroutine
def perform_test(self):
logger.info(
'Starting test... You can interrupt via API /stop/ handler')
self.core.start_test()
def main():
import argparse
parser = argparse.ArgumentParser(description='volta console worker')
parser.add_argument('--debug',
dest='debug',
action='store_true',
default=False)
parser.add_argument('-c', '--config', dest='config')
args = parser.parse_args()
logging.basicConfig(
level="DEBUG" if args.debug else "INFO",
format=
'%(asctime)s [%(levelname)s] [Volta Core] %(filename)s:%(lineno)d %(message)s'
)
if not args.config:
raise RuntimeError('Empty config')
cfg_dict = {}
with open(args.config, 'r') as cfg_stream:
try:
cfg_dict = yaml.safe_load(cfg_stream)
except:
logger.debug('Config file format not yaml or json...',
exc_info=True)
raise RuntimeError('Unknown config file format. Malformed?')
core = Core(cfg_dict)
try:
core.configure()
logger.info(
'Starting test... You can interrupt test w/ Ctrl+C or SIGTERM signal'
)
core.start_test()
while True:
time.sleep(1) # infinite loop until SIGTERM
except KeyboardInterrupt:
logger.info(
'Keyboard interrupt, trying graceful shutdown. Do not press interrupt again...'
)
core.end_test()
except:
logger.error('Uncaught exception in core\n', exc_info=True)
finally:
core.post_process()
def post(self):
global active_test
cfg, cfg_data = None, None
try:
cfg_data = self.get_body_argument("config")
cfg = yaml.load(cfg_data)
except Exception:
self.set_status(400)
self.write('Error parsing config: %s. \nTraceback: %s' % (cfg_data, traceback.format_exc()))
return
logger.debug('Received config: %s. Starting test', cfg)
try:
self.core = Core(cfg)
self.core.configure()
self.perform_test()
active_test = self.core
self.write(json.dumps(self.core.get_current_test_info()))
except Exception:
logger.warning('Failed to start the test', exc_info=True)
self.set_status(500)
self.write('Failed to start the test: %s' % traceback.format_exc())
def __init__(self, core, cfg, name):
self.stats_reader = None
self.reader = None
super(Plugin, self).__init__(core, cfg, name)
self.device = None
try:
self.cfg = cfg['volta_options']
for key, value in self.cfg.iteritems():
if not isinstance(value, dict):
logger.debug('Malformed VoltaConfig key: %s value %s', key, value)
raise RuntimeError('Malformed VoltaConfig passed, key: %s. Should by dict' % key)
except AttributeError:
logger.error('Failed to read Volta config', exc_info=True)
self.volta_core = VoltaCore(self.cfg)
def __init__(self, core, cfg, cfg_updater):
self.stats_reader = None
self.reader = None
try:
super(Plugin, self).__init__(core, cfg, cfg_updater)
self.device = None
self.cfg = cfg['volta_options']
for key, value in self.cfg.iteritems():
if not isinstance(value, dict):
logger.debug('Malformed VoltaConfig key: %s value %s', key, value)
raise RuntimeError('Malformed VoltaConfig passed, key: %s. Should by dict' % key)
except AttributeError:
logger.error('Failed to read Volta config', exc_info=True)
self.volta_core = VoltaCore(self.cfg)
def __init__(self, tank_queue, manager_queue, working_dir, config,
session_id):
# Parameters from manager
self.tank_queue = tank_queue
self.manager_queue = manager_queue
self.working_dir = working_dir
self.session_id = session_id
self.config = config
# State variables
self.stage = 'not started'
self.failures = []
self.retcode = None
self.locked = False
self.done_stages = set()
self.core = VoltaCore(self.config)
self.core.session_id = None
self.core.status = None
class Plugin(AbstractPlugin):
SECTION = "android"
SECTION_META = "meta"
def __init__(self, core, cfg):
self.stats_reader = None
self.reader = None
super(Plugin, self).__init__(core, cfg)
self.device = None
try:
self.cfg = cfg['volta_options']
for key, value in self.cfg.iteritems():
if not isinstance(value, dict):
logger.debug('Malformed VoltaConfig key: %s value %s', key,
value)
raise RuntimeError(
'Malformed VoltaConfig passed, key: %s. Should by dict'
% key)
except AttributeError:
logger.error('Failed to read Volta config', exc_info=True)
self.volta_core = VoltaCore(self.cfg)
@staticmethod
def get_key():
return __file__
def get_available_options(self):
opts = ["volta_options"]
return opts
def configure(self):
self.volta_core.configure()
def get_reader(self):
if self.reader is None:
self.reader = AndroidReader()
return self.reader
def get_stats_reader(self):
if self.stats_reader is None:
self.stats_reader = AndroidStatsReader()
return self.stats_reader
def prepare_test(self):
self.core.add_artifact_file(self.volta_core.currents_fname)
[
self.core.add_artifact_file(fname)
for fname in self.volta_core.event_fnames.values()
]
def start_test(self):
try:
self.volta_core.start_test()
# FIXME raise/catch appropriate exception here
except: # noqa: E722
logger.info('Failed to start test of Android plugin',
exc_info=True)
return 1
def is_test_finished(self):
try:
if hasattr(self.volta_core, 'phone'):
if hasattr(self.volta_core.phone, 'test_performer'):
if not self.volta_core.phone.test_performer:
logger.warning(
'There is no test performer process on the phone, interrupting test'
)
return 1
if not self.volta_core.phone.test_performer.is_finished():
logger.debug('Waiting for phone test to finish...')
return -1
else:
return self.volta_core.phone.test_performer.retcode
# FIXME raise/catch appropriate exception here
except: # noqa: E722
logger.error(
'Unknown exception of Android plugin. Interrupting test',
exc_info=True)
return 1
def end_test(self, retcode):
try:
self.volta_core.end_test()
uploaders = self.core.get_plugins_of_type(DataUploaderPlugin)
for uploader in uploaders:
response = uploader.lp_job.api_client.link_mobile_job(
lp_key=uploader.lp_job.number,
mobile_key=self.volta_core.uploader.jobno)
logger.info(
'Linked mobile job %s to %s for plugin: %s. Response: %s',
self.volta_core.uploader.jobno, uploader.lp_job.number,
uploader.backend_type, response)
# FIXME raise/catch appropriate exception here
except: # noqa: E722
logger.error('Failed to complete end_test of Android plugin',
exc_info=True)
retcode = 1
return retcode
def get_info(self):
return AndroidInfo()
def post_process(self, retcode):
try:
self.volta_core.post_process()
# FIXME raise/catch appropriate exception here
except: # noqa: E722
logger.error('Failed to complete post_process of Android plugin',
exc_info=True)
retcode = 1
return retcode
class VoltaWorker(object):
""" Volta Worker class that runs Volta core """
def __init__(self, tank_queue, manager_queue, working_dir, config,
session_id):
# Parameters from manager
self.tank_queue = tank_queue
self.manager_queue = manager_queue
self.working_dir = working_dir
self.session_id = session_id
self.config = config
# State variables
self.stage = 'not started'
self.failures = []
self.retcode = None
self.locked = False
self.done_stages = set()
self.core = VoltaCore(self.config)
self.core.session_id = None
self.core.status = None
def __add_log_file(self, logger, loglevel, filename):
"""Adds FileHandler to logger; adds filename to artifacts"""
self.core.add_artifact_file(filename)
handler = logging.FileHandler(filename)
handler.setLevel(loglevel)
handler.setFormatter(
logging.Formatter(
"%(asctime)s [%(levelname)s] %(name)s %(message)s"))
logger.addHandler(handler)
def __setup_logging(self):
"""
Logging setup.
Should be called only after the lock is acquired.
"""
logger = logging.getLogger('')
logger.setLevel(logging.DEBUG)
self.__add_log_file(logger, logging.DEBUG, 'test.log')
def report_status(self, status, stage_completed):
"""Report status to manager and dump status.json, if required"""
msg = {
'status': status,
'session': self.session_id,
'current_stage': self.stage,
'stage_completed': stage_completed,
'failures': self.failures,
'retcode': self.retcode,
'test_status': self.core.status,
}
self.manager_queue.put(msg)
if self.locked:
json.dump(msg, open('status.json', 'w'), indent=4)
def process_failure(self, reason):
"""
Act on failure of current test stage:
- log it
- add to failures list
"""
logger.error("Failure in stage %s:\n%s", self.stage, reason)
self.failures.append({'stage': self.stage, 'reason': reason})
def _execute_stage(self, stage):
"""Really execute stage and set retcode"""
new_retcode = {
'configure': self.core.configure,
'start_test': self.core.start_test
}[stage]()
if new_retcode is not None:
self.retcode = new_retcode
def _stop_stage(self):
"""Really execute stage and set retcode"""
self.core.end_test()
self.core.post_process()
def next_stage(self, stage):
"""
Report stage completion.
Switch to the next test stage if allowed.
Run it or skip it
"""
self.stage = stage
self.report_status('running', False)
if stage == common.TEST_STAGE_ORDER[0] or common.TEST_STAGE_DEPS[
stage] in self.done_stages:
try:
self._execute_stage(stage)
except InterruptTest:
self.retcode = self.retcode or 1
self.process_failure("Interrupted")
raise StopTest()
except Exception as ex:
self.retcode = self.retcode or 1
logger.exception(
"Exception occured, trying to exit gracefully...")
self.process_failure("Exception:" + traceback.format_exc(ex))
else:
self.done_stages.add(stage)
else:
self.process_failure("skipped")
self.report_status('running', True)
def perform_test(self):
"""Perform the test sequence via TankCore"""
try:
for stage in common.TEST_STAGE_ORDER[:-1]:
self.next_stage(stage)
except StopTest:
self._stop_stage()
else:
self._stop_stage()
self.stage = 'finished'
self.report_status('failed' if self.failures else 'success', True)
logger.info("Done performing test with code %s", self.retcode)
def main():
import argparse
parser = argparse.ArgumentParser(description='volta console worker')
parser.add_argument('-d',
'--debug',
'-v',
'--verbose',
dest='verbose',
action='store_true',
default=False)
parser.add_argument('-q',
'--quiet',
dest='quiet',
action='store_true',
default=False)
parser.add_argument('-l', '--log', dest='log', default='volta.log')
parser.add_argument('-c', '--config', dest='config')
args = parser.parse_args()
if not args.config:
raise RuntimeError('Empty config')
init_logging(args.log, args.verbose, args.quiet)
cfg_dict = {}
with open(args.config, 'r') as cfg_stream:
try:
cfg_dict = yaml.safe_load(cfg_stream)
except:
logger.debug('Config file format not yaml or json...',
exc_info=True)
raise RuntimeError('Unknown config file format. Malformed?')
core = Core(cfg_dict)
try:
core.configure()
logger.info(
'Starting test... You can interrupt test w/ Ctrl+C or SIGTERM signal'
)
core.start_test()
while True:
time.sleep(1) # infinite loop until SIGTERM
except KeyboardInterrupt:
logger.info(
'Keyboard interrupt, trying graceful shutdown. Do not press interrupt again...'
)
core.end_test()
except:
logger.error('Uncaught exception in core\n', exc_info=True)
finally:
core.post_process()
core.collect_file(args.log)
class Plugin(AbstractPlugin):
SECTION = "android"
SECTION_META = "meta"
def __init__(self, core, cfg, name):
self.stats_reader = None
self.reader = None
super(Plugin, self).__init__(core, cfg, name)
self.device = None
try:
self.cfg = cfg['volta_options']
for key, value in self.cfg.iteritems():
if not isinstance(value, dict):
logger.debug('Malformed VoltaConfig key: %s value %s', key, value)
raise RuntimeError('Malformed VoltaConfig passed, key: %s. Should by dict' % key)
except AttributeError:
logger.error('Failed to read Volta config', exc_info=True)
self.volta_core = VoltaCore(self.cfg)
@staticmethod
def get_key():
return __file__
def get_available_options(self):
opts = ["volta_options"]
return opts
def configure(self):
self.volta_core.configure()
def get_reader(self):
if self.reader is None:
self.reader = AndroidReader()
return self.reader
def get_stats_reader(self):
if self.stats_reader is None:
self.stats_reader = AndroidStatsReader()
return self.stats_reader
def prepare_test(self):
self.core.add_artifact_file(self.volta_core.currents_fname)
[self.core.add_artifact_file(fname) for fname in self.volta_core.event_fnames.values()]
def start_test(self):
try:
self.volta_core.start_test()
# FIXME raise/catch appropriate exception here
except: # noqa: E722
logger.info('Failed to start test of Android plugin', exc_info=True)
return 1
def is_test_finished(self):
try:
if hasattr(self.volta_core, 'phone'):
if hasattr(self.volta_core.phone, 'test_performer'):
if not self.volta_core.phone.test_performer:
logger.warning('There is no test performer process on the phone, interrupting test')
return 1
if not self.volta_core.phone.test_performer.is_finished():
logger.debug('Waiting for phone test to finish...')
return -1
else:
return self.volta_core.phone.test_performer.retcode
# FIXME raise/catch appropriate exception here
except: # noqa: E722
logger.error('Unknown exception of Android plugin. Interrupting test', exc_info=True)
return 1
def end_test(self, retcode):
try:
self.volta_core.end_test()
uploaders = self.core.get_plugins_of_type(DataUploaderPlugin)
for uploader in uploaders:
response = uploader.lp_job.api_client.link_mobile_job(
lp_key=uploader.lp_job.number,
mobile_key=self.volta_core.uploader.jobno
)
logger.info(
'Linked mobile job %s to %s for plugin: %s. Response: %s',
self.volta_core.uploader.jobno, uploader.lp_job.number, uploader.backend_type, response
)
# FIXME raise/catch appropriate exception here
except: # noqa: E722
logger.error('Failed to complete end_test of Android plugin', exc_info=True)
retcode = 1
return retcode
def get_info(self):
return AndroidInfo()
def post_process(self, retcode):
try:
self.volta_core.post_process()
# FIXME raise/catch appropriate exception here
except: # noqa: E722
logger.error('Failed to complete post_process of Android plugin', exc_info=True)
retcode = 1
return retcode
class Plugin(AbstractPlugin, GeneratorPlugin):
SECTION = "android"
SECTION_META = "meta"
def __init__(self, core, cfg, cfg_updater):
try:
super(Plugin, self).__init__(core, cfg, cfg_updater)
self.device = None
self.cfg = cfg['volta_options']
for key, value in self.cfg.iteritems():
if not isinstance(value, dict):
logger.debug('Malformed VoltaConfig key: %s value %s', key,
value)
raise RuntimeError(
'Malformed VoltaConfig passed, key: %s. Should by dict'
% key)
except AttributeError:
logger.error('Failed to read Volta config', exc_info=True)
self.volta_core = VoltaCore(self.cfg)
@staticmethod
def get_key():
return __file__
def get_available_options(self):
opts = ["volta_options"]
return opts
def configure(self):
self.volta_core.configure()
def prepare_test(self):
aggregator = self.core.job.aggregator_plugin
if aggregator:
aggregator.reader = AndroidReader()
aggregator.stats_reader = AndroidStatsReader()
self.core.add_artifact_file(self.volta_core.currents_fname)
[
self.core.add_artifact_file(fname)
for fname in self.volta_core.event_fnames.values()
]
def start_test(self):
self.volta_core.start_test()
def is_test_finished(self):
if hasattr(self.volta_core, 'phone'):
if hasattr(self.volta_core.phone, 'test_performer'):
if not self.volta_core.phone.test_performer._finished:
logger.debug('Waiting for phone test for finish...')
return -1
else:
return self.volta_core.phone.test_performer.retcode
def end_test(self, retcode):
self.volta_core.end_test()
return retcode
def get_info(self):
return AndroidInfo()
def post_process(self, retcode):
self.volta_core.post_process()
return retcode
class Plugin(AbstractPlugin, GeneratorPlugin):
SECTION = "android"
SECTION_META = "meta"
def __init__(self, core, cfg, cfg_updater):
self.stats_reader = None
self.reader = None
try:
super(Plugin, self).__init__(core, cfg, cfg_updater)
self.device = None
self.cfg = cfg['volta_options']
for key, value in self.cfg.iteritems():
if not isinstance(value, dict):
logger.debug('Malformed VoltaConfig key: %s value %s', key, value)
raise RuntimeError('Malformed VoltaConfig passed, key: %s. Should by dict' % key)
except AttributeError:
logger.error('Failed to read Volta config', exc_info=True)
self.volta_core = VoltaCore(self.cfg)
@staticmethod
def get_key():
return __file__
def get_available_options(self):
opts = ["volta_options"]
return opts
def configure(self):
self.volta_core.configure()
def get_reader(self):
if self.reader is None:
self.reader = AndroidReader()
return self.reader
def get_stats_reader(self):
if self.stats_reader is None:
self.stats_reader = AndroidStatsReader()
return self.stats_reader
def prepare_test(self):
self.core.add_artifact_file(self.volta_core.currents_fname)
[self.core.add_artifact_file(fname) for fname in self.volta_core.event_fnames.values()]
def start_test(self):
self.volta_core.start_test()
def is_test_finished(self):
if hasattr(self.volta_core, 'phone'):
if hasattr(self.volta_core.phone, 'test_performer'):
if not self.volta_core.phone.test_performer._finished:
logger.debug('Waiting for phone test for finish...')
return -1
else:
return self.volta_core.phone.test_performer.retcode
def end_test(self, retcode):
self.volta_core.end_test()
mobile_key = self.volta_core.uploader.jobno
logger.info("Mobile jobno: %s", mobile_key)
jobno = self.core.status['uploader']['job_no']
logger.info("Simple jobno: %s", jobno)
web_link = self.core.status['uploader']['web_link']
url = web_link.replace(str(jobno), '')
logger.info("Url: %s", url)
self.link_jobs(url, jobno, mobile_key)
return retcode
def link_jobs(self, url, jobno, mobile_key):
api_client = OverloadClient()
api_client.set_api_address(url)
api_client.session.verify = False
addr = "/api/job/{jobno}/edit.json".format(jobno=jobno)
data = {
'mobile_key': mobile_key
}
logger.info("Jobs link request: url = %s, data = %s", url + addr, data)
while True:
try:
response = api_client.post(addr, data)
return response
except requests.exceptions.HTTPError as ex:
logger.debug("Got error for jobs link request: %s", ex)
if ex.response.status_code == 423:
logger.warn(
"Overload is under maintenance, will retry in 5s...")
time.sleep(5)
else:
raise ex
def get_info(self):
return AndroidInfo()
def post_process(self, retcode):
self.volta_core.post_process()
return retcode
def perform_test(configs, log):
core = Core(configs)
try:
core.configure()
logger.info(
'Starting test... You can interrupt test w/ Ctrl+C or SIGTERM signal'
)
core.start_test()
while True:
time.sleep(1) # infinite loop until SIGTERM
except KeyboardInterrupt:
logger.info(
'Keyboard interrupt, trying graceful shutdown. Do not press interrupt again, '
'otherwise test might be broken')
core.end_test()
except Exception:
logger.error('Uncaught exception in core\n', exc_info=True)
core.end_test()
finally:
core.post_process()
try:
shutil.move(log, os.path.join(core.data_session.artifacts_dir,
log))
except Exception:
logger.warning('Failed to move logfile %s to artifacts dir', log)
logger.debug('Failed to move logfile %s to artifacts dir',
log,
exc_info=True)