def _init(cls, conf_file, *args, **kwargs):
"""
Base class to run LISA test experiments
"""
cls.logger = logging.getLogger('test')
cls.logger.setLevel(logging.INFO)
if 'loglevel' in kwargs:
cls.logger.setLevel(kwargs['loglevel'])
kwargs.pop('loglevel')
cls.conf_file = conf_file
cls.logger.info("%14s - Using configuration:",
"LisaTest")
cls.logger.info("%14s - %s",
"LisaTest", cls.conf_file)
cls.logger.debug("%14s - Load test specific configuration...", "LisaTest")
json_conf = JsonConf(cls.conf_file)
cls.conf = json_conf.load()
cls.logger.debug("%14s - Checking tests configuration...", "LisaTest")
cls._checkConf()
cls._runExperiments()
def _load_board(self, board):
board_path = os.path.join(basepath,
'libs/utils/platforms', board.lower() + '.json')
self._log.debug('Trying to load board descriptor from %s', board_path)
if not os.path.exists(board_path):
return None
self._log.info('Loading board:')
self._log.info(' %s', board_path)
board = JsonConf(board_path)
board.load()
if 'board' not in board.json:
return None
return board.json['board']
def _load_em(self, board):
em_path = os.path.join(basepath,
'libs/utils/platforms', board.lower() + '.json')
self._log.debug('Trying to load default EM from %s', em_path)
if not os.path.exists(em_path):
return None
self._log.info('Loading default EM:')
self._log.info(' %s', em_path)
board = JsonConf(em_path)
board.load()
if 'nrg_model' not in board.json:
return None
return board.json['nrg_model']
def _load_board(self, board):
board_path = os.path.join(basepath,
'libs/utils/platforms', board.lower() + '.json')
self._log.debug('Trying to load board descriptor from %s', board_path)
if not os.path.exists(board_path):
return None
self._log.info('Loading board:')
self._log.info(' %s', board_path)
board = JsonConf(board_path)
board.load()
if 'board' not in board.json:
return None
return board.json['board']
def _load_em(self, board):
em_path = os.path.join(basepath,
'libs/utils/platforms', board.lower() + '.json')
self._log.debug('Trying to load default EM from %s', em_path)
if not os.path.exists(em_path):
return None
self._log.info('Loading default EM:')
self._log.info(' %s', em_path)
board = JsonConf(em_path)
board.load()
if 'nrg_model' not in board.json:
return None
return board.json['nrg_model']
def setUpTest(cls, tests_config):
# Initialize globals
cls.kernel = None
cls.dtb = None
cls.governor = None
cls.cgroup = None
cls.print_section('Main', 'Experiments configuration')
# Load test specific configuration
tests_config = os.path.join('tests/eas', tests_config)
logging.info('%14s - Loading EAS RFC tests configuration [%s]...',
'Main', tests_config)
json_conf = JsonConf(tests_config)
cls.conf = json_conf.load()
# Check for mandatory configurations
if 'confs' not in cls.conf or not cls.conf['confs']:
raise ValueError(
'Configuration error: missing \'conf\' definitions')
if 'wloads' not in cls.conf or not cls.conf['wloads']:
raise ValueError(
'Configuration error: missing \'wloads\' definitions')
# Setup devlib to access the configured target
cls.env = TestEnv(test_conf = cls.conf)
# Compute total number of experiments
cls.exp_count = cls.conf['iterations'] \
* len(cls.conf['wloads']) \
* len(cls.conf['confs'])
cls.print_section('Main', 'Experiments execution')
# Run all the configured experiments
exp_idx = 1
for tc in cls.conf['confs']:
# TARGET: configuration
if not cls.target_configure(tc):
continue
for wl_idx in cls.conf['wloads']:
# TEST: configuration
wload = cls.wload_init(tc, wl_idx)
for itr_idx in range(1, cls.conf['iterations']+1):
# WORKLOAD: execution
cls.wload_run(exp_idx, tc, wl_idx, wload, itr_idx)
exp_idx += 1
cls.print_section('Main', 'Experiments post-processing')
def setUpTest(cls, tests_config):
# Initialize globals
cls.kernel = None
cls.dtb = None
cls.governor = None
cls.cgroup = None
cls.print_section('Main', 'Experiments configuration')
# Load test specific configuration
tests_config = os.path.join('tests/eas', tests_config)
logging.info('%14s - Loading EAS RFC tests configuration [%s]...',
'Main', tests_config)
json_conf = JsonConf(tests_config)
cls.conf = json_conf.load()
# Check for mandatory configurations
if 'confs' not in cls.conf or not cls.conf['confs']:
raise ValueError(
'Configuration error: missing \'conf\' definitions')
if 'wloads' not in cls.conf or not cls.conf['wloads']:
raise ValueError(
'Configuration error: missing \'wloads\' definitions')
# Setup devlib to access the configured target
cls.env = TestEnv(test_conf=cls.conf)
# Compute total number of experiments
cls.exp_count = cls.conf['iterations'] \
* len(cls.conf['wloads']) \
* len(cls.conf['confs'])
cls.print_section('Main', 'Experiments execution')
# Run all the configured experiments
exp_idx = 1
for tc in cls.conf['confs']:
# TARGET: configuration
if not cls.target_configure(tc):
continue
for wl_idx in cls.conf['wloads']:
# TEST: configuration
wload = cls.wload_init(tc, wl_idx)
for itr_idx in range(1, cls.conf['iterations'] + 1):
# WORKLOAD: execution
cls.wload_run(exp_idx, tc, wl_idx, wload, itr_idx)
exp_idx += 1
cls.print_section('Main', 'Experiments post-processing')
def loadTargetConfig(self, filepath='target.config'):
"""
Load the target configuration from the specified file.
:param filepath: Path of the target configuration file. Relative to the
root folder of the test suite.
:type filepath: str
"""
# Loading default target configuration
conf_file = os.path.join(basepath, filepath)
self._log.info('Loading target configuration [%s]...', conf_file)
conf = JsonConf(conf_file)
conf.load()
return conf.json
def loadTargetConfig(self, filepath='target.config'):
"""
Load the target configuration from the specified file.
:param filepath: Path of the target configuration file. Relative to the
root folder of the test suite.
:type filepath: str
"""
# Loading default target configuration
conf_file = os.path.join(basepath, filepath)
self._log.info('Loading target configuration [%s]...', conf_file)
conf = JsonConf(conf_file)
conf.load()
return conf.json
def loadTargetConfig(filepath='target.config'):
"""
Load the target configuration from the specified file.
The configuration file path must be relative to the test suite
installation root folder.
:param filepath: A string representing the path of the target
configuration file. This path must be relative to the root folder of
the test suite.
:type filepath: str
"""
# Loading default target configuration
conf_file = os.path.join(basepath, filepath)
logging.info('%14s - Loading target configuration [%s]...',
'Target', conf_file)
conf = JsonConf(conf_file)
conf.load()
return conf.json
def __init__(self, test_env, experiments_conf):
# Initialize globals
self._default_cgroup = None
self._cgroup = None
self._old_selinux_mode = None
# Setup logging
self._log = logging.getLogger('Executor')
# Setup test configuration
if isinstance(experiments_conf, dict):
self._log.info('Loading custom (inline) test configuration')
self._experiments_conf = experiments_conf
elif isinstance(experiments_conf, str):
self._log.info('Loading custom (file) test configuration')
json_conf = JsonConf(experiments_conf)
self._experiments_conf = json_conf.load()
else:
raise ValueError(
'experiments_conf must be either a dictionary or a filepath')
# Check for mandatory configurations
if not self._experiments_conf.get('confs', None):
raise ValueError('Configuration error: '
'missing "conf" definitions')
if not self._experiments_conf.get('wloads', None):
raise ValueError('Configuration error: '
'missing "wloads" definitions')
self.te = test_env
self.target = self.te.target
self.iterations = self._experiments_conf.get('iterations', 1)
# Compute total number of experiments
self._exp_count = self.iterations \
* len(self._experiments_conf['wloads']) \
* len(self._experiments_conf['confs'])
self._print_section('Experiments configuration')
self._log.info('Configured to run:')
self._log.info(' %3d target configurations:',
len(self._experiments_conf['confs']))
target_confs = [
conf['tag'] for conf in self._experiments_conf['confs']
]
target_confs = ', '.join(target_confs)
self._log.info(' %s', target_confs)
self._log.info(' %3d workloads (%d iterations each)',
len(self._experiments_conf['wloads']), self.iterations)
wload_confs = ', '.join(self._experiments_conf['wloads'])
self._log.info(' %s', wload_confs)
self._log.info('Total: %d experiments', self._exp_count)
self._log.info('Results will be collected under:')
self._log.info(' %s', self.te.res_dir)
if any(wl['type'] == 'rt-app'
for wl in self._experiments_conf['wloads'].values()):
self._log.info('rt-app workloads found, installing tool on target')
self.te.install_tools(['rt-app'])
def __init__(self, test_env, experiments_conf):
# Initialize globals
self._default_cgroup = None
self._cgroup = None
self._old_selinux_mode = None
# Setup logging
self._log = logging.getLogger('Executor')
# Setup test configuration
if isinstance(experiments_conf, dict):
self._log.info('Loading custom (inline) test configuration')
self._experiments_conf = experiments_conf
elif isinstance(experiments_conf, str):
self._log.info('Loading custom (file) test configuration')
json_conf = JsonConf(experiments_conf)
self._experiments_conf = json_conf.load()
else:
raise ValueError(
'experiments_conf must be either a dictionary or a filepath')
# Check for mandatory configurations
if not self._experiments_conf.get('confs', None):
raise ValueError('Configuration error: '
'missing "conf" definitions')
if not self._experiments_conf.get('wloads', None):
raise ValueError('Configuration error: '
'missing "wloads" definitions')
self.te = test_env
self.target = self.te.target
self.iterations = self._experiments_conf.get('iterations', 1)
# Compute total number of experiments
self._exp_count = self.iterations \
* len(self._experiments_conf['wloads']) \
* len(self._experiments_conf['confs'])
self._print_section('Experiments configuration')
self._log.info('Configured to run:')
self._log.info(' %3d target configurations:',
len(self._experiments_conf['confs']))
target_confs = [conf['tag'] for conf in self._experiments_conf['confs']]
target_confs = ', '.join(target_confs)
self._log.info(' %s', target_confs)
self._log.info(' %3d workloads (%d iterations each)',
len(self._experiments_conf['wloads']),
self.iterations)
wload_confs = ', '.join(self._experiments_conf['wloads'])
self._log.info(' %s', wload_confs)
self._log.info('Total: %d experiments', self._exp_count)
self._log.info('Results will be collected under:')
self._log.info(' %s', self.te.res_dir)
if any(wl['type'] == 'rt-app'
for wl in self._experiments_conf['wloads'].values()):
self._log.info('rt-app workloads found, installing tool on target')
self.te.install_tools(['rt-app'])
def __init__(self, target_conf=None, tests_conf=None):
"""
Tests Executor
A tests executor is a module which support the execution of a
configured set of experiments. Each experiment is composed by:
- a target configuration
- a worload to execute
The executor module can be configured to run a set of workloads
(wloads) in each different target configuration of a specified set
(confs). These wloads and confs can be specified by the "tests_config"
input dictionary.
All the results generated by each experiment will be collected a result
folder which is named according to this template:
results/<test_id>/<wltype>:<conf>:<wload>/<run_id>
where:
- <test_id> : the "tid" defined by the tests_config, or a timestamp
based folder in case "tid" is not specified
- <wltype> : the class of workload executed, e.g. rtapp or sched_perf
- <conf> : the identifier of one of the specified configurations
- <wload> : the identified of one of the specified workload
- <run_id> : the progressive execution number from 1 up to the
specified iterations
"""
# Initialize globals
self._default_cgroup = None
self._cgroup = None
# Setup test configuration
if isinstance(tests_conf, dict):
logging.info('%14s - Loading custom (inline) test configuration',
'Target')
self._tests_conf = tests_conf
elif isinstance(tests_conf, str):
logging.info('%14s - Loading custom (file) test configuration',
'Target')
json_conf = JsonConf(tests_conf)
self._tests_conf = json_conf.load()
else:
raise ValueError('test_conf must be either a dictionary or a filepath')
# Check for mandatory configurations
if 'confs' not in self._tests_conf or not self._tests_conf['confs']:
raise ValueError(
'Configuration error: missing \'conf\' definitions')
if 'wloads' not in self._tests_conf or not self._tests_conf['wloads']:
raise ValueError(
'Configuration error: missing \'wloads\' definitions')
# Setup devlib to access the configured target
self.te = TestEnv(target_conf, tests_conf)
self.target = self.te.target
# Compute total number of experiments
self._exp_count = self._tests_conf['iterations'] \
* len(self._tests_conf['wloads']) \
* len(self._tests_conf['confs'])
self._print_section('Executor', 'Experiments configuration')
logging.info('%14s - Configured to run:', 'Executor')
logging.info('%14s - %3d targt configurations:',
'Executor', len(self._tests_conf['confs']))
target_confs = [conf['tag'] for conf in self._tests_conf['confs']]
target_confs = ', '.join(target_confs)
logging.info('%14s - %s', 'Executor', target_confs)
logging.info('%14s - %3d workloads (%d iterations each)',
'Executor', len(self._tests_conf['wloads']),
self._tests_conf['iterations'])
wload_confs = ', '.join(self._tests_conf['wloads'])
logging.info('%14s - %s', 'Executor', wload_confs)
logging.info('%14s - Total: %d experiments',
'Executor', self._exp_count)
logging.info('%14s - Results will be collected under:', 'Executor')
logging.info('%14s - %s', 'Executor', self.te.res_dir)
import json
import os
from conf import JsonConf
# Add identifiers for each of the platforms we have JSON descriptors for
this_dir = os.path.dirname(__file__)
for file_name in os.listdir(this_dir):
name, ext = os.path.splitext(file_name)
if ext == '.json':
platform = JsonConf(os.path.join(this_dir, file_name)).load()
identifier = name.replace('-', '_')
globals()[identifier] = platform
def __init__(self, target_conf=None, tests_conf=None):
"""
Tests Executor
A tests executor is a module which support the execution of a
configured set of experiments. Each experiment is composed by:
- a target configuration
- a worload to execute
The executor module can be configured to run a set of workloads
(wloads) in each different target configuration of a specified set
(confs). These wloads and confs can be specified by the "tests_config"
input dictionary.
All the results generated by each experiment will be collected a result
folder which is named according to this template:
results/<test_id>/<wltype>:<conf>:<wload>/<run_id>
where:
- <test_id> : the "tid" defined by the tests_config, or a timestamp
based folder in case "tid" is not specified
- <wltype> : the class of workload executed, e.g. rtapp or sched_perf
- <conf> : the identifier of one of the specified configurations
- <wload> : the identified of one of the specified workload
- <run_id> : the progressive execution number from 1 up to the
specified iterations
"""
# Initialize globals
self._default_cgroup = None
self._cgroup = None
# Setup test configuration
if isinstance(tests_conf, dict):
logging.info('%14s - Loading custom (inline) test configuration',
'Target')
self._tests_conf = tests_conf
elif isinstance(tests_conf, str):
logging.info('%14s - Loading custom (file) test configuration',
'Target')
json_conf = JsonConf(tests_conf)
self._tests_conf = json_conf.load()
else:
raise ValueError('test_conf must be either a dictionary or a filepath')
# Check for mandatory configurations
if 'confs' not in self._tests_conf or not self._tests_conf['confs']:
raise ValueError(
'Configuration error: missing \'conf\' definitions')
if 'wloads' not in self._tests_conf or not self._tests_conf['wloads']:
raise ValueError(
'Configuration error: missing \'wloads\' definitions')
# Setup devlib to access the configured target
self.te = TestEnv(target_conf, tests_conf)
self.target = self.te.target
# Compute total number of experiments
self._exp_count = self._tests_conf['iterations'] \
* len(self._tests_conf['wloads']) \
* len(self._tests_conf['confs'])
self._print_section('Executor', 'Experiments configuration')
logging.info('%14s - Configured to run:', 'Executor')
logging.info('%14s - %3d targt configurations:',
'Executor', len(self._tests_conf['confs']))
target_confs = [conf['tag'] for conf in self._tests_conf['confs']]
target_confs = ', '.join(target_confs)
logging.info('%14s - %s', 'Executor', target_confs)
logging.info('%14s - %3d workloads (%d iterations each)',
'Executor', len(self._tests_conf['wloads']),
self._tests_conf['iterations'])
wload_confs = ', '.join(self._tests_conf['wloads'])
logging.info('%14s - %s', 'Executor', wload_confs)
logging.info('%14s - Total: %d experiments',
'Executor', self._exp_count)
logging.info('%14s - Results will be collected under:', 'Executor')
logging.info('%14s - %s', 'Executor', self.te.res_dir)
