def setUp(self):
self.config_v1= PiraConfiguration()
self.config_v1.set_build_directories([tempdir + '/home/pira/build_dir'])
self.config_v1.populate_build_dict(self.config_v1.directories)
self.config_v1.set_items(['item1', 'item2'], self.config_v1.directories[0])
self.config_v1.initialize_item_dict(self.config_v1.directories[0], self.config_v1.builds[self.config_v1.directories[0]]['items'])
for build_dir in self.config_v1.directories:
for item in self.config_v1.builds[build_dir]['items']:
self.config_v1.set_item_instrument_analysis(dep_aw_ins_anal[item], build_dir, item)
self.config_v1.set_item_builders(tempdir + '/home/pira/build_dir/item1/functors', build_dir, item)
self.config_v1.set_item_args([], build_dir, item)
self.config_v1.set_item_runner("/" + item + "/runner/functors.dir", build_dir, item)
self.config_v2 = PiraConfigurationII()
pira_item1 = PiraItem("item1")
pira_item1.set_analyzer_dir(tempdir + "/home/pira/build_dir/item1/analyzer")
pira_item1.set_cubes_dir(tempdir + "/home/pira/build_dir/item1/cubes")
pira_item1.set_flavors(["ct"])
pira_item1.set_functors_base_path(tempdir + "/home/pira/build_dir/item1/functors")
pira_item1.set_mode("CT")
self.config_v2.add_item(tempdir + "/home/pira/build_dir/item1/",pira_item1)
pira_item2 = PiraItem("item2")
pira_item2.set_analyzer_dir(tempdir + "/home/pira/build_dir/item2/analyzer")
pira_item2.set_cubes_dir(tempdir + "/home/pira/build_dir/item2/cubes")
pira_item2.set_flavors([])
pira_item2.set_functors_base_path(tempdir + "/home/pira/build_dir/item2/functors")
pira_item2.set_mode("CT")
self.config_v2.add_item(tempdir + "/home/pira/build_dir/item2/",pira_item2)
self.config_adapter = PiraConfigurationAdapter(self.config_v2)
self.create_tempfiles(self)
def setUp(self):
self._path_to_config = '/tmp'
self._pira_dir = os.path.join(os.path.expanduser('~'), '.pira')
self._compile_t_filter = True
self._pira_iters = 3
self._num_reps = 4
self._ic = InvocationConfiguration(self._path_to_config,
self._compile_t_filter,
self._pira_iters, self._num_reps)
self._pira_one_cfg = PiraConfiguration()
self._pira_two_cfg = PiraConfigurationII()
self._it_dir = '/tmp/test_item'
item = PiraItem('/tmp/test_item')
item.set_analyzer_dir('/analyzer')
item.set_cubes_dir('/cubes')
item.set_flavors(['dflt'])
item.set_functors_base_path('/functors')
item.set_mode('ct')
run_opts = CmdlineLinearArgumentMapper({'x': [1]})
item.set_run_options(run_opts)
self._item = item
self._pira_two_cfg.add_item(self._it_dir, item)
self._pira_two_cfg._empty = False # This is usually done in ConfigurationLoader
self._pira_two_adapter = PiraConfigurationAdapter(self._pira_two_cfg)
def test_init_empty_config(self):
prf = PiraRunnerFactory(self._ic, PiraConfiguration())
self.assertIsNotNone(prf)
self.assertTrue(isinstance(prf, PiraRunnerFactory))
prfII = PiraRunnerFactory(self._ic, PiraConfigurationII())
self.assertIsNotNone(prf)
self.assertTrue(isinstance(prfII, PiraRunnerFactory))
def setUp(self):
# Pira I configuration (we probably drop the support anyway...)
self._p_cfg = PiraConfiguration()
# Pira II configuration and adapter
self._pira_two_cfg = PiraConfigurationII()
self._it_dir = '/tmp/test_item'
item = PiraItem('/tmp/test_item')
item.set_analyzer_dir('/analyzer')
item.set_cubes_dir('/cubes')
item.set_flavors(['dflt'])
item.set_functors_base_path('/functors')
item.set_mode('ct')
run_opts = CmdlineLinearArgumentMapper({'x': [1]})
item.set_run_options(run_opts)
self._item = item
self._pira_two_cfg.add_item(self._it_dir, item)
self._pira_two_cfg._empty = False # This is usually done in ConfigurationLoader
self._pira_two_adapter = PiraConfigurationAdapter(self._pira_two_cfg)
def test_get_scalability_runner_empty_config(self):
prf = PiraRunnerFactory(self._ic, PiraConfiguration())
self.assertIsNotNone(prf)
ep_cfg = ExtrapConfiguration('/extrap', 'pre', 'post')
with self.assertRaises(PiraConfigurationErrorException):
prf.get_scalability_runner(ep_cfg)
prfII = PiraRunnerFactory(self._ic, PiraConfigurationII())
self.assertIsNotNone(prfII)
with self.assertRaises(PiraConfigurationErrorException):
prfII.get_scalability_runner(ep_cfg)
class CheckerTestCase(unittest.TestCase):
@classmethod
def setUp(self):
self.config_v1= PiraConfiguration()
self.config_v1.set_build_directories([tempdir + '/home/pira/build_dir'])
self.config_v1.populate_build_dict(self.config_v1.directories)
self.config_v1.set_items(['item1', 'item2'], self.config_v1.directories[0])
self.config_v1.initialize_item_dict(self.config_v1.directories[0], self.config_v1.builds[self.config_v1.directories[0]]['items'])
for build_dir in self.config_v1.directories:
for item in self.config_v1.builds[build_dir]['items']:
self.config_v1.set_item_instrument_analysis(dep_aw_ins_anal[item], build_dir, item)
self.config_v1.set_item_builders(tempdir + '/home/pira/build_dir/item1/functors', build_dir, item)
self.config_v1.set_item_args([], build_dir, item)
self.config_v1.set_item_runner("/" + item + "/runner/functors.dir", build_dir, item)
self.config_v2 = PiraConfigurationII()
pira_item1 = PiraItem("item1")
pira_item1.set_analyzer_dir(tempdir + "/home/pira/build_dir/item1/analyzer")
pira_item1.set_cubes_dir(tempdir + "/home/pira/build_dir/item1/cubes")
pira_item1.set_flavors(["ct"])
pira_item1.set_functors_base_path(tempdir + "/home/pira/build_dir/item1/functors")
pira_item1.set_mode("CT")
self.config_v2.add_item(tempdir + "/home/pira/build_dir/item1/",pira_item1)
pira_item2 = PiraItem("item2")
pira_item2.set_analyzer_dir(tempdir + "/home/pira/build_dir/item2/analyzer")
pira_item2.set_cubes_dir(tempdir + "/home/pira/build_dir/item2/cubes")
pira_item2.set_flavors([])
pira_item2.set_functors_base_path(tempdir + "/home/pira/build_dir/item2/functors")
pira_item2.set_mode("CT")
self.config_v2.add_item(tempdir + "/home/pira/build_dir/item2/",pira_item2)
self.config_adapter = PiraConfigurationAdapter(self.config_v2)
self.create_tempfiles(self)
def tearDown(self):
self.delete_tempfolders()
def create_tempfiles(self):
for directory in directories_to_create:
U.make_dirs(tempdir + directory)
for filepath in functor_files:
tempfile = open(tempdir + filepath,'a')
tempfile.close()
def delete_tempfolders(self):
U.remove_dir(tempdir + "/home/pira/")
def test_checker_v1_valid_config(self):
C.Checker.check_configfile_v1(self.config_v1)
def test_checker_v1_general_valid_config(self):
C.Checker.check_configfile(self.config_v1,1)
def test_checker_v1_dirs_missing(self):
for directory in directories_to_create:
U.remove_dir(tempdir + directory)
with self.assertRaises(PiraConfigurationErrorException): C.Checker.check_configfile_v1(self.config_v1)
self.create_tempfiles()
def test_checker_v2_valid_config(self):
C.Checker.check_configfile_v2(self.config_v2)
def test_checker_v2_general_valid_config(self):
C.Checker.check_configfile(self.config_v2,2)
def test_checker_v2_adapter_valid_config(self):
C.Checker.check_configfile_v2(self.config_adapter)
def test_checker_v2_functors_missing(self):
for file in functor_files:
U.remove_file(tempdir + file)
with self.assertRaises(PiraConfigurationErrorException): C.Checker.check_configfile_v2(self.config_v2)
self.create_tempfiles()
def test_checker_v2_dirs_missing(self):
for directory in directories_to_create:
U.remove_dir(tempdir + directory)
with self.assertRaises(PiraConfigurationErrorException): C.Checker.check_configfile_v2(self.config_v2)
self.create_tempfiles()
def test_check_basic_config_005(self):
cl = CL.SimplifiedConfigurationLoader()
cfg = cl.load_conf('../inputs/configs/basic_config_005.json')
C.Checker.check_configfile_v2(cfg)
class TestRunnerFactory(unittest.TestCase):
def setUp(self):
self._path_to_config = '/tmp'
self._pira_dir = os.path.join(os.path.expanduser('~'), '.pira')
self._compile_t_filter = True
self._pira_iters = 3
self._num_reps = 4
self._ic = InvocationConfiguration(self._path_to_config,
self._compile_t_filter,
self._pira_iters, self._num_reps)
self._pira_one_cfg = PiraConfiguration()
self._pira_two_cfg = PiraConfigurationII()
self._it_dir = '/tmp/test_item'
item = PiraItem('/tmp/test_item')
item.set_analyzer_dir('/analyzer')
item.set_cubes_dir('/cubes')
item.set_flavors(['dflt'])
item.set_functors_base_path('/functors')
item.set_mode('ct')
run_opts = CmdlineLinearArgumentMapper({'x': [1]})
item.set_run_options(run_opts)
self._item = item
self._pira_two_cfg.add_item(self._it_dir, item)
self._pira_two_cfg._empty = False # This is usually done in ConfigurationLoader
self._pira_two_adapter = PiraConfigurationAdapter(self._pira_two_cfg)
def test_init_empty_config(self):
prf = PiraRunnerFactory(self._ic, PiraConfiguration())
self.assertIsNotNone(prf)
self.assertTrue(isinstance(prf, PiraRunnerFactory))
prfII = PiraRunnerFactory(self._ic, PiraConfigurationII())
self.assertIsNotNone(prf)
self.assertTrue(isinstance(prfII, PiraRunnerFactory))
def test_init_nonempty_config(self):
prf = PiraRunnerFactory(self._ic, self._pira_two_cfg)
self.assertIsNotNone(prf)
self.assertTrue(isinstance(prf, PiraRunnerFactory))
prfII = PiraRunnerFactory(self._ic, self._pira_two_adapter)
self.assertIsNotNone(prfII)
self.assertTrue(isinstance(prf, PiraRunnerFactory))
def test_get_simple_local_runner_empty_config(self):
prf = PiraRunnerFactory(self._ic, PiraConfiguration())
self.assertIsNotNone(prf)
runner = prf.get_simple_local_runner()
self.assertIsNotNone(runner)
self.assertTrue(isinstance(runner, LocalRunner))
self.assertTrue(runner.has_sink())
def test_get_scalability_runner_empty_config(self):
prf = PiraRunnerFactory(self._ic, PiraConfiguration())
self.assertIsNotNone(prf)
ep_cfg = ExtrapConfiguration('/extrap', 'pre', 'post')
with self.assertRaises(PiraConfigurationErrorException):
prf.get_scalability_runner(ep_cfg)
prfII = PiraRunnerFactory(self._ic, PiraConfigurationII())
self.assertIsNotNone(prfII)
with self.assertRaises(PiraConfigurationErrorException):
prfII.get_scalability_runner(ep_cfg)
def test_get_scalability_runner_nonempty_config(self):
prfII = PiraRunnerFactory(self._ic, self._pira_two_cfg)
self.assertIsNotNone(prfII)
ep_cfg = ExtrapConfiguration('/extrap', 'pre', 'post')
runner = prfII.get_scalability_runner(ep_cfg)
self.assertIsNotNone(runner)
self.assertTrue(isinstance(runner, LocalScalingRunner))
def test_get_scalability_runner_config_adapter(self):
prfII = PiraRunnerFactory(self._ic, self._pira_two_adapter)
self.assertIsNotNone(prfII)
ep_cfg = ExtrapConfiguration('/extrap', 'pre', 'post')
runner = prfII.get_scalability_runner(ep_cfg)
self.assertIsNotNone(runner)
self.assertTrue(isinstance(runner, LocalScalingRunner))
def __init__(self):
self._config = PiraConfigurationII()
self.base_mapper = None
class SimplifiedConfigurationLoader:
def __init__(self):
self._config = PiraConfigurationII()
self.base_mapper = None
def load_conf(self, config_file: str) -> PiraConfiguration:
if not U.is_file(config_file):
raise RuntimeError(
'SimplifiedConfigurationLoader::load_conf: Invalid config file location "'
+ config_file + '" [no such file].')
config_abs = U.get_absolute_path(config_file)
config_abs_path = config_abs[:config_abs.rfind('/')]
self._config.set_absolute_base_path(config_abs_path)
try:
file_content = U.read_file(config_file)
json_tree = json.loads(file_content)
self.parse_from_json(json_tree)
except Exception as e:
L.get_logger().log(
'SimplifiedConfigurationLoader::load_conf: Caught exception "'
+ str(e))
return PiraConfigurationAdapter(self._config)
def is_escaped(self, string: str) -> bool:
return string.startswith('%')
def get_parameter(self, item_tree, item_key):
run_opts = {}
run_opts['mapper'] = U.json_to_canonic(
item_tree[item_key]['argmap']['mapper'])
params = {}
param_tree = item_tree[item_key]['argmap']
file_mapper = False
if 'pira-file' in param_tree:
run_opts['pira-file'] = []
run_opts['pira-file'] = U.json_to_canonic(
param_tree['pira-file']['names'])
param_tree = param_tree['pira-file']
file_mapper = True
for param in param_tree:
parameter = U.json_to_canonic(param)
if param == 'mapper':
continue
if file_mapper and param == 'names':
continue
try:
params[parameter]
except:
params[parameter] = []
params[parameter] = U.json_to_canonic(param_tree[param])
run_opts['argmap'] = params
return run_opts
def create_item_from_json(self, item_key: str, item_tree):
pira_item = PiraItem(item_key)
analyzer_dir = item_tree[item_key]['analyzer']
if analyzer_dir is '':
analyzer_dir = U.get_base_dir(
__file__) + '/../extern/install/pgis/bin'
L.get_logger().log('Analyzer: using analyzer default: ' +
analyzer_dir,
level='debug')
cubes_dir = item_tree[item_key]['cubes']
flavors = item_tree[item_key]['flavors']
functors_base_path = item_tree[item_key]['functors']
mode = item_tree[item_key]['mode']
run_opts = self.get_parameter(item_tree, item_key)
run_options = ArgumentMapperFactory.get_mapper(run_opts)
pira_item.set_analyzer_dir(analyzer_dir)
pira_item.set_cubes_dir(cubes_dir)
pira_item.set_flavors(flavors)
pira_item.set_functors_base_path(functors_base_path)
pira_item.set_mode(mode)
pira_item.set_run_options(run_options)
return pira_item
def parse_from_json(self, json_tree) -> None:
# Top-level key elements // theoretically not required
try:
directories = U.json_to_canonic(json_tree[_DIRS])
except Exception as e:
L.get_logger().log(
'SimplifiedConfigurationLoader::parse_from_json: ' + str(e))
directories = {}
for tld_build in json_tree[_BUILDS]:
# These are the elements, i.e., %astar and alike
directory_for_item = U.json_to_canonic(tld_build)
if self.is_escaped(directory_for_item):
directory_for_item = directories[directory_for_item[1:]]
item_tree = U.json_to_canonic(
json_tree[_BUILDS][tld_build][_ITEMS])
for item_key in item_tree:
L.get_logger().log(
'SimplifiedConfigurationLoader::parse_from_json: ' +
str(item_key))
pira_item = self.create_item_from_json(item_key, item_tree)
self._config.add_item(directory_for_item, pira_item)
self._config._empty = False
def test_empty_pira_config_adapter(self):
with self.assertRaises(A.PiraAnalyzerException):
analyzer = A.Analyzer(PiraConfigurationAdapter(PiraConfigurationII()))
class TestAnalyzer(unittest.TestCase):
def setUp(self):
# Pira I configuration (we probably drop the support anyway...)
self._p_cfg = PiraConfiguration()
# Pira II configuration and adapter
self._pira_two_cfg = PiraConfigurationII()
self._it_dir = '/tmp/test_item'
item = PiraItem('/tmp/test_item')
item.set_analyzer_dir('/analyzer')
item.set_cubes_dir('/cubes')
item.set_flavors(['dflt'])
item.set_functors_base_path('/functors')
item.set_mode('ct')
run_opts = CmdlineLinearArgumentMapper({'x': [1]})
item.set_run_options(run_opts)
self._item = item
self._pira_two_cfg.add_item(self._it_dir, item)
self._pira_two_cfg._empty = False # This is usually done in ConfigurationLoader
self._pira_two_adapter = PiraConfigurationAdapter(self._pira_two_cfg)
def test_empty_pira_config(self):
with self.assertRaises(A.PiraAnalyzerException):
analyzer = A.Analyzer(PiraConfiguration())
def test_empty_pira_configII(self):
with self.assertRaises(A.PiraAnalyzerException):
analyzer = A.Analyzer(PiraConfigurationII())
def test_empty_pira_config_adapter(self):
with self.assertRaises(A.PiraAnalyzerException):
analyzer = A.Analyzer(PiraConfigurationAdapter(PiraConfigurationII()))
def test_pira_configII(self):
analyzer = A.Analyzer(self._pira_two_cfg)
self.assertIsNotNone(analyzer)
def test_pira_config_adapter(self):
analyzer = A.Analyzer(self._pira_two_adapter)
self.assertIsNotNone(analyzer)
def test_config_empty_sink(self):
analyzer = A.Analyzer(self._pira_two_cfg)
tc = TargetConfiguration(self._it_dir, self._it_dir, self._it_dir, 'dflt', 'asdf')
with self.assertRaises(RuntimeError):
analyzer.analyze(tc, 0)
def test_empty_target_config(self):
analyzer = A.Analyzer(self._pira_two_cfg)
with self.assertRaises(RuntimeError):
analyzer.analyze(None, 0)
def test_analyze_local(self):
ld = C.SimplifiedConfigurationLoader()
cfg = ld.load_conf('../inputs/configs/basic_config_005.json')
analyzer = A.Analyzer(cfg)
fm = F.FunctorManager(cfg)
a_f = fm.get_or_load_functor('/tmp', 'test_item', 'ct', 'analyze')
self.assertIsNotNone(a_f)
self.assertTrue(a_f.get_method()['passive'])
self.assertEqual(a_f.get_it(), 0)
tc = TargetConfiguration(cfg.get_place('/tmp'), '/tmp', 'test_item', 'ct', 'asdf')
with self.assertRaises(RuntimeError) as assert_cm:
analyzer.analyze(tc, 0)
rt_err = assert_cm.exception
self.assertEqual(str(rt_err), 'Analyzer::analyze: Profile Sink in Analyzer not set!')
analyzer.set_profile_sink(TestProfileSink())
analyzer.analyze(tc, 0)
self.assertEqual(a_f.get_it(), 1)
@unittest.skip('Skip the test of the slurm Analyzer as we do not have any implementation for now.')
def test_analyze_slurm(self):
pass
class SimplifiedConfigurationLoader:
def __init__(self):
self._config = PiraConfigurationII()
self.base_mapper = None
def load_conf(self, config_file: str) -> PiraConfiguration:
if not util.is_file(config_file):
raise RuntimeError(
'SimplifiedConfigurationLoader::load_conf: Invalid config file location "'
+ config_file + '" [no such file].')
config_abs = util.get_absolute_path(config_file)
config_abs_path = config_abs[:config_abs.rfind('/')]
self._config.set_absolute_base_path(config_abs_path)
try:
file_content = util.read_file(config_file)
json_tree = json.loads(file_content)
self.parse_from_json(json_tree)
except Exception as e:
log.get_logger().log(
'SimplifiedConfigurationLoader::load_conf: Caught exception "'
+ str(e))
return PiraConfigurationAdapter(self._config)
def is_escaped(self, string: str) -> bool:
return string.startswith('%')
def get_parameter(self, item_tree, item_key):
run_opts = {}
run_opts['mapper'] = util.json_to_canonic(
item_tree[item_key]['argmap']['mapper'])
params = {}
param_tree = item_tree[item_key]['argmap']
file_mapper = False
if 'pira-file' in param_tree:
run_opts['pira-file'] = []
run_opts['pira-file'] = util.json_to_canonic(
param_tree['pira-file']['names'])
param_tree = param_tree['pira-file']
file_mapper = True
for param in param_tree:
parameter = util.json_to_canonic(param)
if param == 'mapper':
continue
if file_mapper and param == 'names':
continue
try:
params[parameter]
except:
params[parameter] = []
params[parameter] = util.json_to_canonic(param_tree[param])
run_opts['argmap'] = params
return run_opts
def create_item_from_json(self, item_key, item_tree):
pira_item = PiraItem(item_key)
analyzer_dir = item_tree[item_key]['analyzer']
cubes_dir = item_tree[item_key]['cubes']
flavors = item_tree[item_key]['flavors']
functors_base_path = item_tree[item_key]['functors']
mode = item_tree[item_key]['mode']
run_opts = self.get_parameter(item_tree, item_key)
run_options = ArgumentMapperFactory.get_mapper(run_opts)
pira_item.set_analyzer_dir(analyzer_dir)
pira_item.set_cubes_dir(cubes_dir)
pira_item.set_flavors(flavors)
pira_item.set_functors_base_path(functors_base_path)
pira_item.set_mode(mode)
pira_item.set_run_options(run_options)
return pira_item
def parse_from_json(self, json_tree) -> None:
# Top-level key elements // theoretically not required
try:
directories = util.json_to_canonic(json_tree[_DIRS])
except Exception as e:
log.get_logger().log(
'SimplifiedConfigurationLoader::parse_from_json: ' + str(e))
directories = {}
for tld_build in json_tree[_BUILDS]:
# These are the elements, i.e., %astar and alike
directory_for_item = util.json_to_canonic(tld_build)
if self.is_escaped(directory_for_item):
directory_for_item = directories[directory_for_item[1:]]
item_tree = util.json_to_canonic(
json_tree[_BUILDS][tld_build][_ITEMS])
for item_key in item_tree:
pira_item = self.create_item_from_json(item_key, item_tree)
self._config.add_item(directory_for_item, pira_item)
def check_paths(self, config):
error_message = "Error in configuration-file:\n\n"
exception_occured = False
for dir in config.get_directories():
if not (util.check_provided_directory(dir)):
error_message += "Directory " + dir + " does not exist.\n"
exception_occured = True
for item in config.get_items(dir):
if not (util.check_provided_directory(
item.get_analyzer_dir())):
error_message += "Analyzer-Directory " + item.get_analyzer_dir(
) + " does not exist\n"
exception_occured = True
if not (util.check_provided_directory(
item.get_analyzer_dir())):
error_message += "Cubes-Directory " + item.get_cubes_dir(
) + " does not exist\n"
exception_occured = True
if not (util.check_provided_directory(
item.get_functor_base_path())):
error_message += "Functors-Base-Directory " + item.get_functor_base_path(
) + " does not exist\n"
exception_occured = True
for flavor in item.get_flavors():
if not (util.is_file(item.get_functor_base_path() +
"/analyse_" + item._name + "_" +
flavor + ".py")):
error_message += "analyse-functor of flavor " + flavor + " does not exist.\n"
exception_occured = True
if not (util.is_file(item.get_functor_base_path() +
"/clean_" + item._name + "_" +
flavor + ".py")):
error_message += "clean-functor of flavor " + flavor + " does not exist.\n"
exception_occured = True
if not (util.is_file(item.get_functor_base_path() +
"/no_instr_" + item._name + "_" +
flavor + ".py")):
error_message += "no_instr-functor of flavor " + flavor + " does not exist.\n"
exception_occured = True
if not (util.is_file(item.get_functor_base_path() +
"/runner_" + item._name + "_" +
flavor + ".py")):
error_message += "runner-functor of flavor " + flavor + " does not exist.\n"
exception_occured = True
if not (util.is_file(item.get_functor_base_path() + "/" +
item._name + "_" + flavor + ".py")):
error_message += "plain-functor of flavor " + flavor + " does not exist.\n"
exception_occured = True
if exception_occured:
raise PiraConfigurationErrorException(error_message)