def test_construction_from_empty_config(self):
with self.assertRaises(F.FunctorManagementException):
fm = F.FunctorManager(CO.PiraConfiguration())
with self.assertRaises(F.FunctorManagementException):
fm = F.FunctorManager(CO.PiraConfigurationII())
with self.assertRaises(F.FunctorManagementException):
fm = F.FunctorManager(
CO.PiraConfigurationAdapter(CO.PiraConfigurationII()))
def prep_db_for_build_item_in_flavor(self, config, build, item, flavor):
"""Generates all the necessary build work to write to the db.
:config: PIRA configuration
:build: the build
:item: current item
:flavor: current flavor
:returns: unique ID for current item
"""
build_tuple = (u.generate_random_string(), build, '', flavor, build)
self.insert_data_builds(build_tuple)
# XXX My implementation returns the full path, including the file extension.
# In case something in the database goes wild, this could be it.
func_manager = fm.FunctorManager()
analyse_functor = func_manager.get_analyzer_file(build, item, flavor)
build_functor = func_manager.get_builder_file(build, item, flavor)
run_functor = func_manager.get_runner_file(build, item, flavor)
# TODO implement the get_submitter_file(build, item, flavor) method!
benchmark_name = config.get_benchmark_name(item)
submitter_functor = config.get_runner_func(build, item) + '/slurm_submitter_' + benchmark_name + flavor
exp_dir = config.get_analyser_exp_dir(build, item)
db_item_id = u.generate_random_string()
db_item_data = (db_item_id, benchmark_name, analyse_functor, build_functor, '', run_functor, submitter_functor,
exp_dir, build)
self.insert_data_items(db_item_data)
return db_item_id
def test_construction_is_singleton(self):
F.FunctorManager.instance = None
self.assertIsNone(F.FunctorManager.instance)
cfg_loader = C.ConfigurationLoader()
fm = F.FunctorManager.from_config(
cfg_loader.load_conf('./input/unit_input_001.json'))
fm2 = F.FunctorManager()
self.assertEqual(fm.instance, fm2.instance)
fm.reset()
def setUp(self):
self.build = '/home/something/top_dir'
self.b_i_01 = '/builder/item01/directory'
self.ia_i_01 = '/ins_anal/directory/for/functors'
self.r_i_01 = '/path/to/runner_functors/item01'
self.i_01 = 'item01'
self.flavor = 'vanilla'
self.cl = C.ConfigurationLoader()
self.cfg = self.cl.load_conf('./input/unit_input_001.json')
self.fm = F.FunctorManager(self.cfg)
def build_flavors(self, kwargs) -> None:
L.get_logger().log(
'Builder::build_flavors: Building for ' + self.target_config.get_target() + ' in ' +
self.target_config.get_flavor(),
level='debug')
build = self.target_config.get_build()
benchmark = self.target_config.get_target()
flavor = self.target_config.get_flavor()
f_man = F.FunctorManager() # Returns the currently loaded FM
clean_functor = f_man.get_or_load_functor(build, benchmark, flavor, 'clean')
kwargs = {}
if self.build_instr:
L.get_logger().log('Builder::build_flavors: Instrumentation', level='debug')
try:
self.check_build_prerequisites()
L.get_logger().log('Builder::build_flavors: Prerequisite check successfull.')
except Exception as e:
raise BuilderException('Precheck failed.\n' + str(e))
if not self.target_config.is_compile_time_filtering():
L.get_logger().log('Builder::build_flavors: Runtime filtering enabled.')
self.target_config.set_instr_file(self.instrumentation_file)
build_functor = f_man.get_or_load_functor(build, benchmark, flavor, 'build')
kwargs = self.construct_pira_instr_kwargs()
ScorepSystemHelper.prepare_MPI_filtering(self.instrumentation_file)
else:
L.get_logger().log('Builder::build_flavors: No instrumentation', level='debug')
build_functor = f_man.get_or_load_functor(build, benchmark, flavor, 'basebuild')
kwargs = self.construct_pira_kwargs()
if build_functor.get_method()['active']:
L.get_logger().log('Builder::build_flavors: Running the passive functor.', level='debug')
build_functor.active(benchmark, **kwargs)
else:
try:
L.get_logger().log('Builder::build_flavors: Running the passive functor.', level='debug')
''' The build command uses CC and CXX to pass flags that are needed by PIRA for the given toolchain. '''
build_command = build_functor.passive(benchmark, **kwargs)
clean_command = clean_functor.passive(benchmark, **kwargs)
L.get_logger().log(
'Builder::build_flavors: Clean in ' + benchmark + '\n Using ' + clean_command, level='debug')
U.shell(clean_command)
L.get_logger().log('Builder::build_flavors: Building: ' + build_command, level='debug')
U.shell(build_command)
except Exception as e:
L.get_logger().log('Builder::build_flavors: ' + str(e), level='error')
def run(self, target_config: TargetConfiguration,
instrument_config: InstrumentConfig,
compile_time_filtering: bool) -> float:
""" Implements the actual invocation """
functor_manager = fm.FunctorManager()
run_functor = functor_manager.get_or_load_functor(
target_config.get_build(), target_config.get_target(),
target_config.get_flavor(), 'run')
default_provider = defaults.BackendDefaults()
kwargs = default_provider.get_default_kwargs()
kwargs['util'] = util
kwargs['LD_PRELOAD'] = default_provider.get_MPI_wrap_LD_PRELOAD()
runtime = .0
if run_functor.get_method()['active']:
run_functor.active(target_config.get_target(), **kwargs)
log.get_logger().log(
'For the active functor we can barely measure runtime',
level='warn')
runtime = 1.0
try:
util.change_cwd(target_config.get_place())
invoke_arguments = target_config.get_args_for_invocation()
kwargs['args'] = invoke_arguments
if invoke_arguments is not None:
log.get_logger().log('LocalBaseRunner::run: (args) ' +
str(invoke_arguments))
command = run_functor.passive(target_config.get_target(), **kwargs)
_, runtime = util.shell(command, time_invoc=True)
log.get_logger().log(
'LocalBaseRunner::run::passive_invocation -> Returned runtime: '
+ str(runtime),
level='debug')
except Exception as e:
log.get_logger().log('LocalBaseRunner::run Exception\n' + str(e),
level='error')
raise RuntimeError('LocalBaseRunner::run caught exception. ' +
str(e))
# TODO: Insert the data into the database
return runtime
def test_analyze_local(self):
ld = C.SimplifiedConfigurationLoader()
cfg = ld.load_conf('../inputs/configs/basic_config_005.json')
analyzer = A.Analyzer(cfg)
fm = F.FunctorManager(cfg)
a_f = fm.get_or_load_functor('/tmp', 'test_item', 'ct', 'analyze')
self.assertIsNotNone(a_f)
self.assertTrue(a_f.get_method()['passive'])
self.assertEqual(a_f.get_it(), 0)
tc = TargetConfiguration(cfg.get_place('/tmp'), '/tmp', 'test_item', 'ct', 'asdf')
with self.assertRaises(RuntimeError) as assert_cm:
analyzer.analyze(tc, 0)
rt_err = assert_cm.exception
self.assertEqual(str(rt_err), 'Analyzer::analyze: Profile Sink in Analyzer not set!')
analyzer.set_profile_sink(TestProfileSink())
analyzer.analyze(tc, 0)
self.assertEqual(a_f.get_it(), 1)
def main(arguments) -> None:
""" Main function for pira framework. Used to invoke the various components. """
show_pira_invoc_info(arguments)
invoc_cfg = process_args_for_invoc(arguments)
use_extra_p, extrap_config = process_args_for_extrap(arguments)
home_dir = U.get_cwd()
U.set_home_dir(home_dir)
U.make_dir(invoc_cfg.get_pira_dir())
BackendDefaults(invoc_cfg)
csv_config = process_args_for_csv(arguments)
try:
if arguments.config_version is 1:
config_loader = CLoader()
else:
config_loader = SCLoader()
configuration = config_loader.load_conf(invoc_cfg.get_path_to_cfg())
checker.check_configfile(configuration, arguments.config_version)
if B.check_queued_job():
# FIXME: Implement
L.get_logger().log(
'In this version of PIRA it is not yet implemented',
level='error')
assert (False)
else:
'''
This branch is running PIRA actively on the local machine.
It is blocking, and the user can track the progress in the terminal.
'''
L.get_logger().log('Running the local case')
# The FunctorManager manages loaded functors and generates the respective names
F.FunctorManager(configuration)
dbm = D.DBManager(D.DBManager.db_name + '.' + D.DBManager.db_ext)
dbm.create_cursor()
analyzer = A(configuration)
runner_factory = PiraRunnerFactory(invoc_cfg, configuration)
runner = runner_factory.get_simple_local_runner()
if use_extra_p:
L.get_logger().log('Running with Extra-P runner')
runner = runner_factory.get_scalability_runner(extrap_config)
if runner.has_sink():
analyzer.set_profile_sink(runner.get_sink())
# A build/place is a top-level directory
for build in configuration.get_builds():
L.get_logger().log('Build: ' + str(build))
app_tuple = (U.generate_random_string(), build, '', '')
dbm.insert_data_application(app_tuple)
# An item is a target/software in that directory
for item in configuration.get_items(build):
L.get_logger().log('Running for item ' + str(item))
# A flavor is a specific version to build
if configuration.has_local_flavors(build, item):
for flavor in configuration.get_flavors(build, item):
L.get_logger().log('Running for local flavor ' +
flavor,
level='debug')
# prepare database, and get a unique handle for current item.
db_item_id = dbm.prep_db_for_build_item_in_flavor(
configuration, build, item, flavor)
# Create configuration object for the item currently processed.
place = configuration.get_place(build)
t_config = TargetConfiguration(
place, build, item, flavor, db_item_id,
invoc_cfg.is_compile_time_filtering(),
invoc_cfg.get_hybrid_filter_iters())
# Execute using a local runner, given the generated target description
execute_with_config(runner, analyzer,
invoc_cfg.get_pira_iters(),
t_config, csv_config)
# If global flavor
else:
# TODO: Implement
L.get_logger().log(
'In this version of PIRA it is not yet implemented',
level='error')
assert (False)
U.change_cwd(home_dir)
except RuntimeError as rt_err:
U.change_cwd(home_dir)
L.get_logger().log('Runner.run caught exception. Message: ' +
str(rt_err),
level='error')
L.get_logger().dump_tape()
sys.exit(-1)
def test_construction_from_config(self):
cfg_loader = C.ConfigurationLoader()
fm = F.FunctorManager(
cfg_loader.load_conf('./input/unit_input_001.json'))
fm.reset()
def test_get_analyzer_command(self):
self.fm = F.FunctorManager(
self.scl.load_conf(self.get_filename(self.cfg004)))
def test_get_builder_functor(self):
self.fm = F.FunctorManager(
self.scl.load_conf(self.get_filename(self.cfg004)))
def test_get_valid_path(self):
self.fm = F.FunctorManager(
self.scl.load_conf(self.get_filename(self.cfg004)))
def analyze_local(self, flavor: str, build: str, benchmark: str,
kwargs: dict, iterationNumber: int) -> str:
fm = F.FunctorManager()
analyze_functor = fm.get_or_load_functor(build, benchmark, flavor,
'analyze')
analyzer_dir = self.config.get_analyzer_dir(build, benchmark)
kwargs['analyzer_dir'] = analyzer_dir
# The invoke args can be retrieved from the configuration object.
# Since the invoke args are iterable, we can create all necessary argument tuples here.
# We construct a json file that contains the necesary information to be parsed vy the
# PGIS tool. That way, we can make it easily traceable and debug from manual inspection.
# This will be the new standard way of pusing information to PGIS.
pgis_cfg_file = None
if self._profile_sink.has_config_output():
pgis_cfg_file = self._profile_sink.output_config(
benchmark, analyzer_dir)
if analyze_functor.get_method()['active']:
analyze_functor.active(benchmark, **kwargs)
else:
L.get_logger().log('Analyzer::analyze_local: Using passive mode')
try:
exp_dir = self.config.get_analyzer_exp_dir(build, benchmark)
isdirectory_good = U.check_provided_directory(analyzer_dir)
command = analyze_functor.passive(benchmark, **kwargs)
L.get_logger().log('Analyzer::analyze_local: Command = ' +
command)
benchmark_name = self.config.get_benchmark_name(benchmark)
if isdirectory_good:
U.change_cwd(analyzer_dir)
L.get_logger().log('Analyzer::analyzer_local: Flavor = ' +
flavor + ' | benchmark_name = ' +
benchmark_name)
instr_files = U.build_instr_file_path(
analyzer_dir, flavor, benchmark_name)
L.get_logger().log(
'Analyzer::analyzer_local: instrumentation file = ' +
instr_files)
prev_instr_file = U.build_previous_instr_file_path(
analyzer_dir, flavor, benchmark_name)
tracker = T.TimeTracker()
# TODO: Alternate between expansion and pure filtering.
if iterationNumber > 0 and U.is_file(instr_files):
L.get_logger().log(
'Analyzer::analyze_local: instr_file available')
U.rename(instr_files, prev_instr_file)
tracker.m_track('Analysis', U, 'run_analyzer_command',
command, analyzer_dir, flavor,
benchmark_name, exp_dir, iterationNumber,
pgis_cfg_file)
L.get_logger().log(
'Analyzer::analyze_local: command finished',
level='debug')
else:
tracker.m_track('Initial analysis', U,
'run_analyzer_command_noInstr', command,
analyzer_dir, flavor, benchmark_name)
self.tear_down(build, exp_dir)
return instr_files
except Exception as e:
L.get_logger().log(str(e), level='error')
raise Exception('Problem in Analyzer')