Example #1
0
    def test_construction_from_empty_config(self):
        with self.assertRaises(F.FunctorManagementException):
            fm = F.FunctorManager(CO.PiraConfiguration())

        with self.assertRaises(F.FunctorManagementException):
            fm = F.FunctorManager(CO.PiraConfigurationII())

        with self.assertRaises(F.FunctorManagementException):
            fm = F.FunctorManager(
                CO.PiraConfigurationAdapter(CO.PiraConfigurationII()))
Example #2
0
    def prep_db_for_build_item_in_flavor(self, config, build, item, flavor):
      """Generates all the necessary build work to write to the db.
  
      :config: PIRA configuration
      :build: the build
      :item: current item
      :flavor: current flavor
      :returns: unique ID for current item
  
      """
      build_tuple = (u.generate_random_string(), build, '', flavor, build)
      self.insert_data_builds(build_tuple)
      # XXX My implementation returns the full path, including the file extension.
      #     In case something in the database goes wild, this could be it.
      func_manager = fm.FunctorManager()
      analyse_functor = func_manager.get_analyzer_file(build, item, flavor)
      build_functor = func_manager.get_builder_file(build, item, flavor)
      run_functor = func_manager.get_runner_file(build, item, flavor)
      # TODO implement the get_submitter_file(build, item, flavor) method!

      benchmark_name = config.get_benchmark_name(item)
      submitter_functor = config.get_runner_func(build, item) + '/slurm_submitter_' + benchmark_name + flavor
      exp_dir = config.get_analyser_exp_dir(build, item)

      db_item_id = u.generate_random_string()
      db_item_data = (db_item_id, benchmark_name, analyse_functor, build_functor, '', run_functor, submitter_functor,
                      exp_dir, build)
      self.insert_data_items(db_item_data)

      return db_item_id
Example #3
0
 def test_construction_is_singleton(self):
     F.FunctorManager.instance = None
     self.assertIsNone(F.FunctorManager.instance)
     cfg_loader = C.ConfigurationLoader()
     fm = F.FunctorManager.from_config(
         cfg_loader.load_conf('./input/unit_input_001.json'))
     fm2 = F.FunctorManager()
     self.assertEqual(fm.instance, fm2.instance)
     fm.reset()
Example #4
0
 def setUp(self):
     self.build = '/home/something/top_dir'
     self.b_i_01 = '/builder/item01/directory'
     self.ia_i_01 = '/ins_anal/directory/for/functors'
     self.r_i_01 = '/path/to/runner_functors/item01'
     self.i_01 = 'item01'
     self.flavor = 'vanilla'
     self.cl = C.ConfigurationLoader()
     self.cfg = self.cl.load_conf('./input/unit_input_001.json')
     self.fm = F.FunctorManager(self.cfg)
Example #5
0
  def build_flavors(self, kwargs) -> None:
    L.get_logger().log(
        'Builder::build_flavors: Building for ' + self.target_config.get_target() + ' in ' +
        self.target_config.get_flavor(),
        level='debug')
    build = self.target_config.get_build()
    benchmark = self.target_config.get_target()
    flavor = self.target_config.get_flavor()
    f_man = F.FunctorManager()  # Returns the currently loaded FM
    clean_functor = f_man.get_or_load_functor(build, benchmark, flavor, 'clean')
    kwargs = {}

    if self.build_instr:
      L.get_logger().log('Builder::build_flavors: Instrumentation', level='debug')
      try:
        self.check_build_prerequisites()
        L.get_logger().log('Builder::build_flavors: Prerequisite check successfull.')
      except Exception as e:
        raise BuilderException('Precheck failed.\n' + str(e))

      if not self.target_config.is_compile_time_filtering():
        L.get_logger().log('Builder::build_flavors: Runtime filtering enabled.')
        self.target_config.set_instr_file(self.instrumentation_file)

      build_functor = f_man.get_or_load_functor(build, benchmark, flavor, 'build')
      kwargs = self.construct_pira_instr_kwargs()
      ScorepSystemHelper.prepare_MPI_filtering(self.instrumentation_file)

    else:
      L.get_logger().log('Builder::build_flavors: No instrumentation', level='debug')
      build_functor = f_man.get_or_load_functor(build, benchmark, flavor, 'basebuild')
      kwargs = self.construct_pira_kwargs()

    if build_functor.get_method()['active']:
      L.get_logger().log('Builder::build_flavors: Running the passive functor.', level='debug')
      build_functor.active(benchmark, **kwargs)

    else:
      try:
        L.get_logger().log('Builder::build_flavors: Running the passive functor.', level='debug')
        ''' The build command uses CC and CXX to pass flags that are needed by PIRA for the given toolchain. '''
        build_command = build_functor.passive(benchmark, **kwargs)
        clean_command = clean_functor.passive(benchmark, **kwargs)
        L.get_logger().log(
            'Builder::build_flavors: Clean in ' + benchmark + '\n  Using ' + clean_command, level='debug')
        U.shell(clean_command)
        L.get_logger().log('Builder::build_flavors: Building: ' + build_command, level='debug')
        U.shell(build_command)

      except Exception as e:
        L.get_logger().log('Builder::build_flavors: ' + str(e), level='error')
Example #6
0
    def run(self, target_config: TargetConfiguration,
            instrument_config: InstrumentConfig,
            compile_time_filtering: bool) -> float:
        """ Implements the actual invocation """
        functor_manager = fm.FunctorManager()
        run_functor = functor_manager.get_or_load_functor(
            target_config.get_build(), target_config.get_target(),
            target_config.get_flavor(), 'run')
        default_provider = defaults.BackendDefaults()
        kwargs = default_provider.get_default_kwargs()
        kwargs['util'] = util
        kwargs['LD_PRELOAD'] = default_provider.get_MPI_wrap_LD_PRELOAD()
        runtime = .0

        if run_functor.get_method()['active']:
            run_functor.active(target_config.get_target(), **kwargs)
            log.get_logger().log(
                'For the active functor we can barely measure runtime',
                level='warn')
            runtime = 1.0

        try:
            util.change_cwd(target_config.get_place())

            invoke_arguments = target_config.get_args_for_invocation()
            kwargs['args'] = invoke_arguments
            if invoke_arguments is not None:
                log.get_logger().log('LocalBaseRunner::run: (args) ' +
                                     str(invoke_arguments))

            command = run_functor.passive(target_config.get_target(), **kwargs)
            _, runtime = util.shell(command, time_invoc=True)
            log.get_logger().log(
                'LocalBaseRunner::run::passive_invocation -> Returned runtime: '
                + str(runtime),
                level='debug')

        except Exception as e:
            log.get_logger().log('LocalBaseRunner::run Exception\n' + str(e),
                                 level='error')
            raise RuntimeError('LocalBaseRunner::run caught exception. ' +
                               str(e))

        # TODO: Insert the data into the database
        return runtime
Example #7
0
  def test_analyze_local(self):
    ld = C.SimplifiedConfigurationLoader()
    cfg = ld.load_conf('../inputs/configs/basic_config_005.json')

    analyzer = A.Analyzer(cfg)
    fm = F.FunctorManager(cfg)

    a_f = fm.get_or_load_functor('/tmp', 'test_item', 'ct', 'analyze')
    self.assertIsNotNone(a_f)
    self.assertTrue(a_f.get_method()['passive'])
    self.assertEqual(a_f.get_it(), 0)


    tc = TargetConfiguration(cfg.get_place('/tmp'), '/tmp', 'test_item', 'ct', 'asdf')
    with self.assertRaises(RuntimeError) as assert_cm:
      analyzer.analyze(tc, 0)
    rt_err = assert_cm.exception
    self.assertEqual(str(rt_err), 'Analyzer::analyze: Profile Sink in Analyzer not set!')

    analyzer.set_profile_sink(TestProfileSink())
    analyzer.analyze(tc, 0)
    self.assertEqual(a_f.get_it(), 1)
Example #8
0
def main(arguments) -> None:
    """ Main function for pira framework. Used to invoke the various components. """
    show_pira_invoc_info(arguments)

    invoc_cfg = process_args_for_invoc(arguments)
    use_extra_p, extrap_config = process_args_for_extrap(arguments)

    home_dir = U.get_cwd()
    U.set_home_dir(home_dir)

    U.make_dir(invoc_cfg.get_pira_dir())
    BackendDefaults(invoc_cfg)

    csv_config = process_args_for_csv(arguments)

    try:
        if arguments.config_version is 1:
            config_loader = CLoader()
        else:
            config_loader = SCLoader()

        configuration = config_loader.load_conf(invoc_cfg.get_path_to_cfg())
        checker.check_configfile(configuration, arguments.config_version)

        if B.check_queued_job():
            # FIXME: Implement
            L.get_logger().log(
                'In this version of PIRA it is not yet implemented',
                level='error')
            assert (False)

        else:
            '''
      This branch is running PIRA actively on the local machine.
      It is blocking, and the user can track the progress in the terminal.
      '''
            L.get_logger().log('Running the local case')

            # The FunctorManager manages loaded functors and generates the respective names
            F.FunctorManager(configuration)
            dbm = D.DBManager(D.DBManager.db_name + '.' + D.DBManager.db_ext)
            dbm.create_cursor()
            analyzer = A(configuration)

            runner_factory = PiraRunnerFactory(invoc_cfg, configuration)
            runner = runner_factory.get_simple_local_runner()
            if use_extra_p:
                L.get_logger().log('Running with Extra-P runner')
                runner = runner_factory.get_scalability_runner(extrap_config)

            if runner.has_sink():
                analyzer.set_profile_sink(runner.get_sink())

            # A build/place is a top-level directory
            for build in configuration.get_builds():
                L.get_logger().log('Build: ' + str(build))
                app_tuple = (U.generate_random_string(), build, '', '')
                dbm.insert_data_application(app_tuple)

                # An item is a target/software in that directory
                for item in configuration.get_items(build):
                    L.get_logger().log('Running for item ' + str(item))

                    # A flavor is a specific version to build
                    if configuration.has_local_flavors(build, item):
                        for flavor in configuration.get_flavors(build, item):
                            L.get_logger().log('Running for local flavor ' +
                                               flavor,
                                               level='debug')

                            # prepare database, and get a unique handle for current item.
                            db_item_id = dbm.prep_db_for_build_item_in_flavor(
                                configuration, build, item, flavor)
                            # Create configuration object for the item currently processed.
                            place = configuration.get_place(build)
                            t_config = TargetConfiguration(
                                place, build, item, flavor, db_item_id,
                                invoc_cfg.is_compile_time_filtering(),
                                invoc_cfg.get_hybrid_filter_iters())

                            # Execute using a local runner, given the generated target description
                            execute_with_config(runner, analyzer,
                                                invoc_cfg.get_pira_iters(),
                                                t_config, csv_config)

                    # If global flavor
                    else:
                        # TODO: Implement
                        L.get_logger().log(
                            'In this version of PIRA it is not yet implemented',
                            level='error')
                        assert (False)

        U.change_cwd(home_dir)

    except RuntimeError as rt_err:
        U.change_cwd(home_dir)
        L.get_logger().log('Runner.run caught exception. Message: ' +
                           str(rt_err),
                           level='error')
        L.get_logger().dump_tape()
        sys.exit(-1)
Example #9
0
 def test_construction_from_config(self):
     cfg_loader = C.ConfigurationLoader()
     fm = F.FunctorManager(
         cfg_loader.load_conf('./input/unit_input_001.json'))
     fm.reset()
Example #10
0
 def test_get_analyzer_command(self):
     self.fm = F.FunctorManager(
         self.scl.load_conf(self.get_filename(self.cfg004)))
Example #11
0
 def test_get_builder_functor(self):
     self.fm = F.FunctorManager(
         self.scl.load_conf(self.get_filename(self.cfg004)))
Example #12
0
 def test_get_valid_path(self):
     self.fm = F.FunctorManager(
         self.scl.load_conf(self.get_filename(self.cfg004)))
Example #13
0
    def analyze_local(self, flavor: str, build: str, benchmark: str,
                      kwargs: dict, iterationNumber: int) -> str:
        fm = F.FunctorManager()
        analyze_functor = fm.get_or_load_functor(build, benchmark, flavor,
                                                 'analyze')
        analyzer_dir = self.config.get_analyzer_dir(build, benchmark)
        kwargs['analyzer_dir'] = analyzer_dir

        # The invoke args can be retrieved from the configuration object.
        # Since the invoke args are iterable, we can create all necessary argument tuples here.

        # We construct a json file that contains the necesary information to be parsed vy the
        # PGIS tool. That way, we can make it easily traceable and debug from manual inspection.
        # This will be the new standard way of pusing information to PGIS.
        pgis_cfg_file = None
        if self._profile_sink.has_config_output():
            pgis_cfg_file = self._profile_sink.output_config(
                benchmark, analyzer_dir)

        if analyze_functor.get_method()['active']:
            analyze_functor.active(benchmark, **kwargs)

        else:
            L.get_logger().log('Analyzer::analyze_local: Using passive mode')
            try:
                exp_dir = self.config.get_analyzer_exp_dir(build, benchmark)
                isdirectory_good = U.check_provided_directory(analyzer_dir)
                command = analyze_functor.passive(benchmark, **kwargs)

                L.get_logger().log('Analyzer::analyze_local: Command = ' +
                                   command)

                benchmark_name = self.config.get_benchmark_name(benchmark)

                if isdirectory_good:
                    U.change_cwd(analyzer_dir)
                    L.get_logger().log('Analyzer::analyzer_local: Flavor = ' +
                                       flavor + ' | benchmark_name = ' +
                                       benchmark_name)
                    instr_files = U.build_instr_file_path(
                        analyzer_dir, flavor, benchmark_name)
                    L.get_logger().log(
                        'Analyzer::analyzer_local: instrumentation file = ' +
                        instr_files)
                    prev_instr_file = U.build_previous_instr_file_path(
                        analyzer_dir, flavor, benchmark_name)

                tracker = T.TimeTracker()

                # TODO: Alternate between expansion and pure filtering.

                if iterationNumber > 0 and U.is_file(instr_files):
                    L.get_logger().log(
                        'Analyzer::analyze_local: instr_file available')
                    U.rename(instr_files, prev_instr_file)
                    tracker.m_track('Analysis', U, 'run_analyzer_command',
                                    command, analyzer_dir, flavor,
                                    benchmark_name, exp_dir, iterationNumber,
                                    pgis_cfg_file)
                    L.get_logger().log(
                        'Analyzer::analyze_local: command finished',
                        level='debug')
                else:

                    tracker.m_track('Initial analysis', U,
                                    'run_analyzer_command_noInstr', command,
                                    analyzer_dir, flavor, benchmark_name)

                self.tear_down(build, exp_dir)
                return instr_files

            except Exception as e:
                L.get_logger().log(str(e), level='error')
                raise Exception('Problem in Analyzer')