Beispiel #1
0
    def test_report_generation_all_nodes(self):
        name = 'test_report_generation_all_nodes'
        report_path = name + '.report'
        num_node = 1
        num_rank = 1
        delay = 1.0
        app_conf = geopmpy.io.BenchConf(name + '_app.config')
        self._tmp_files.append(app_conf.get_path())
        app_conf.append_region('sleep', delay)
        ctl_conf = geopmpy.io.CtlConf(name + '_ctl.config', self._mode, self._options)
        self._tmp_files.append(ctl_conf.get_path())
        launcher = geopm_test_launcher.TestLauncher(app_conf, ctl_conf, report_path)
        launcher.set_num_node(num_node)
        launcher.set_num_rank(num_rank)
        time.sleep(5)  # Wait a moment to finish cleaning-up from a previous test
        idle_nodes = launcher.get_idle_nodes()
        idle_nodes_copy = list(idle_nodes)
        alloc_nodes = launcher.get_alloc_nodes()
        launcher.write_log(name, 'Idle nodes : {nodes}'.format(nodes=idle_nodes))
        launcher.write_log(name, 'Alloc\'d  nodes : {nodes}'.format(nodes=alloc_nodes))
        node_names = []
        reports = {}
        for nn in idle_nodes_copy:
            launcher.set_node_list(nn.split())  # Hack to convert string to list
            try:
                launcher.run(name)
                node_names += nn.split()
                oo = geopmpy.io.AppOutput(report_path)
                reports[nn] = oo.get_report(nn)
            except subprocess.CalledProcessError as e:
                if e.returncode == 1 and nn not in launcher.get_idle_nodes():
                    launcher.write_log(name, '{node} has disappeared from the idle list!'.format(node=nn))
                    idle_nodes.remove(nn)
                else:
                    launcher.write_log(name, 'Return code = {code}'.format(code=e.returncode))
                    raise e

        self.assertEqual(len(node_names), len(idle_nodes))
        for nn in node_names:
            report = reports[nn]
            self.assertNotEqual(0, len(report))
            self.assertNear(delay, report['sleep'].get_runtime())
            self.assertGreater(report.get_runtime(), report['sleep'].get_runtime())
            self.assertEqual(1, report['sleep'].get_count())
Beispiel #2
0
    def test_sample_rate(self):
        """
        Check that sample rate is regular and fast.
        """
        name = 'test_sample_rate'
        report_path = name + '.report'
        trace_path = name + '.trace'
        num_node = 1
        num_rank = 16
        loop_count = 10
        big_o = 10.0
        region = 'dgemm-progress'
        max_mean = 0.01  # 10 millisecond max sample period
        max_nstd = 0.1  # 10% normalized standard deviation (std / mean)
        app_conf = geopmpy.io.AppConf(name + '_app.config')
        self._tmp_files.append(app_conf.get_path())
        app_conf.set_loop_count(loop_count)
        app_conf.append_region(region, big_o)
        ctl_conf = geopmpy.io.CtlConf(name + '_ctl.config', self._mode,
                                      self._options)
        self._tmp_files.append(ctl_conf.get_path())
        launcher = geopm_test_launcher.TestLauncher(app_conf, ctl_conf,
                                                    report_path, trace_path)
        launcher.set_num_node(num_node)
        launcher.set_num_rank(num_rank)
        launcher.run(name)
        self._output = geopmpy.io.AppOutput(report_path, trace_path + '*')
        node_names = self._output.get_node_names()
        self.assertEqual(num_node, len(node_names))

        for nn in node_names:
            rr = self._output.get_report(nn)
            tt = self._output.get_trace(nn)
            delta_t = tt['seconds'].diff()
            delta_t = delta_t.loc[delta_t != 0]
            self.assertGreater(max_mean, delta_t.mean())
            # WARNING : The following line may mask issues in the sampling rate. To do a fine grained analysis, comment
            # out the next line and do NOT run on the BSP. This will require modifications to the launcher or manual testing.
            size_orig = len(delta_t)
            delta_t = delta_t[
                (delta_t - delta_t.mean()) < 3 *
                delta_t.std()]  # Only keep samples within 3 stds of the mean
            self.assertGreater(0.06, 1 - (float(len(delta_t)) / size_orig))
            self.assertGreater(max_nstd, delta_t.std() / delta_t.mean())
Beispiel #3
0
    def setUpClass(cls):
        """Create launcher, execute benchmark and set up class variables.

        """
        sys.stdout.write('(' + os.path.basename(__file__).split('.')[0] +
                         '.' + cls.__name__ + ') ...')
        test_name = 'test_timed_scaling_region'
        cls._report_path = test_name + '.report'
        cls._trace_path = test_name + '.trace'
        cls._skip_launch = not util.do_launch()
        cls._agent_conf_path = test_name + '-agent-config.json'
        # region_hash() of the sequence:
        # timed_scaling_region_0, timed_scaling_region_1, ... , timed_scaling_region_30
        cls._region_hash = [geopmpy.hash.crc32_str('timed_scaling_region_{}'.format(ii))
                            for ii in range(31)]
        geopmpy.error.exc_clear()
        if not cls._skip_launch:
            num_node = 1
            num_rank = 1
            # Set up agent configuration so that each region is assigned a different frequency
            freq_min = geopm_test_launcher.geopmread("CPUINFO::FREQ_MIN board 0")
            freq_sticker = geopm_test_launcher.geopmread("CPUINFO::FREQ_STICKER board 0")
            freq_step = geopm_test_launcher.geopmread("CPUINFO::FREQ_STEP board 0")
            num_step = int((freq_sticker - freq_min) / freq_step + 0.5) + 1
            agent_conf_dict = {'FREQ_DEFAULT': freq_sticker}
            cls._region_freq = [freq_min + idx * freq_step
                                for idx in range(num_step)]
            freq_idx = 0
            for freq_idx in range(len(cls._region_freq)):
                agent_conf_dict['HASH_{}'.format(freq_idx)] = cls._region_hash[freq_idx]
                agent_conf_dict['FREQ_{}'.format(freq_idx)] = cls._region_freq[freq_idx]

            agent_conf = geopmpy.io.AgentConf(cls._agent_conf_path,
                                              'frequency_map',
                                              agent_conf_dict)
            launcher = geopm_test_launcher.TestLauncher(AppConf(),
                                                        agent_conf,
                                                        cls._report_path,
                                                        cls._trace_path,
                                                        time_limit=6000)
            launcher.set_num_node(num_node)
            launcher.set_num_rank(num_rank)
            launcher.run(test_name)
Beispiel #4
0
    def test_unmarked_ompt(self):
        name = 'test_unmarked_ompt'
        report_path = name + '.report'
        num_node = 4
        num_rank = 16
        app_conf = geopmpy.io.AppConf(name + '_app.config')
        self._tmp_files.append(app_conf.get_path())
        app_conf.append_region('stream-unmarked', 1.0)
        app_conf.append_region('dgemm-unmarked', 1.0)
        app_conf.append_region('all2all-unmarked', 1.0)
        ctl_conf = geopmpy.io.CtlConf(name + '_ctl.config', self._mode,
                                      self._options)
        self._tmp_files.append(ctl_conf.get_path())
        launcher = geopm_test_launcher.TestLauncher(app_conf, ctl_conf,
                                                    report_path)
        launcher.set_num_node(num_node)
        launcher.set_num_rank(num_rank)
        launcher.run(name)

        self._output = geopmpy.io.AppOutput(report_path)
        node_names = self._output.get_node_names()
        self.assertEqual(len(node_names), num_node)
        stream_id = None
        stream_name = '[OMPT]geopm_test_integration:geopm::StreamModelRegion::run()'
        for nn in node_names:
            rr = self._output.get_report(nn)
            region_names = rr.keys()
            self.assertTrue(stream_name in rr)
            stream_region = rr[stream_name]
            self.assertEqual(1, stream_region.get_count())
            if stream_id:
                self.assertEqual(stream_id, stream_region.get_id())
            else:
                stream_id = stream_region.get_id()
            ompt_regions = [
                key for key in region_names if key.startswith('[OMPT]')
            ]
            self.assertLessEqual(2, len(ompt_regions))
            self.assertTrue(('MPI_Alltoall' in rr))
            gemm_region = [
                key for key in region_names if key.lower().find('gemm') != -1
            ]
            self.assertLessEqual(1, len(gemm_region))
Beispiel #5
0
    def test_progress_exit(self):
        """
        Check that when we always see progress exit before the next entry.
        Make sure that progress only decreases when a new region is entered.
        """
        name = 'test_progress_exit'
        report_path = name + '.report'
        trace_path = name + '.trace'
        num_node = 1
        num_rank = 16
        loop_count = 100
        big_o = 0.01
        app_conf = geopmpy.io.AppConf(name + '_app.config')
        self._tmp_files.append(app_conf.get_path())
        app_conf.set_loop_count(loop_count)
        app_conf.append_region('dgemm-progress', big_o)
        app_conf.append_region('spin-progress', 0.01)
        ctl_conf = geopmpy.io.CtlConf(name + '_ctl.config', self._mode,
                                      self._options)
        self._tmp_files.append(ctl_conf.get_path())
        launcher = geopm_test_launcher.TestLauncher(app_conf, ctl_conf,
                                                    report_path, trace_path)
        launcher.set_num_node(num_node)
        launcher.set_num_rank(num_rank)
        launcher.run(name)

        self._output = geopmpy.io.AppOutput(report_path, trace_path + '*')
        node_names = self._output.get_node_names()
        self.assertEqual(num_node, len(node_names))

        for nn in node_names:
            rr = self._output.get_report(nn)
            tt = self._output.get_trace(nn)

            tt = tt.set_index(['region_id'], append=True)
            tt = tt.groupby(level=['region_id'])

            for region_id, data in tt:
                tmp = data['progress-0'].diff()
                negative_progress = tmp.loc[(tmp > -1) & (tmp < 0)]
                launcher.write_log(name, '{}'.format(negative_progress))
                self.assertEqual(0, len(negative_progress))
Beispiel #6
0
    def test_trace_runtimes(self):
        name = 'test_trace_runtimes'
        report_path = name + '.report'
        trace_path = name + '.trace'
        num_node = 4
        num_rank = 16
        app_conf = geopmpy.io.AppConf(name + '_app.config')
        self._tmp_files.append(app_conf.get_path())
        app_conf.append_region('sleep', 1.0)
        app_conf.append_region('dgemm', 1.0)
        app_conf.append_region('all2all', 1.0)
        ctl_conf = geopmpy.io.CtlConf(name + '_ctl.config', self._mode,
                                      self._options)
        self._tmp_files.append(ctl_conf.get_path())
        launcher = geopm_test_launcher.TestLauncher(app_conf, ctl_conf,
                                                    report_path, trace_path)
        launcher.set_num_node(num_node)
        launcher.set_num_rank(num_rank)
        launcher.run(name)

        self._output = geopmpy.io.AppOutput(report_path, trace_path + '*')
        node_names = self._output.get_node_names()
        self.assertEqual(len(node_names), num_node)

        for nn in node_names:
            report = self._output.get_report(nn)
            trace = self._output.get_trace(nn)
            self.assertNear(trace.iloc[-1]['seconds'], report.get_runtime())

            # Calculate runtime totals for each region in each trace, compare to report
            tt = trace.set_index(['region_id'], append=True)
            tt = tt.groupby(level=['region_id'])
            for region_name, region_data in report.iteritems():
                if region_name != 'unmarked-region' and region_data.get_runtime(
                ) != 0:
                    trace_data = tt.get_group((region_data.get_id()))
                    trace_elapsed_time = trace_data.iloc[-1][
                        'seconds'] - trace_data.iloc[0]['seconds']
                    self.assertNear(trace_elapsed_time,
                                    region_data.get_runtime())
Beispiel #7
0
 def test_runtime(self):
     name = 'test_runtime'
     report_path = name + '.report'
     num_node = 1
     num_rank = 5
     delay = 3.0
     app_conf = geopmpy.io.BenchConf(name + '_app.config')
     self._tmp_files.append(app_conf.get_path())
     app_conf.append_region('sleep', delay)
     ctl_conf = geopmpy.io.CtlConf(name + '_ctl.config', self._mode, self._options)
     self._tmp_files.append(ctl_conf.get_path())
     launcher = geopm_test_launcher.TestLauncher(app_conf, ctl_conf, report_path)
     launcher.set_num_node(num_node)
     launcher.set_num_rank(num_rank)
     launcher.run(name)
     self._output = geopmpy.io.AppOutput(report_path)
     node_names = self._output.get_node_names()
     self.assertEqual(num_node, len(node_names))
     for nn in node_names:
         rr = self._output.get_report(nn)
         self.assertNear(delay, rr['sleep'].get_runtime())
         self.assertGreater(rr.get_runtime(), rr['sleep'].get_runtime())
Beispiel #8
0
    def test_ignore_runtime(self):
        name = 'test_ignore_runtimes'
        report_path = name + '.report'
        trace_path = name + '.trace'
        num_node = 4
        num_rank = 16
        app_conf = geopmpy.io.BenchConf(name + '_app.config')
        self._tmp_files.append(app_conf.get_path())
        app_conf.append_region('ignore', 1.0)
        app_conf.append_region('dgemm', 1.0)
        app_conf.append_region('all2all', 1.0)
        ctl_conf = geopmpy.io.CtlConf(name + '_ctl.config', self._mode, self._options)
        self._tmp_files.append(ctl_conf.get_path())
        launcher = geopm_test_launcher.TestLauncher(app_conf, ctl_conf, report_path, trace_path)
        launcher.set_num_node(num_node)
        launcher.set_num_rank(num_rank)
        launcher.run(name)

        self._output = geopmpy.io.AppOutput(report_path, trace_path + '*')
        node_names = self._output.get_node_names()
        self.assertEqual(len(node_names), num_node)
        for nn in node_names:
            rr = self._output.get_report(nn)
            self.assertEqual(rr['ignore'].get_runtime(), rr.get_ignore_runtime())
Beispiel #9
0
    def test_mpi_runtimes(self):
        name = 'test_mpi_runtimes'
        report_path = name + '.report'
        trace_path = name + '.trace'
        num_node = 4
        num_rank = 16
        app_conf = geopmpy.io.BenchConf(name + '_app.config')
        self._tmp_files.append(app_conf.get_path())
        app_conf.append_region('sleep', 1.0)
        app_conf.append_region('dgemm', 1.0)
        app_conf.append_region('all2all', 1.0)
        ctl_conf = geopmpy.io.CtlConf(name + '_ctl.config', self._mode, self._options)
        self._tmp_files.append(ctl_conf.get_path())
        launcher = geopm_test_launcher.TestLauncher(app_conf, ctl_conf, report_path, trace_path)
        launcher.set_num_node(num_node)
        launcher.set_num_rank(num_rank)
        launcher.run(name)

        self._output = geopmpy.io.AppOutput(report_path, trace_path + '*')
        node_names = self._output.get_node_names()
        self.assertEqual(len(node_names), num_node)
        for nn in node_names:
            rr = self._output.get_report(nn)
            self.assertEqual(0, rr['unmarked-region'].get_count())
            # Since MPI time is is counted if any rank on a node is in
            # an MPI call, but region time is counted only when all
            # ranks on a node are in a region, we must use the
            # unmarked-region time as our error term when comparing
            # MPI time and all2all time.
            mpi_epsilon = max(rr['unmarked-region'].get_runtime() / rr['all2all'].get_mpi_runtime(), 0.05)
            self.assertNear(rr['all2all'].get_mpi_runtime(), rr['all2all'].get_runtime(), mpi_epsilon)
            self.assertEqual(rr['all2all'].get_mpi_runtime(), rr['epoch'].get_mpi_runtime())
            self.assertEqual(rr['all2all'].get_mpi_runtime(), rr.get_mpi_runtime())
            self.assertEqual(0, rr['unmarked-region'].get_mpi_runtime())
            self.assertEqual(0, rr['sleep'].get_mpi_runtime())
            self.assertEqual(0, rr['dgemm'].get_mpi_runtime())
Beispiel #10
0
    def test_report_and_trace_generation(self):
        name = 'test_report_and_trace_generation'
        report_path = name + '.report'
        trace_path = name + '.trace'
        num_node = 4
        num_rank = 16
        app_conf = geopmpy.io.BenchConf(name + '_app.config')
        self._tmp_files.append(app_conf.get_path())
        app_conf.append_region('sleep', 1.0)
        ctl_conf = geopmpy.io.CtlConf(name + '_ctl.config', self._mode, self._options)
        self._tmp_files.append(ctl_conf.get_path())
        launcher = geopm_test_launcher.TestLauncher(app_conf, ctl_conf, report_path, trace_path)
        launcher.set_num_node(num_node)
        launcher.set_num_rank(num_rank)
        launcher.run(name)

        self._output = geopmpy.io.AppOutput(report_path, trace_path + '*')
        node_names = self._output.get_node_names()
        self.assertEqual(num_node, len(node_names))
        for nn in node_names:
            report = self._output.get_report(nn)
            self.assertNotEqual(0, len(report))
            trace = self._output.get_trace(nn)
            self.assertNotEqual(0, len(trace))
Beispiel #11
0
    def setUpClass(cls):
        """Create launcher, execute benchmark and set up class variables.

        """
        cls._test_name = 'test_power_balancer'
        cls._num_node = 4
        cls._agent_list = ['power_governor', 'power_balancer']
        cls._skip_launch = False
        cls._show_details = True
        cls._tmp_files = []

        # Clear out exception record for python 2 support
        geopmpy.error.exc_clear()

        if not cls._skip_launch:
            num_rank = 16
            loop_count = 500
            fam, mod = geopm_test_launcher.get_platform()
            power_budget = 200
            if fam == 6 and mod == 87:
                # budget for KNL
                power_budget = 130
            options = {'power_budget': power_budget}
            gov_agent_conf_path = cls._test_name + '_gov_agent.config'
            bal_agent_conf_path = cls._test_name + '_bal_agent.config'
            cls._tmp_files.append(gov_agent_conf_path)
            cls._tmp_files.append(bal_agent_conf_path)
            path_dict = {
                'power_governor': gov_agent_conf_path,
                'power_balancer': bal_agent_conf_path
            }

            for app_name in ['geopmbench', 'socket_imbalance']:
                app_conf = None
                if app_name == 'geopmbench':
                    app_conf = geopmpy.io.BenchConf(cls._test_name +
                                                    '_app.config')
                    cls._tmp_files.append(app_conf.get_path())
                    app_conf.append_region('dgemm-imbalance', 8.0)
                    app_conf.append_region('all2all', 0.05)
                    app_conf.set_loop_count(loop_count)
                    # Update app config with imbalance
                    alloc_nodes = geopm_test_launcher.TestLauncher.get_alloc_nodes(
                    )
                    for nn in range(len(alloc_nodes) // 2):
                        app_conf.append_imbalance(alloc_nodes[nn], 0.5)
                elif app_name == 'socket_imbalance':
                    app_conf = cls.AppConf()
                else:
                    raise RuntimeError(
                        'No application config for app name {}'.format(
                            app_name))
                for agent in cls._agent_list:
                    agent_conf = geopmpy.io.AgentConf(path_dict[agent], agent,
                                                      options)
                    run_name = '{}_{}_{}'.format(cls._test_name, agent,
                                                 app_name)
                    report_path = '{}.report'.format(run_name)
                    trace_path = '{}.trace'.format(run_name)
                    cls._tmp_files.append(report_path)
                    cls._tmp_files.append(trace_path)
                    launcher = geopm_test_launcher.TestLauncher(
                        app_conf,
                        agent_conf,
                        report_path,
                        trace_path,
                        time_limit=2700)
                    launcher.set_num_node(cls._num_node)
                    launcher.set_num_rank(num_rank)
                    launcher.write_log(run_name,
                                       'Power cap = {}W'.format(power_budget))
                    launcher.run(run_name)
                    time.sleep(60)
Beispiel #12
0
    def test_power_consumption(self):
        name = 'test_power_consumption'
        report_path = name + '.report'
        trace_path = name + '.trace'
        num_node = 4
        num_rank = 16
        loop_count = 500
        app_conf = geopmpy.io.AppConf(name + '_app.config')
        self._tmp_files.append(app_conf.get_path())
        app_conf.append_region('dgemm', 8.0)
        app_conf.set_loop_count(loop_count)
        self._options['power_budget'] = 150
        ctl_conf = geopmpy.io.CtlConf(name + '_ctl.config', self._mode,
                                      self._options)
        self._tmp_files.append(ctl_conf.get_path())
        launcher = geopm_test_launcher.TestLauncher(app_conf,
                                                    ctl_conf,
                                                    report_path,
                                                    trace_path,
                                                    time_limit=900)
        launcher.set_num_node(num_node)
        launcher.set_num_rank(num_rank)
        launcher.write_log(
            name, 'Power cap = {}W'.format(self._options['power_budget']))
        launcher.run(name)

        self._output = geopmpy.io.AppOutput(report_path, trace_path + '*')
        node_names = self._output.get_node_names()
        self.assertEqual(num_node, len(node_names))
        all_power_data = {}
        # Total power consumed will be Socket(s) + DRAM
        for nn in node_names:
            rr = self._output.get_report(nn)
            tt = self._output.get_trace(nn)

            first_epoch_index = tt.loc[tt['region_id'] ==
                                       '9223372036854775808'][:1].index[0]
            epoch_dropped_data = tt[
                first_epoch_index:]  # Drop all startup data

            power_data = epoch_dropped_data.filter(regex='energy')
            power_data['seconds'] = epoch_dropped_data['seconds']
            power_data = power_data.diff().dropna()
            power_data.rename(columns={'seconds': 'elapsed_time'},
                              inplace=True)
            power_data = power_data.loc[(power_data != 0).all(
                axis=1)]  # Will drop any row that is all 0's

            pkg_energy_cols = [
                s for s in power_data.keys() if 'pkg_energy' in s
            ]
            dram_energy_cols = [
                s for s in power_data.keys() if 'dram_energy' in s
            ]
            power_data['socket_power'] = power_data[pkg_energy_cols].sum(
                axis=1) / power_data['elapsed_time']
            power_data['dram_power'] = power_data[dram_energy_cols].sum(
                axis=1) / power_data['elapsed_time']
            power_data['combined_power'] = power_data[
                'socket_power'] + power_data['dram_power']

            pandas.set_option('display.width', 100)
            launcher.write_log(
                name,
                'Power stats from {} :\n{}'.format(nn, power_data.describe()))

            all_power_data[nn] = power_data

        for node_name, power_data in all_power_data.iteritems():
            # Allow for overages of 2% at the 75th percentile.
            self.assertGreater(self._options['power_budget'] * 1.02,
                               power_data['combined_power'].quantile(.75))
Beispiel #13
0
    def test_region_runtimes(self):
        name = 'test_region_runtime'
        report_path = name + '.report'
        trace_path = name + '.trace'
        num_node = 4
        num_rank = 16
        loop_count = 500
        app_conf = geopmpy.io.AppConf(name + '_app.config')
        self._tmp_files.append(app_conf.get_path())
        app_conf.append_region('dgemm', 8.0)
        app_conf.set_loop_count(loop_count)
        ctl_conf = geopmpy.io.CtlConf(name + '_ctl.config', self._mode,
                                      self._options)
        self._tmp_files.append(ctl_conf.get_path())
        launcher = geopm_test_launcher.TestLauncher(app_conf,
                                                    ctl_conf,
                                                    report_path,
                                                    trace_path,
                                                    time_limit=900)
        launcher.set_num_node(num_node)
        launcher.set_num_rank(num_rank)
        launcher.run(name)

        self._output = geopmpy.io.AppOutput(report_path, trace_path + '*')
        node_names = self._output.get_node_names()
        self.assertEqual(len(node_names), num_node)

        # Calculate region times from traces
        region_times = collections.defaultdict(
            lambda: collections.defaultdict(dict))
        for nn in node_names:
            tt = self._output.get_trace(nn).set_index(
                ['region_id'], append=True).groupby(level=['region_id'])

            for region_id, data in tt:
                # Build a df with only the first region entry and the exit.
                last_index = 0
                filtered_df = pandas.DataFrame()
                row_list = []
                progress_1s = data['progress-0'].loc[data['progress-0'] == 1]
                for index, junk in progress_1s.iteritems():
                    row = data.ix[last_index:index].head(1)
                    row_list += [row[['seconds', 'progress-0']]]
                    row = data.ix[last_index:index].tail(1)
                    row_list += [row[['seconds', 'progress-0']]]
                    last_index = index[
                        0] + 1  # Set the next starting index to be one past where we are
                filtered_df = pandas.concat(row_list)

                filtered_df = filtered_df.diff()
                # Since I'm not separating out the progress 0's from 1's, when I do the diff I only care about the
                # case where 1 - 0 = 1 for the progress column.
                filtered_df = filtered_df.loc[filtered_df['progress-0'] == 1]

                if len(filtered_df) > 1:
                    launcher.write_log(name, 'Region elapsed time stats from {} - {} :\n{}'\
                    .format(nn, region_id, filtered_df['seconds'].describe()))
                    filtered_df['seconds'].describe()
                    region_times[nn][region_id] = filtered_df

            launcher.write_log(name, '{}'.format('-' * 80))

        # Loop through the reports to see if the region runtimes line up with what was calculated from the trace files above.
        write_regions = True
        for nn in node_names:
            rr = self._output.get_report(nn)
            for region_name, region in rr.iteritems():
                if region.get_id() != 0 and region.get_count() > 1:
                    if write_regions:
                        launcher.write_log(
                            name,
                            'Region {} is {}.'.format(region.get_id(),
                                                      region_name))
                    self.assertNear(
                        region.get_runtime(),
                        region_times[nn][region.get_id()]['seconds'].sum())
            write_regions = False

        # Test to ensure every region detected in the trace is captured in the report.
        for nn in node_names:
            rr = self._output.get_report(nn)
            report_ids = [rr[ii].get_id() for ii in rr]
            for region_id in region_times[nn].keys():
                self.assertTrue(
                    region_id in report_ids,
                    msg='Report from {} missing region_id {}'.format(
                        nn, region_id))
    def setUpClass(cls):
        """Create launcher, execute benchmark and set up class variables.

        """
        sys.stdout.write('(' + os.path.basename(__file__).split('.')[0] + '.' +
                         cls.__name__ + ') ...')
        test_name = 'frequency_hint_usage'
        cls._skip_launch = not util.do_launch()

        cls._fmap_report_path = 'test_{}_fmap.report'.format(test_name)
        cls._fmap_trace_path = 'test_{}_fmap.trace'.format(test_name)
        cls._fmap_agent_conf_path = 'test_' + test_name + '-fmap-agent-config.json'

        cls._ee_report_path = 'test_{}_ee.report'.format(test_name)
        cls._ee_trace_path = 'test_{}_ee.trace'.format(test_name)
        cls._ee_agent_conf_path = 'test_' + test_name + '-ee-agent-config.json'

        geopmpy.error.exc_clear(
        )  # Clear out exception record for python 2 support

        cls._freq_min = geopm_test_launcher.geopmread(
            "CPUINFO::FREQ_MIN board 0")
        cls._freq_sticker = geopm_test_launcher.geopmread(
            "CPUINFO::FREQ_STICKER board 0")
        cls._freq_step = geopm_test_launcher.geopmread(
            "FREQUENCY_STEP board 0")
        cls._freq_default = cls._freq_sticker - cls._freq_step

        if not cls._skip_launch:
            # Set the job size parameters
            num_node = 1
            num_rank = 1
            time_limit = 6000
            # Configure the test application
            app_conf = AppConf()

            # Configure the agents
            agent_conf_dict = {'FREQ_DEFAULT': cls._freq_default}
            fmap_agent_conf = geopmpy.io.AgentConf(cls._fmap_agent_conf_path,
                                                   'frequency_map',
                                                   agent_conf_dict)
            agent_conf_dict = {
                'FREQ_MIN': cls._freq_min,
                'FREQ_MAX': cls._freq_default
            }
            ee_agent_conf = geopmpy.io.AgentConf(cls._ee_agent_conf_path,
                                                 'energy_efficient',
                                                 agent_conf_dict)

            # Fmap run
            launcher = geopm_test_launcher.TestLauncher(app_conf,
                                                        fmap_agent_conf,
                                                        cls._fmap_report_path,
                                                        cls._fmap_trace_path,
                                                        time_limit=time_limit)
            launcher.set_num_node(num_node)
            launcher.set_num_rank(num_rank)
            launcher.run('test_' + test_name)

            # EE run
            launcher = geopm_test_launcher.TestLauncher(app_conf,
                                                        ee_agent_conf,
                                                        cls._ee_report_path,
                                                        cls._ee_trace_path,
                                                        time_limit=time_limit)
            launcher.set_num_node(num_node)
            launcher.set_num_rank(num_rank)
            launcher.run(test_name)