Пример #1
0
    def setUpClass(cls):
        # set up parser and kernels
        cls.parser_x86 = ParserX86ATT()
        cls.parser_AArch64 = ParserAArch64()
        with open(cls._find_file('kernel_x86.s')) as f:
            cls.code_x86 = f.read()
        with open(cls._find_file('kernel_aarch64.s')) as f:
            cls.code_AArch64 = f.read()
        cls.kernel_x86 = reduce_to_section(
            cls.parser_x86.parse_file(cls.code_x86), 'x86')
        cls.kernel_AArch64 = reduce_to_section(
            cls.parser_AArch64.parse_file(cls.code_AArch64), 'aarch64')

        # set up machine models
        cls.machine_model_csx = MachineModel(
            path_to_yaml=os.path.join(cls.MODULE_DATA_DIR, 'csx.yml'))
        cls.machine_model_tx2 = MachineModel(
            path_to_yaml=os.path.join(cls.MODULE_DATA_DIR, 'tx2.yml'))
        cls.semantics_csx = ArchSemantics(cls.machine_model_csx,
                                          path_to_yaml=os.path.join(
                                              cls.MODULE_DATA_DIR,
                                              'isa/x86.yml'))
        cls.semantics_tx2 = ArchSemantics(
            cls.machine_model_tx2,
            path_to_yaml=os.path.join(cls.MODULE_DATA_DIR, 'isa/aarch64.yml'),
        )
        cls.machine_model_zen = MachineModel(arch='zen1')

        for i in range(len(cls.kernel_x86)):
            cls.semantics_csx.assign_src_dst(cls.kernel_x86[i])
            cls.semantics_csx.assign_tp_lt(cls.kernel_x86[i])
        for i in range(len(cls.kernel_AArch64)):
            cls.semantics_tx2.assign_src_dst(cls.kernel_AArch64[i])
            cls.semantics_tx2.assign_tp_lt(cls.kernel_AArch64[i])
Пример #2
0
    def setUpClass(self):
        # set up parser and kernels
        self.parser_x86 = ParserX86ATT()
        self.parser_AArch64 = ParserAArch64()
        with open(self._find_file('kernel_x86.s')) as f:
            code_x86 = f.read()
        with open(self._find_file('kernel_aarch64.s')) as f:
            code_AArch64 = f.read()
        self.kernel_x86 = self.parser_x86.parse_file(code_x86)
        self.kernel_AArch64 = self.parser_AArch64.parse_file(code_AArch64)

        # set up machine models
        self.machine_model_csx = MachineModel(
            path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'csx.yml'))
        self.machine_model_tx2 = MachineModel(arch='tx2')
        self.semantics_csx = ArchSemantics(self.machine_model_csx,
                                           path_to_yaml=os.path.join(
                                               self.MODULE_DATA_DIR,
                                               'isa/x86.yml'))
        self.semantics_tx2 = ArchSemantics(
            self.machine_model_tx2,
            path_to_yaml=os.path.join(self.MODULE_DATA_DIR, 'isa/aarch64.yml'),
        )
        for i in range(len(self.kernel_x86)):
            self.semantics_csx.assign_src_dst(self.kernel_x86[i])
            self.semantics_csx.assign_tp_lt(self.kernel_x86[i])
        for i in range(len(self.kernel_AArch64)):
            self.semantics_tx2.assign_src_dst(self.kernel_AArch64[i])
            self.semantics_tx2.assign_tp_lt(self.kernel_AArch64[i])
Пример #3
0
def sanity_check(arch: str,
                 verbose=False,
                 internet_check=False,
                 output_file=sys.stdout):
    """
    Checks the database for missing TP/LT values, instructions might missing int the ISA DB and
    duplicate instructions.

    :param arch: micro-arch key to define DB to check
    :type arch: str
    :param verbose: verbose output flag, defaults to `False`
    :type verbose: bool, optional
    :param internet_check: indicates if OSACA should try to look up the src/dst distribution in the
                           internet, defaults to False
    :type internet_check: boolean, optional
    :param output_file: output stream specifying where to write output,
                        defaults to :class:`sys.stdout`
    :type output_file: stream, optional

    :return: True if everything checked out
    """
    # load arch machine model
    arch_mm = MachineModel(arch=arch)
    data = arch_mm["instruction_forms"]
    # load isa machine model
    isa = arch_mm.get_ISA()
    isa_mm = MachineModel(arch="isa/{}".format(isa))
    num_of_instr = len(data)

    # check arch DB entries
    (
        missing_throughput,
        missing_latency,
        missing_port_pressure,
        suspicious_instructions,
        duplicate_instr_arch,
        bad_operand,
    ) = _check_sanity_arch_db(arch_mm, isa_mm, internet_check=internet_check)
    # check ISA DB entries
    duplicate_instr_isa, only_in_isa = _check_sanity_isa_db(arch_mm, isa_mm)

    report = _get_sanity_report(
        num_of_instr,
        missing_throughput,
        missing_latency,
        missing_port_pressure,
        suspicious_instructions,
        duplicate_instr_arch,
        duplicate_instr_isa,
        only_in_isa,
        bad_operand,
        verbose=verbose,
        colors=True if output_file == sys.stdout else False,
    )
    print(report, file=output_file)

    return not any([missing_port_pressure, bad_operand])
Пример #4
0
 def test_invalid_MachineModel(self):
     with self.assertRaises(ValueError):
         MachineModel()
     with self.assertRaises(ValueError):
         MachineModel(arch='CSX',
                      path_to_yaml=os.path.join(self.MODULE_DATA_DIR,
                                                'csx.yml'))
     with self.assertRaises(FileNotFoundError):
         MachineModel(arch='THE_MACHINE')
     with self.assertRaises(FileNotFoundError):
         MachineModel(path_to_yaml=os.path.join(self.MODULE_DATA_DIR,
                                                'THE_MACHINE.yml'))
Пример #5
0
def import_benchmark_output(arch, bench_type, filepath, output=sys.stdout):
    """
    Import benchmark results from micro-benchmarks.

    :param arch: target architecture key
    :type arch: str
    :param bench_type: key for defining type of benchmark output
    :type bench_type: str
    :param filepath: filepath to the output file
    :type filepath: str
    :param output: output stream to dump, defaults to sys.stdout
    :type output: stream
    """
    supported_bench_outputs = ["ibench", "asmbench"]
    assert os.path.exists(filepath)
    if bench_type not in supported_bench_outputs:
        raise ValueError("Benchmark type is not supported.")
    with open(filepath, "r") as f:
        input_data = f.readlines()
    db_entries = None
    mm = MachineModel(arch)
    if bench_type == "ibench":
        db_entries = _get_ibench_output(input_data, mm.get_ISA())
    elif bench_type == "asmbench":
        db_entries = _get_asmbench_output(input_data, mm.get_ISA())
    # write entries to DB
    for entry in db_entries:
        mm.set_instruction_entry(db_entries[entry])
    if output is None:
        print(mm.dump())
    else:
        mm.dump(stream=output)
Пример #6
0
    def test_add_single_entry(self):
        mm_csx = MachineModel("csx")
        mm_tx2 = MachineModel("tx2")
        mm_zen1 = MachineModel("zen1")
        num_entries_csx = len(mm_csx["instruction_forms"])
        num_entries_tx2 = len(mm_tx2["instruction_forms"])
        num_entries_zen1 = len(mm_zen1["instruction_forms"])

        mm_csx.set_instruction_entry(self.entry_csx)
        mm_tx2.set_instruction_entry(self.entry_tx2)
        mm_zen1.set_instruction_entry({"name": "empty_operation"})

        num_entries_csx = len(mm_csx["instruction_forms"]) - num_entries_csx
        num_entries_tx2 = len(mm_tx2["instruction_forms"]) - num_entries_tx2
        num_entries_zen1 = len(mm_zen1["instruction_forms"]) - num_entries_zen1

        self.assertEqual(num_entries_csx, 1)
        self.assertEqual(num_entries_tx2, 1)
        self.assertEqual(num_entries_zen1, 1)
Пример #7
0
    def test_add_single_entry(self):
        mm_csx = MachineModel('csx')
        mm_tx2 = MachineModel('tx2')
        mm_zen1 = MachineModel('zen1')
        num_entries_csx = len(mm_csx['instruction_forms'])
        num_entries_tx2 = len(mm_tx2['instruction_forms'])
        num_entries_zen1 = len(mm_zen1['instruction_forms'])

        mm_csx.set_instruction_entry(self.entry_csx)
        mm_tx2.set_instruction_entry(self.entry_tx2)
        mm_zen1.set_instruction_entry({'name': 'empty_operation'})

        num_entries_csx = len(mm_csx['instruction_forms']) - num_entries_csx
        num_entries_tx2 = len(mm_tx2['instruction_forms']) - num_entries_tx2
        num_entries_zen1 = len(mm_zen1['instruction_forms']) - num_entries_zen1

        self.assertEqual(num_entries_csx, 1)
        self.assertEqual(num_entries_tx2, 1)
        self.assertEqual(num_entries_zen1, 1)
Пример #8
0
    def __init__(self, arch, code):
        self.machine_model = MachineModel(arch=arch)
        self.semantics = ArchSemantics(self.machine_model)
        isa = self.machine_model.get_ISA().lower()
        if isa == 'aarch64':
            self.parser = ParserAArch64()
        elif isa == 'x86':
            self.parser = ParserX86ATT()

        parsed_code = self.parser.parse_file(code)
        self.kernel = reduce_to_section(parsed_code,
                                        self.machine_model.get_ISA())
        self.semantics.add_semantics(self.kernel)
Пример #9
0
    def __init__(self, filename='', arch=None, path_to_yaml=None):
        """
        Constructor method.

        :param filename: path to the analyzed kernel file for documentation, defaults to ''
        :type filename: str, optional
        :param arch: micro-arch code for getting the machine model, defaults to None
        :type arch: str, optional
        :param path_to_yaml: path to the YAML file for getting the machine model, defaults to None
        :type path_to_yaml: str, optional
        """
        self._filename = filename
        if not arch and not path_to_yaml:
            raise ValueError('Either arch or path_to_yaml required.')
        if arch and path_to_yaml:
            raise ValueError('Only one of arch and path_to_yaml is allowed.')
        self._arch = arch
        if arch:
            self._arch = arch.lower()
            self._machine_model = MachineModel(arch=arch, lazy=True)
        elif path_to_yaml:
            self._machine_model = MachineModel(path_to_yaml=path_to_yaml, lazy=True)
            self._arch = self._machine_model.get_arch()
Пример #10
0
    def test_hidden_load(self):
        machine_model_hld = MachineModel(
            path_to_yaml=self._find_file('hidden_load_machine_model.yml'))
        self.assertTrue(machine_model_hld.has_hidden_loads())
        semantics_hld = ArchSemantics(machine_model_hld)
        kernel_hld = self.parser_x86.parse_file(self.code_x86)
        kernel_hld_2 = self.parser_x86.parse_file(self.code_x86)
        kernel_hld_2 = self.parser_x86.parse_file(self.code_x86)[-3:]
        kernel_hld_3 = self.parser_x86.parse_file(self.code_x86)[5:8]
        semantics_hld.add_semantics(kernel_hld)
        semantics_hld.add_semantics(kernel_hld_2)
        semantics_hld.add_semantics(kernel_hld_3)

        num_hidden_loads = len(
            [x for x in kernel_hld if INSTR_FLAGS.HIDDEN_LD in x['flags']])
        num_hidden_loads_2 = len(
            [x for x in kernel_hld_2 if INSTR_FLAGS.HIDDEN_LD in x['flags']])
        num_hidden_loads_3 = len(
            [x for x in kernel_hld_3 if INSTR_FLAGS.HIDDEN_LD in x['flags']])
        self.assertEqual(num_hidden_loads, 1)
        self.assertEqual(num_hidden_loads_2, 0)
        self.assertEqual(num_hidden_loads_3, 1)
Пример #11
0
    def test_machine_model_various_functions(self):
        # check dummy MachineModel creation
        try:
            MachineModel(isa='x86')
            MachineModel(isa='aarch64')
        except ValueError:
            self.fail()
        test_mm_x86 = MachineModel(
            path_to_yaml=self._find_file('test_db_x86.yml'))
        test_mm_arm = MachineModel(
            path_to_yaml=self._find_file('test_db_aarch64.yml'))

        # test get_instruction without mnemonic
        self.assertIsNone(test_mm_x86.get_instruction(None, []))
        self.assertIsNone(test_mm_arm.get_instruction(None, []))

        # test get_instruction from DB
        self.assertIsNone(test_mm_x86.get_instruction(None, []))
        self.assertIsNone(test_mm_arm.get_instruction(None, []))
        self.assertIsNone(test_mm_x86.get_instruction('NOT_IN_DB', []))
        self.assertIsNone(test_mm_arm.get_instruction('NOT_IN_DB', []))
        name_x86_1 = 'vaddpd'
        operands_x86_1 = [
            {
                'class': 'register',
                'name': 'xmm'
            },
            {
                'class': 'register',
                'name': 'xmm'
            },
            {
                'class': 'register',
                'name': 'xmm'
            },
        ]
        instr_form_x86_1 = test_mm_x86.get_instruction(name_x86_1,
                                                       operands_x86_1)
        self.assertEqual(
            instr_form_x86_1,
            test_mm_x86.get_instruction(name_x86_1, operands_x86_1))
        self.assertEqual(
            test_mm_x86.get_instruction('jg', [{
                'class': 'identifier'
            }]),
            test_mm_x86.get_instruction('jg', [{
                'class': 'identifier'
            }]),
        )
        name_arm_1 = 'fadd'
        operands_arm_1 = [
            {
                'class': 'register',
                'prefix': 'v',
                'shape': 's'
            },
            {
                'class': 'register',
                'prefix': 'v',
                'shape': 's'
            },
            {
                'class': 'register',
                'prefix': 'v',
                'shape': 's'
            },
        ]
        instr_form_arm_1 = test_mm_arm.get_instruction(name_arm_1,
                                                       operands_arm_1)
        self.assertEqual(
            instr_form_arm_1,
            test_mm_arm.get_instruction(name_arm_1, operands_arm_1))
        self.assertEqual(
            test_mm_arm.get_instruction('b.ne', [{
                'class': 'identifier'
            }]),
            test_mm_arm.get_instruction('b.ne', [{
                'class': 'identifier'
            }]),
        )

        # test full instruction name
        self.assertEqual(
            MachineModel.get_full_instruction_name(instr_form_x86_1),
            'vaddpd  register(name:xmm),register(name:xmm),register(name:xmm)',
        )
        self.assertEqual(
            MachineModel.get_full_instruction_name(instr_form_arm_1),
            'fadd  register(prefix:v,shape:s),register(prefix:v,shape:s),' +
            'register(prefix:v,shape:s)',
        )

        # test get_store_tp
        self.assertEqual(
            test_mm_x86.get_store_throughput({
                'base': {
                    'name': 'x'
                },
                'offset': None,
                'index': None,
                'scale': 1
            }),
            [[2, '237'], [2, '4']],
        )
        self.assertEqual(
            test_mm_x86.get_store_throughput({
                'base': {
                    'prefix': 'NOT_IN_DB'
                },
                'offset': None,
                'index': 'NOT_NONE',
                'scale': 1
            }),
            [[1, '23'], [1, '4']],
        )
        self.assertEqual(
            test_mm_arm.get_store_throughput({
                'base': {
                    'prefix': 'x'
                },
                'offset': None,
                'index': None,
                'scale': 1
            }),
            [[2, '34'], [2, '5']],
        )
        self.assertEqual(
            test_mm_arm.get_store_throughput({
                'base': {
                    'prefix': 'NOT_IN_DB'
                },
                'offset': None,
                'index': None,
                'scale': 1
            }),
            [[1, '34'], [1, '5']],
        )

        # test get_store_lt
        self.assertEqual(
            test_mm_x86.get_store_latency({
                'base': {
                    'name': 'x'
                },
                'offset': None,
                'index': None,
                'scale': '1'
            }),
            0,
        )
        self.assertEqual(
            test_mm_arm.get_store_latency({
                'base': {
                    'prefix': 'x'
                },
                'offset': None,
                'index': None,
                'scale': '1'
            }),
            0,
        )

        # test has_hidden_load
        self.assertFalse(test_mm_x86.has_hidden_loads())

        # test default load tp
        self.assertEqual(
            test_mm_x86.get_load_throughput({
                'base': {
                    'name': 'x'
                },
                'offset': None,
                'index': None,
                'scale': 1
            }),
            [[1, '23'], [1, ['2D', '3D']]],
        )

        # test adding port
        test_mm_x86.add_port('dummyPort')
        test_mm_arm.add_port('dummyPort')

        # test dump of DB
        with open('/dev/null', 'w') as dev_null:
            test_mm_x86.dump(stream=dev_null)
            test_mm_arm.dump(stream=dev_null)
Пример #12
0
 def test_creation_by_name(self):
     try:
         tmp_mm = MachineModel(arch='CSX')
         ArchSemantics(tmp_mm)
     except ValueError:
         self.fail()
Пример #13
0
def extract_model(tree, arch, skip_mem=True):
    try:
        isa = MachineModel.get_isa_for_arch(arch)
    except Exception:
        print("Skipping...", file=sys.stderr)
        return None
    mm = MachineModel(isa=isa)
    parser = get_parser(isa)

    for instruction_tag in tree.findall(".//instruction"):
        ignore = False

        mnemonic = instruction_tag.attrib["asm"]
        iform = instruction_tag.attrib["iform"]
        # reduce to second part if mnemonic contain space (e.g., "REX CRC32")
        if " " in mnemonic:
            mnemonic = mnemonic.split(" ", 1)[1]

        # Extract parameter components
        try:
            parameters = extract_paramters(instruction_tag, parser, isa)
            if isa == "x86":
                parameters.reverse()
        except ValueError as e:
            print(e, file=sys.stderr)

        # Extract port occupation, throughput and latency
        port_pressure, throughput, latency, uops = [], None, None, None
        arch_tag = instruction_tag.find('architecture[@name="' + arch.upper() + '"]')
        if arch_tag is None:
            continue
        # skip any instructions without port utilization
        if not any(["ports" in x.attrib for x in arch_tag.findall("measurement")]):
            print("Couldn't find port utilization, skip: ", iform, file=sys.stderr)
            continue
        # skip if measured TP is smaller than computed
        if [float(x.attrib["TP_ports"]) > min(float(x.attrib["TP_loop"]),
                                              float(x.attrib["TP_unrolled"]))
                for x in arch_tag.findall("measurement")][0]:
            print(
                "Calculated TP is greater than measured TP.",
                iform,
                file=sys.stderr,
            )
        # skip if instruction contains memory operand
        if skip_mem and any(
            [x.attrib["type"] == "mem" for x in instruction_tag.findall("operand")]
        ):
            print("Contains memory operand, skip: ", iform, file=sys.stderr)
            continue
        # We collect all measurement and IACA information and compare them later
        for measurement_tag in arch_tag.iter("measurement"):
            if "TP_ports" in measurement_tag.attrib:
                throughput = float(measurement_tag.attrib["TP_ports"])
            else:
                throughput = min(
                    measurement_tag.attrib.get("TP_loop", float('inf')),
                    measurement_tag.attrib.get("TP_unroll", float('inf')),
                    measurement_tag.attrib.get("TP", float('inf')),
                )
                if throughput == float('inf'):
                    throughput = None
            uops = int(measurement_tag.attrib["uops"]) if "uops" in measurement_tag.attrib else None
            if "ports" in measurement_tag.attrib:
                port_pressure.append(port_pressure_from_tag_attributes(measurement_tag.attrib))
            latencies = [
                int(l_tag.attrib["cycles"])
                for l_tag in measurement_tag.iter("latency")
                if "cycles" in l_tag.attrib
            ]
            if len(latencies) == 0:
                latencies = [
                    int(l_tag.attrib["max_cycles"])
                    for l_tag in measurement_tag.iter("latency")
                    if "max_cycles" in l_tag.attrib
                ]
            if latencies[1:] != latencies[:-1]:
                print(
                    "Contradicting latencies found, using smallest:",
                    iform,
                    latencies,
                    file=sys.stderr,
                )
            if latencies:
                latency = min(latencies)
        if ignore:
            continue

        # Ordered by IACA version (newest last)
        for iaca_tag in sorted(
            arch_tag.iter("IACA"), key=lambda i: StrictVersion(i.attrib["version"])
        ):
            if "ports" in iaca_tag.attrib:
                port_pressure.append(port_pressure_from_tag_attributes(iaca_tag.attrib))

        # Check if all are equal
        if port_pressure:
            if port_pressure[1:] != port_pressure[:-1]:
                print("Contradicting port occupancies, using latest IACA:", iform, file=sys.stderr)
            port_pressure = port_pressure[-1]
        else:
            # print("No data available for this architecture:", mnemonic, file=sys.stderr)
            continue

        # Adding Intel's 2D and 3D pipelines on Intel µarchs, without Ice Lake:
        if arch.upper() in intel_archs and not arch.upper() in ["ICL"]:
            if any([p["class"] == "memory" for p in parameters]):
                # We have a memory parameter, if ports 2 & 3 are present, also add 2D & 3D
                # TODO remove port7 on 'hsw' onward and split entries depending on addressing mode
                port_23 = False
                port_4 = False
                for i, pp in enumerate(port_pressure):
                    if "2" in pp[1] and "3" in pp[1]:
                        port_23 = True
                    if "4" in pp[1]:
                        port_4 = True
                # Add (x, ['2D', '3D']) if load ports (2 & 3) are used, but not the store port (4)
                if port_23 and not port_4:
                    if arch.upper() in ["SNB", "IVB"] and any(
                            [p.get('name', '') == 'ymm' for p in parameters]) and \
                            not '128' in mnemonic:
                        # x = 2 if SNB or IVB and ymm regiser in any operand and not '128' in 
                        # instruction name
                        port2D3D_pressure = 2
                    else:
                        # otherwiese x = 1
                        port2D3D_pressure = 1
                    port_pressure.append((port2D3D_pressure, ["2D", "3D"]))

        # Add missing ports:
        for ports in [pp[1] for pp in port_pressure]:
            for p in ports:
                mm.add_port(p)

        throughput = max(mm.average_port_pressure(port_pressure))
        mm.set_instruction(mnemonic, parameters, latency, port_pressure, throughput, uops)
    # TODO eliminate entries which could be covered by automatic load / store expansion
    return mm
Пример #14
0
    def test_machine_model_various_functions(self):
        # check dummy MachineModel creation
        try:
            MachineModel(isa="x86")
            MachineModel(isa="aarch64")
        except ValueError:
            self.fail()
        test_mm_x86 = MachineModel(
            path_to_yaml=self._find_file("test_db_x86.yml"))
        test_mm_arm = MachineModel(
            path_to_yaml=self._find_file("test_db_aarch64.yml"))

        # test get_instruction without mnemonic
        self.assertIsNone(test_mm_x86.get_instruction(None, []))
        self.assertIsNone(test_mm_arm.get_instruction(None, []))

        # test get_instruction from DB
        self.assertIsNone(test_mm_x86.get_instruction(None, []))
        self.assertIsNone(test_mm_arm.get_instruction(None, []))
        self.assertIsNone(test_mm_x86.get_instruction("NOT_IN_DB", []))
        self.assertIsNone(test_mm_arm.get_instruction("NOT_IN_DB", []))
        name_x86_1 = "vaddpd"
        operands_x86_1 = [
            {
                "class": "register",
                "name": "xmm"
            },
            {
                "class": "register",
                "name": "xmm"
            },
            {
                "class": "register",
                "name": "xmm"
            },
        ]
        instr_form_x86_1 = test_mm_x86.get_instruction(name_x86_1,
                                                       operands_x86_1)
        self.assertEqual(
            instr_form_x86_1,
            test_mm_x86.get_instruction(name_x86_1, operands_x86_1))
        self.assertEqual(
            test_mm_x86.get_instruction("jg", [{
                "class": "identifier"
            }]),
            test_mm_x86.get_instruction("jg", [{
                "class": "identifier"
            }]),
        )
        name_arm_1 = "fadd"
        operands_arm_1 = [
            {
                "class": "register",
                "prefix": "v",
                "shape": "s"
            },
            {
                "class": "register",
                "prefix": "v",
                "shape": "s"
            },
            {
                "class": "register",
                "prefix": "v",
                "shape": "s"
            },
        ]
        instr_form_arm_1 = test_mm_arm.get_instruction(name_arm_1,
                                                       operands_arm_1)
        self.assertEqual(
            instr_form_arm_1,
            test_mm_arm.get_instruction(name_arm_1, operands_arm_1))
        self.assertEqual(
            test_mm_arm.get_instruction("b.ne", [{
                "class": "identifier"
            }]),
            test_mm_arm.get_instruction("b.ne", [{
                "class": "identifier"
            }]),
        )

        # test full instruction name
        self.assertEqual(
            MachineModel.get_full_instruction_name(instr_form_x86_1),
            "vaddpd  register(name:xmm),register(name:xmm),register(name:xmm)",
        )
        self.assertEqual(
            MachineModel.get_full_instruction_name(instr_form_arm_1),
            "fadd  register(prefix:v,shape:s),register(prefix:v,shape:s)," +
            "register(prefix:v,shape:s)",
        )

        # test get_store_tp
        self.assertEqual(
            test_mm_x86.get_store_throughput({
                "base": {
                    "name": "x"
                },
                "offset": None,
                "index": None,
                "scale": 1
            }),
            [[2, "237"], [2, "4"]],
        )
        self.assertEqual(
            test_mm_x86.get_store_throughput({
                "base": {
                    "prefix": "NOT_IN_DB"
                },
                "offset": None,
                "index": "NOT_NONE",
                "scale": 1
            }),
            [[1, "23"], [1, "4"]],
        )
        self.assertEqual(
            test_mm_arm.get_store_throughput({
                "base": {
                    "prefix": "x"
                },
                "offset": None,
                "index": None,
                "scale": 1
            }),
            [[2, "34"], [2, "5"]],
        )
        self.assertEqual(
            test_mm_arm.get_store_throughput({
                "base": {
                    "prefix": "NOT_IN_DB"
                },
                "offset": None,
                "index": None,
                "scale": 1
            }),
            [[1, "34"], [1, "5"]],
        )

        # test get_store_lt
        self.assertEqual(
            test_mm_x86.get_store_latency({
                "base": {
                    "name": "x"
                },
                "offset": None,
                "index": None,
                "scale": "1"
            }),
            0,
        )
        self.assertEqual(
            test_mm_arm.get_store_latency({
                "base": {
                    "prefix": "x"
                },
                "offset": None,
                "index": None,
                "scale": "1"
            }),
            0,
        )

        # test has_hidden_load
        self.assertFalse(test_mm_x86.has_hidden_loads())

        # test default load tp
        self.assertEqual(
            test_mm_x86.get_load_throughput({
                "base": {
                    "name": "x"
                },
                "offset": None,
                "index": None,
                "scale": 1
            }),
            [[1, "23"], [1, ["2D", "3D"]]],
        )

        # test adding port
        test_mm_x86.add_port("dummyPort")
        test_mm_arm.add_port("dummyPort")

        # test dump of DB
        with open("/dev/null", "w") as dev_null:
            test_mm_x86.dump(stream=dev_null)
            test_mm_arm.dump(stream=dev_null)
Пример #15
0
def inspect(args, output_file=sys.stdout):
    """
    Does the actual throughput and critical path analysis of OSACA and prints it to the
    terminal.

    :param args: arguments given from :class:`~argparse.ArgumentParser` after parsing
    :param output_file: Define the stream for output, defaults to :class:`sys.stdout`
    :type output_file: stream, optional
    """
    # Read file
    code = args.file.read()

    # Detect ISA if necessary
    arch = args.arch if args.arch is not None else DEFAULT_ARCHS[BaseParser.detect_ISA(code)]
    print_arch_warning = False if args.arch else True
    isa = MachineModel.get_isa_for_arch(arch)
    verbose = args.verbose
    ignore_unknown = args.ignore_unknown

    # Parse file
    parser = get_asm_parser(arch)
    try:
        parsed_code = parser.parse_file(code)
    except:
        # probably the wrong parser based on heuristic
        if args.arch is None:
            # change ISA and try again
            arch = DEFAULT_ARCHS['x86'] if BaseParser.detect_ISA(code) == 'aarch64' else DEFAULT_ARCHS['aarch64']
            isa = MachineModel.get_isa_for_arch(arch)
            parser = get_asm_parser(arch)
            parsed_code = parser.parse_file(code)
        else:
            traceback.print_exc(file=sys.stderr)
            sys.exit(1)

    # Reduce to marked kernel or chosen section and add semantics
    if args.lines:
        line_range = get_line_range(args.lines)
        kernel = [line for line in parsed_code if line['line_number'] in line_range]
        print_length_warning = False
    else:
        kernel = reduce_to_section(parsed_code, isa)
        # Print warning if kernel has no markers and is larger than threshold (100)
        print_length_warning = True if len(kernel) == len(parsed_code) and len(kernel) > 100 else False
    machine_model = MachineModel(arch=arch)
    semantics = ArchSemantics(machine_model)
    semantics.add_semantics(kernel)
    # Do optimal schedule for kernel throughput if wished
    if not args.fixed:
        semantics.assign_optimal_throughput(kernel)

    # Create DiGrahps
    kernel_graph = KernelDG(kernel, parser, machine_model)
    if args.dotpath is not None:
        kernel_graph.export_graph(args.dotpath if args.dotpath != '.' else None)
    # Print analysis
    frontend = Frontend(args.file.name, arch=arch)
    print(
        frontend.full_analysis(
            kernel,
            kernel_graph,
            ignore_unknown=ignore_unknown,
            arch_warning=print_arch_warning,
            length_warning=print_length_warning,
            verbose=verbose
        ),
        file=output_file,
    )
Пример #16
0
def extract_model(tree, arch, skip_mem=True):
    try:
        isa = MachineModel.get_isa_for_arch(arch)
    except Exception:
        print("Skipping...", file=sys.stderr)
        return None
    mm = MachineModel(isa=isa)
    parser = get_parser(isa)

    for instruction_tag in tree.findall('.//instruction'):
        ignore = False

        mnemonic = instruction_tag.attrib['asm']
        iform = instruction_tag.attrib['iform']
        # skip any mnemonic which contain spaces (e.g., "REX CRC32")
        if ' ' in mnemonic:
            continue

        # Extract parameter components
        try:
            parameters = extract_paramters(instruction_tag, parser, isa)
            if isa == 'x86':
                parameters.reverse()
        except ValueError as e:
            print(e, file=sys.stderr)

        # Extract port occupation, throughput and latency
        port_pressure, throughput, latency, uops = [], None, None, None
        arch_tag = instruction_tag.find('architecture[@name="' + arch.upper() + '"]')
        if arch_tag is None:
            continue
        # skip any instructions without port utilization
        if not any(['ports' in x.attrib for x in arch_tag.findall('measurement')]):
            print("Couldn't find port utilization, skip: ", iform, file=sys.stderr)
            continue
        # skip if computed and measured TP don't match
        if not [x.attrib['TP_ports'] == x.attrib['TP'] for x in arch_tag.findall('measurement')][
            0
        ]:
            print(
                "Calculated TP from port utilization doesn't match TP, skip: ",
                iform,
                file=sys.stderr,
            )
            continue
        # skip if instruction contains memory operand
        if skip_mem and any(
            [x.attrib['type'] == 'mem' for x in instruction_tag.findall('operand')]
        ):
            print("Contains memory operand, skip: ", iform, file=sys.stderr)
            continue
        # We collect all measurement and IACA information and compare them later
        for measurement_tag in arch_tag.iter('measurement'):
            if 'TP_ports' in measurement_tag.attrib:
                throughput = measurement_tag.attrib['TP_ports']
            else:
                throughput = (
                    measurement_tag.attrib['TP'] if 'TP' in measurement_tag.attrib else None
                )
            uops = (
                int(measurement_tag.attrib['uops']) if 'uops' in measurement_tag.attrib else None
            )
            if 'ports' in measurement_tag.attrib:
                port_pressure.append(port_pressure_from_tag_attributes(measurement_tag.attrib))
            latencies = [
                int(l_tag.attrib['cycles'])
                for l_tag in measurement_tag.iter('latency')
                if 'cycles' in l_tag.attrib
            ]
            if len(latencies) == 0:
                latencies = [
                    int(l_tag.attrib['max_cycles'])
                    for l_tag in measurement_tag.iter('latency')
                    if 'max_cycles' in l_tag.attrib
                ]
            if latencies[1:] != latencies[:-1]:
                print(
                    "Contradicting latencies found, using smallest:",
                    iform,
                    latencies,
                    file=sys.stderr,
                )
            if latencies:
                latency = min(latencies)
        if ignore:
            continue

        # Ordered by IACA version (newest last)
        for iaca_tag in sorted(
            arch_tag.iter('IACA'), key=lambda i: StrictVersion(i.attrib['version'])
        ):
            if 'ports' in iaca_tag.attrib:
                port_pressure.append(port_pressure_from_tag_attributes(iaca_tag.attrib))

        # Check if all are equal
        if port_pressure:
            if port_pressure[1:] != port_pressure[:-1]:
                print("Contradicting port occupancies, using latest IACA:", iform, file=sys.stderr)
            port_pressure = port_pressure[-1]
        else:
            # print("No data available for this architecture:", mnemonic, file=sys.stderr)
            continue

        # Adding Intel's 2D and 3D pipelines on Intel µarchs, without Ice Lake:
        if arch.upper() in intel_archs and not arch.upper() in ['ICL']:
            if any([p['class'] == 'memory' for p in parameters]):
                # We have a memory parameter, if ports 2 & 3 are present, also add 2D & 3D
                # TODO remove port7 on 'hsw' onward and split entries depending on addressing mode
                port_23 = False
                port_4 = False
                for i, pp in enumerate(port_pressure):
                    if '2' in pp[1] and '3' in pp[1]:
                        port_23 = True
                    if '4' in pp[1]:
                        port_4 = True
                # Add (X, ['2D', '3D']) if load ports (2 & 3) are used, but not the store port (4)
                # X = 2 on SNB and IVB IFF used in combination with ymm register, otherwise X = 1
                if arch.upper() in ['SNB', 'IVB'] and \
                    any([p['class'] == 'register' and p['name'] == 'ymm' for p in parameters]):
                    data_port_throughput = 2
                else:
                    data_port_throughput = 1
                if port_23 and not port_4:
                    port_pressure.append((data_port_throughput, ['2D', '3D']))

        # Add missing ports:
        for ports in [pp[1] for pp in port_pressure]:
            for p in ports:
                mm.add_port(p)

        throughput = max(mm.average_port_pressure(port_pressure))

        mm.set_instruction(mnemonic, parameters, latency, port_pressure, throughput, uops)
    # TODO eliminate entries which could be covered by automatic load / store expansion
    return mm
Пример #17
0
    def setUpClass(cls):
        # set up parser and kernels
        cls.parser_x86 = ParserX86ATT()
        cls.parser_AArch64 = ParserAArch64()
        with open(cls._find_file("kernel_x86.s")) as f:
            cls.code_x86 = f.read()
        with open(cls._find_file("kernel_x86_memdep.s")) as f:
            cls.code_x86_memdep = f.read()
        with open(cls._find_file("kernel_x86_long_LCD.s")) as f:
            cls.code_x86_long_LCD = f.read()
        with open(cls._find_file("kernel_aarch64_memdep.s")) as f:
            cls.code_aarch64_memdep = f.read()
        with open(cls._find_file("kernel_aarch64.s")) as f:
            cls.code_AArch64 = f.read()
        with open(cls._find_file("kernel_aarch64_sve.s")) as f:
            cls.code_AArch64_SVE = f.read()
        cls.kernel_x86 = reduce_to_section(
            cls.parser_x86.parse_file(cls.code_x86), "x86")
        cls.kernel_x86_memdep = reduce_to_section(
            cls.parser_x86.parse_file(cls.code_x86_memdep), "x86")
        cls.kernel_x86_long_LCD = reduce_to_section(
            cls.parser_x86.parse_file(cls.code_x86_long_LCD), "x86")
        cls.kernel_AArch64 = reduce_to_section(
            cls.parser_AArch64.parse_file(cls.code_AArch64), "aarch64")
        cls.kernel_aarch64_memdep = reduce_to_section(
            cls.parser_AArch64.parse_file(cls.code_aarch64_memdep), "aarch64")
        cls.kernel_aarch64_SVE = reduce_to_section(
            cls.parser_AArch64.parse_file(cls.code_AArch64_SVE), "aarch64")

        # set up machine models
        cls.machine_model_csx = MachineModel(
            path_to_yaml=os.path.join(cls.MODULE_DATA_DIR, "csx.yml"))
        cls.machine_model_tx2 = MachineModel(
            path_to_yaml=os.path.join(cls.MODULE_DATA_DIR, "tx2.yml"))
        cls.machine_model_a64fx = MachineModel(
            path_to_yaml=os.path.join(cls.MODULE_DATA_DIR, "a64fx.yml"))
        cls.semantics_x86 = ISASemantics("x86")
        cls.semantics_csx = ArchSemantics(cls.machine_model_csx,
                                          path_to_yaml=os.path.join(
                                              cls.MODULE_DATA_DIR,
                                              "isa/x86.yml"))
        cls.semantics_aarch64 = ISASemantics("aarch64")
        cls.semantics_tx2 = ArchSemantics(
            cls.machine_model_tx2,
            path_to_yaml=os.path.join(cls.MODULE_DATA_DIR, "isa/aarch64.yml"),
        )
        cls.semantics_a64fx = ArchSemantics(
            cls.machine_model_a64fx,
            path_to_yaml=os.path.join(cls.MODULE_DATA_DIR, "isa/aarch64.yml"),
        )
        cls.machine_model_zen = MachineModel(arch="zen1")

        for i in range(len(cls.kernel_x86)):
            cls.semantics_csx.assign_src_dst(cls.kernel_x86[i])
            cls.semantics_csx.assign_tp_lt(cls.kernel_x86[i])
        for i in range(len(cls.kernel_x86_memdep)):
            cls.semantics_csx.assign_src_dst(cls.kernel_x86_memdep[i])
            cls.semantics_csx.assign_tp_lt(cls.kernel_x86_memdep[i])
        for i in range(len(cls.kernel_x86_long_LCD)):
            cls.semantics_csx.assign_src_dst(cls.kernel_x86_long_LCD[i])
            cls.semantics_csx.assign_tp_lt(cls.kernel_x86_long_LCD[i])
        for i in range(len(cls.kernel_AArch64)):
            cls.semantics_tx2.assign_src_dst(cls.kernel_AArch64[i])
            cls.semantics_tx2.assign_tp_lt(cls.kernel_AArch64[i])
        for i in range(len(cls.kernel_aarch64_memdep)):
            cls.semantics_tx2.assign_src_dst(cls.kernel_aarch64_memdep[i])
            cls.semantics_tx2.assign_tp_lt(cls.kernel_aarch64_memdep[i])
        for i in range(len(cls.kernel_aarch64_SVE)):
            cls.semantics_a64fx.assign_src_dst(cls.kernel_aarch64_SVE[i])
            cls.semantics_a64fx.assign_tp_lt(cls.kernel_aarch64_SVE[i])
Пример #18
0
 def test_invalid_add(self):
     entry = {}
     with self.assertRaises(KeyError):
         MachineModel('csx').set_instruction_entry(entry)
     with self.assertRaises(TypeError):
         MachineModel('csx').set_instruction()