Ejemplo n.º 1
0
def is_ctu_capable(extdef_map_cmd):
    """ Detects if the current (or given) clang and external definition mapping
    executables are CTU compatible. """

    try:
        run_command([extdef_map_cmd, '-version'])
    except (OSError, subprocess.CalledProcessError):
        return False
    return True
Ejemplo n.º 2
0
def report_failure(opts):
    # type: (Dict[str, Any]) -> None
    """ Create report when analyzer failed.

    The major report is the preprocessor output. The output filename generated
    randomly. The compiler output also captured into '.stderr.txt' file.
    And some more execution context also saved into '.info.txt' file. """

    def extension():
        # type: () -> str
        """ Generate preprocessor file extension. """

        mapping = {'objective-c++': '.mii', 'objective-c': '.mi', 'c++': '.ii'}
        return mapping.get(opts['language'], '.i')

    def destination():
        # type: () -> str
        """ Creates failures directory if not exits yet. """

        failures_dir = os.path.join(opts['output_dir'], 'failures')
        if not os.path.isdir(failures_dir):
            os.makedirs(failures_dir)
        return failures_dir

    # Classify error type: when Clang terminated by a signal it's a 'Crash'.
    # (python subprocess Popen.returncode is negative when child terminated
    # by signal.) Everything else is 'Other Error'.
    error = 'crash' if opts['exit_code'] < 0 else 'other_error'
    # Create preprocessor output file name. (This is blindly following the
    # Perl implementation.)
    (fd, name) = tempfile.mkstemp(suffix=extension(),
                                  prefix='clang_' + error + '_',
                                  dir=destination())
    os.close(fd)
    # Execute Clang again, but run the syntax check only.
    try:
        cwd = opts['directory']
        cmd = get_arguments([opts['clang'], '-fsyntax-only', '-E'] +
                            opts['flags'] + [opts['source'], '-o', name], cwd)
        run_command(cmd, cwd=cwd)
        # write general information about the crash
        with open(name + '.info.txt', 'w') as handle:
            handle.write(opts['source'] + os.linesep)
            handle.write(error.title().replace('_', ' ') + os.linesep)
            handle.write(' '.join(cmd) + os.linesep)
            handle.write(' '.join(platform.uname()) + os.linesep)
            handle.write(get_version(opts['clang']))
            handle.close()
        # write the captured output too
        with open(name + '.stderr.txt', 'w') as handle:
            for line in opts['error_output']:
                handle.write(line)
            handle.close()
    except (OSError, subprocess.CalledProcessError):
        logging.warning('failed to report failure', exc_info=True)
Ejemplo n.º 3
0
    def map_extdefs(triple_arch):
        """ Generate external definition map file for the current source. """

        args = opts['direct_args'] + opts['flags']
        extdefmap_command = [opts['ctu'].extdef_map_cmd]
        extdefmap_command.append(opts['file'])
        extdefmap_command.append('--')
        extdefmap_command.extend(args)
        logging.debug("Generating external definition map using '%s'",
                      extdefmap_command)
        extdef_src_list = run_command(extdefmap_command, cwd=opts['directory'])
        extdef_ast_list = extdef_map_list_src_to_ast(extdef_src_list)
        extern_defs_map_folder = os.path.join(opts['ctu'].dir, triple_arch,
                                             CTU_TEMP_DEFMAP_FOLDER)
        if not os.path.isdir(extern_defs_map_folder):
            try:
                os.makedirs(extern_defs_map_folder)
            except OSError:
                # In case an other process already created it.
                pass
        if extdef_ast_list:
            with tempfile.NamedTemporaryFile(mode='w',
                                             dir=extern_defs_map_folder,
                                             delete=False) as out_file:
                out_file.write("\n".join(extdef_ast_list) + "\n")
Ejemplo n.º 4
0
def run_analyzer(opts, continuation=report_failure):
    """ It assembles the analysis command line and executes it. Capture the
    output of the analysis and returns with it. If failure reports are
    requested, it calls the continuation to generate it. """

    def target():
        """ Creates output file name for reports. """
        if opts['output_format'] in {
                'plist',
                'plist-html',
                'plist-multi-file'}:
            (handle, name) = tempfile.mkstemp(prefix='report-',
                                              suffix='.plist',
                                              dir=opts['output_dir'])
            os.close(handle)
            return name
        return opts['output_dir']

    try:
        cwd = opts['directory']
        cmd = get_arguments([opts['clang'], '--analyze'] +
                            opts['direct_args'] + opts['flags'] +
                            [opts['file'], '-o', target()],
                            cwd)
        output = run_command(cmd, cwd=cwd)
        return {'error_output': output, 'exit_code': 0}
    except subprocess.CalledProcessError as ex:
        result = {'error_output': ex.output, 'exit_code': ex.returncode}
        if opts.get('output_failures', False):
            opts.update(result)
            continuation(opts)
        return result
Ejemplo n.º 5
0
def run_analyzer(opts, continuation=report_failure):
    # type: (...) -> Dict[str, Any]
    """ It assembles the analysis command line and executes it. Capture the
    output of the analysis and returns with it. If failure reports are
    requested, it calls the continuation to generate it. """

    def target():
        # type: () -> str
        """ Creates output file name for reports. """
        if opts['output_format'].startswith('plist'):
            (handle, name) = tempfile.mkstemp(prefix='report-',
                                              suffix='.plist',
                                              dir=opts['output_dir'])
            os.close(handle)
            return name
        return opts['output_dir']

    try:
        cwd = opts['directory']
        cmd = get_arguments([opts['clang'], '--analyze'] +
                            opts['direct_args'] + opts['flags'] +
                            [opts['source'], '-o', target()],
                            cwd)
        output = run_command(cmd, cwd=cwd)
        return {'error_output': output, 'exit_code': 0}
    except OSError:
        message = 'failed to execute "{0}"'.format(opts['clang'])
        return {'error_output': [message], 'exit_code': 127}
    except subprocess.CalledProcessError as ex:
        logging.warning('analysis failed', exc_info=True)
        result = {'error_output': ex.output, 'exit_code': ex.returncode}
        if opts.get('output_failures', False):
            opts.update(result)
            continuation(opts)
        return result
Ejemplo n.º 6
0
def get_checkers(clang, plugins):
    # type: (str, List[str]) -> Dict[str, Tuple[str, bool]]
    """ Get all the available checkers from default and from the plugins.

    :param clang:   the compiler we are using
    :param plugins: list of plugins which was requested by the user
    :return:        a dictionary of all available checkers and its status

    {<checker name>: (<checker description>, <is active by default>)} """

    load = [elem for plugin in plugins for elem in ['-load', plugin]]
    cmd = [clang, '-cc1'] + load + ['-analyzer-checker-help']

    lines = run_command(cmd)

    is_active_checker = is_active(get_active_checkers(clang, plugins))

    checkers = {
        name: (description, is_active_checker(name))
        for name, description in parse_checkers(lines)
    }
    if not checkers:
        raise Exception('Could not query Clang for available checkers.')

    return checkers
Ejemplo n.º 7
0
def get_version(clang):
    """ Returns the compiler version as string.

    :param clang:   the compiler we are using
    :return:        the version string printed to stderr """

    output = run_command([clang, '-v'])
    # the relevant version info is in the first line
    return output[0]
Ejemplo n.º 8
0
def find_compiler_config(compiler, tmp_file_name, tmp_dir):
    filePath = os.path.join(tmp_dir, tmp_file_name)
    with open(filePath, 'w'):
        pass
    config_output = run_command([compiler, '-E', '-dM', '-v', tmp_file_name],
                                cwd=tmp_dir)
    logging.debug('config_output is %s' % config_output)
    compiler_conf = recognize_config(config_output)
    logging.debug('Recognized config: %s' % compiler_conf)
    return compiler_conf
Ejemplo n.º 9
0
def get_version(clang):
    # type: (str) -> str
    """ Returns the compiler version as string.

    :param clang:   the compiler we are using
    :return:        the version string printed to stderr """

    output = run_command([clang, '-v'])
    # the relevant version info is in the first line
    return output[0]
Ejemplo n.º 10
0
    def generate_ast(triple_arch):
        """ Generates ASTs for the current compilation command. """

        args = opts['direct_args'] + opts['flags']
        ast_joined_path = os.path.join(opts['ctu'].dir, triple_arch, 'ast',
                                       os.path.realpath(opts['file'])[1:] +
                                       '.ast')
        ast_path = os.path.abspath(ast_joined_path)
        ast_dir = os.path.dirname(ast_path)
        if not os.path.isdir(ast_dir):
            try:
                os.makedirs(ast_dir)
            except OSError:
                pass
        ast_command = [opts['clang'], '-emit-ast']
        ast_command.extend(args)
        ast_command.append('-w')
        ast_command.append(opts['file'])
        ast_command.append('-o')
        ast_command.append(ast_path)
        logging.debug("Generating AST using '%s'", ast_command)
        run_command(ast_command, cwd=opts['directory'])
Ejemplo n.º 11
0
    def generate_ast(triple_arch):
        """ Generates ASTs for the current compilation command. """

        args = opts['direct_args'] + opts['flags']
        ast_joined_path = os.path.join(opts['ctu'].dir, triple_arch, 'ast',
                                       os.path.realpath(opts['file'])[1:] +
                                       '.ast')
        ast_path = os.path.abspath(ast_joined_path)
        ast_dir = os.path.dirname(ast_path)
        if not os.path.isdir(ast_dir):
            try:
                os.makedirs(ast_dir)
            except OSError:
                # In case an other process already created it.
                pass
        ast_command = [opts['clang'], '-emit-ast']
        ast_command.extend(args)
        ast_command.append('-w')
        ast_command.append(opts['file'])
        ast_command.append('-o')
        ast_command.append(ast_path)
        logging.debug("Generating AST using '%s'", ast_command)
        run_command(ast_command, cwd=opts['directory'])
Ejemplo n.º 12
0
def get_mpi_call(wrapper):
    # type: (str) -> List[str]
    """ Provide information on how the underlying compiler would have been
    invoked without the MPI compiler wrapper. """

    for query_flags in [['-show'], ['--showme']]:
        try:
            output = run_command([wrapper] + query_flags)
            if output:
                return shell_split(output[0])
        except (OSError, subprocess.CalledProcessError):
            pass
    # Fail loud
    raise RuntimeError("Could not determinate MPI flags.")
Ejemplo n.º 13
0
def get_arguments(command, cwd):
    """ Capture Clang invocation.

    :param command: the compilation command
    :param cwd:     the current working directory
    :return:        the detailed front-end invocation command """

    cmd = command[:]
    cmd.insert(1, '-###')

    output = run_command(cmd, cwd=cwd)
    # The relevant information is in the last line of the output.
    # Don't check if finding last line fails, would throw exception anyway.
    last_line = output[-1]
    if re.search(r'clang(.*): error:', last_line):
        raise Exception(last_line)
    return decode(last_line)
Ejemplo n.º 14
0
def get_arguments(command, cwd):
    """ Capture Clang invocation.

    :param command: the compilation command
    :param cwd:     the current working directory
    :return:        the detailed front-end invocation command """

    cmd = command[:]
    cmd.insert(1, '-###')

    output = run_command(cmd, cwd=cwd)
    # The relevant information is in the last line of the output.
    # Don't check if finding last line fails, would throw exception anyway.
    last_line = output[-1]
    if re.search(r'clang(.*): error:', last_line):
        raise Exception(last_line)
    return decode(last_line)
Ejemplo n.º 15
0
def run_analyzer(opts, continuation=report_failure):
    """ It assembles the analysis command line and executes it. Capture the
    output of the analysis and returns with it. If failure reports are
    requested, it calls the continuation to generate it. """

    def target():
        """ Creates output file name for reports. """
        if opts['output_format'] in {
                'plist',
                'plist-html',
                'plist-multi-file'}:
            (handle, name) = tempfile.mkstemp(prefix='report-',
                                              suffix='.plist',
                                              dir=opts['output_dir'])
            os.close(handle)
            return name
        elif opts['output_format'] in {
                'sarif',
                'sarif-html'}:
            (handle, name) = tempfile.mkstemp(prefix='result-',
                                              suffix='.sarif',
                                              dir=opts['output_dir'])
            os.close(handle)
            return name
        return opts['output_dir']

    try:
        cwd = opts['directory']
        cmd = get_arguments([opts['clang'], '--analyze'] +
                            opts['direct_args'] + opts['flags'] +
                            [opts['file'], '-o', target()],
                            cwd)
        output = run_command(cmd, cwd=cwd)
        return {'error_output': output, 'exit_code': 0}
    except subprocess.CalledProcessError as ex:
        result = {'error_output': ex.output, 'exit_code': ex.returncode}
        if opts.get('output_failures', False):
            opts.update(result)
            continuation(opts)
        return result
    except ClangErrorException as ex:
        result = {'error_output': ex.error, 'exit_code': 0}
        if opts.get('output_failures', False):
            opts.update(result)
            continuation(opts)
        return result
Ejemplo n.º 16
0
def is_preload_disabled(platform):
    """ Library-based interposition will fail silently if SIP is enabled,
    so this should be detected. You can detect whether SIP is enabled on
    Darwin by checking whether (1) there is a binary called 'csrutil' in
    the path and, if so, (2) whether the output of executing 'csrutil status'
    contains 'System Integrity Protection status: enabled'.

    :param platform: name of the platform (returned by sys.platform),
    :return: True if library preload will fail by the dynamic linker. """

    if platform in WRAPPER_ONLY_PLATFORMS:
        return True
    elif platform == 'darwin':
        command = ['csrutil', 'status']
        pattern = re.compile(r'System Integrity Protection status:\s+enabled')
        try:
            return any(pattern.match(line) for line in run_command(command))
        except:
            return False
    else:
        return False
Ejemplo n.º 17
0
    def map_functions(triple_arch):
        """ Generate function map file for the current source. """

        args = opts['direct_args'] + opts['flags']
        funcmap_command = [opts['ctu'].func_map_cmd]
        funcmap_command.append(opts['file'])
        funcmap_command.append('--')
        funcmap_command.extend(args)
        logging.debug("Generating function map using '%s'", funcmap_command)
        func_src_list = run_command(funcmap_command, cwd=opts['directory'])
        func_ast_list = func_map_list_src_to_ast(func_src_list, triple_arch)
        extern_fns_map_folder = os.path.join(opts['ctu'].dir, triple_arch,
                                             CTU_TEMP_FNMAP_FOLDER)
        if not os.path.isdir(extern_fns_map_folder):
            try:
                os.makedirs(extern_fns_map_folder)
            except OSError:
                pass
        if func_ast_list:
            with tempfile.NamedTemporaryFile(mode='w',
                                             dir=extern_fns_map_folder,
                                             delete=False) as out_file:
                out_file.write("\n".join(func_ast_list) + "\n")
Ejemplo n.º 18
0
def get_checkers(clang, plugins):
    """ Get all the available checkers from default and from the plugins.

    :param clang:   the compiler we are using
    :param plugins: list of plugins which was requested by the user
    :return:        a dictionary of all available checkers and its status

    {<checker name>: (<checker description>, <is active by default>)} """

    load = [elem for plugin in plugins for elem in ['-load', plugin]]
    cmd = [clang, '-cc1'] + load + ['-analyzer-checker-help']

    lines = run_command(cmd)

    is_active_checker = is_active(get_active_checkers(clang, plugins))

    checkers = {
        name: (description, is_active_checker(name))
        for name, description in parse_checkers(lines)
    }
    if not checkers:
        raise Exception('Could not query Clang for available checkers.')

    return checkers