Beispiel #1
0
    def run(self, outfile_name):
        error_file_name = "{0}.err".format(outfile_name)
        return_code = execute_command(self.parts, outfile_name)
        error_file_lines = read_file(error_file_name)
        outfile_lines = read_file(outfile_name)

        return ExternalCommandResult(return_code, outfile_name, outfile_lines, error_file_name, error_file_lines)
 def run_on_production_source_files(self, logger, include_test_sources=False, include_scripts=False):
     execution_result = execute_tool_on_source_files(project=self.project,
                                                     name=self.command_name,
                                                     command_and_arguments=self.parts,
                                                     include_test_sources=include_test_sources,
                                                     include_scripts=include_scripts,
                                                     logger=logger)
     exit_code, report_file = execution_result
     report_lines = read_file(report_file)
     error_report_file = '{0}.err'.format(report_file)  # TODO @mriehl not dry, execute_tool... should return this
     error_report_lines = read_file(error_report_file)
     return ExternalCommandResult(exit_code, report_file, report_lines, error_report_file, error_report_lines)
def run_single_test(logger, project, reports_dir, test, output_test_names=True):
    name, _ = os.path.splitext(os.path.basename(test))
    if output_test_names:
        logger.info("Running integration test %s", name)

    env = prepare_environment(project)
    test_time = Timer.start()
    command_and_arguments = (sys.executable, test)
    report_file_name = os.path.join(reports_dir, name)
    error_file_name = report_file_name + ".err"
    return_code = execute_command(
        command_and_arguments, report_file_name, env, error_file_name=error_file_name)
    test_time.stop()
    report_item = {
        "test": name,
        "test_file": test,
        "time": test_time.get_millis(),
        "success": True
    }
    if return_code != 0:

        logger.error("Integration test failed: %s", test)
        report_item["success"] = False

        if project.get_property("verbose"):
            print_file_content(report_file_name)
            print_text_line()
            print_file_content(error_file_name)
            report_item['exception'] = ''.join(read_file(error_file_name)).replace('\'', '')

    return report_item
def _install_external_plugin(project, name, version, logger, plugin_module_name, upgrade=False, force_reinstall=False):
    if not name.startswith(PYPI_PLUGIN_PROTOCOL) and not name.startswith(VCS_PLUGIN_PROTOCOL):
        message = "Only plugins starting with '{0}' are currently supported"
        raise MissingPluginException(name, message.format((PYPI_PLUGIN_PROTOCOL, VCS_PLUGIN_PROTOCOL)))

    if name.startswith(PYPI_PLUGIN_PROTOCOL):
        pip_package = name.replace(PYPI_PLUGIN_PROTOCOL, "")
        if version:
            pip_package += str(version)
            upgrade = True
    elif name.startswith(VCS_PLUGIN_PROTOCOL):
        pip_package = name.replace(VCS_PLUGIN_PROTOCOL, "")
        force_reinstall = True

    with tempfile.NamedTemporaryFile(delete=True) as log_file:
        log_file_name = log_file.name
        result = pip_install(pip_package,
                             index_url=project.get_property("install_dependencies_index_url"),
                             extra_index_url=project.get_property("install_dependencies_extra_index_url"),
                             trusted_host=project.get_property("install_dependencies_trusted_host"),
                             upgrade=upgrade,
                             force_reinstall=force_reinstall,
                             logger=logger,
                             outfile_name=log_file_name,
                             error_file_name=log_file_name,
                             cwd=".")
        if result != 0:
            logger.error("The following pip error was encountered:\n" + "".join(read_file(log_file_name)))
            message = "Failed to install plugin from {0}".format(pip_package)
            raise MissingPluginException(name, message)
def run_cram_tests(project, logger):
    logger.info("Running Cram command line tests")

    command_and_arguments = _cram_command_for(project)
    command_and_arguments.extend(_find_files(project))
    report_file = _report_file(project)

    env = os.environ.copy()
    source_dir = project.expand_path("$dir_source_main_python")
    _prepend_path(env, "PYTHONPATH", source_dir)
    script_dir = project.expand_path('$dir_source_main_scripts')
    _prepend_path(env, "PATH", script_dir)

    return_code = execute_command(command_and_arguments,
                                  report_file,
                                  env=env,
                                  error_file_name=report_file)

    report = read_file(report_file)
    result = report[-1][2:].strip()

    if return_code != 0:
        logger.error("Cram tests failed!")
        if project.get_property("verbose"):
            for line in report:
                logger.error(line.rstrip())
        else:
            logger.error(result)

        logger.error("See: '{0}' for details".format(report_file))
        raise BuildFailedException("Cram tests failed!")

    logger.info("Cram tests were fine")
    logger.info(result)
Beispiel #6
0
def analyze(project, logger):
    logger.info("Executing pep8 on project sources")
    _, report_file = execute_tool_on_source_files(project, "pep8", ["pep8"])

    reports = read_file(report_file)

    if len(reports) > 0:
        logger.warn("Found %d warning%s produced by pep8",
                    len(reports), "" if len(reports) == 1 else "s")
Beispiel #7
0
def _install_external_plugin(name, logger):
    if not name.startswith(PYPI_PLUGIN_PROTOCOL):
        message = "Only plugins starting with '{0}' are currently supported"
        raise MissingPluginException(name, message.format(PYPI_PLUGIN_PROTOCOL))
    plugin_name_on_pypi = name.replace(PYPI_PLUGIN_PROTOCOL, "")
    log_file = tempfile.NamedTemporaryFile(delete=False).name
    result = execute_command(
        "pip install {0}".format(plugin_name_on_pypi), log_file, error_file_name=log_file, shell=True
    )
    if result != 0:
        logger.error("The following pip error was encountered:\n" + "".join(read_file(log_file)))
        message = "Failed to install from PyPI".format(plugin_name_on_pypi)
        raise MissingPluginException(name, message)
def execute_tool_on_source_files(project, name, command_and_arguments, logger=None, include_test_sources=False):
    files = discover_affected_files(include_test_sources, project)

    command = as_list(command_and_arguments) + [f for f in files]

    report_file = project.expand_path("$dir_reports/{0}".format(name))

    execution_result = execute_command(command, report_file), report_file

    report_file = execution_result[1]
    report_lines = read_file(report_file)

    if project.get_property(name + "_verbose_output") and logger:
        log_report(logger, name, report_lines)

    return execution_result
Beispiel #9
0
def execute_pylint(project, logger):
    logger.info("Executing pylint on project sources")

    command_and_arguments = ["pylint"] + project.get_property("pylint_options")
    result_tuple = execute_tool_on_modules(project, "pylint", command_and_arguments, True)

    if project.get_property("pylint_break_build"):
        report_file = result_tuple[1]  # access by position to avoid changing mocking behaviour

        warnings = [line
                    for line in read_file(report_file)
                    if line.find('.py:') >= 0]
        warning_count = len(warnings)

        if warning_count > 0:
            message = "Pylint found {} warning(s).".format(warning_count)
            logger.error(message)
            raise BuildFailedException(message)
Beispiel #10
0
def execute_pychecker(project, logger):
    command_line = build_command_line(project)
    logger.info("Executing pychecker on project sources: %s" % (' '.join(command_line)))

    _, report_file = execute_tool_on_modules(project, "pychecker", command_line, True)

    warnings = read_file(report_file)

    report = parse_pychecker_output(project, warnings)
    project.write_report("pychecker.json", render_report(report.to_json_dict()))

    if len(warnings) != 0:
        logger.warn("Found %d warning%s produced by pychecker. See %s for details.",
                    len(warnings),
                    "s" if len(warnings) != 1 else "",
                    report_file)

        threshold = project.get_property("pychecker_break_build_threshold")

        if project.get_property("pychecker_break_build") and len(warnings) > threshold:
            raise BuildFailedException("Found warnings produced by pychecker")
Beispiel #11
0
def run_cram_tests(project, logger):
    logger.info("Running Cram command line tests")

    cram_tests = list(_find_files(project))
    if not cram_tests or len(cram_tests) == 0:
        if project.get_property("cram_fail_if_no_tests"):
            raise BuildFailedException("No Cram tests found!")
        else:
            return

    command_and_arguments = _cram_command_for(project)
    command_and_arguments.extend(cram_tests)
    report_file = _report_file(project)

    env = os.environ.copy()
    if project.get_property('cram_run_test_from_target'):
        dist_dir = project.expand_path("$dir_dist")
        _prepend_path(env, "PYTHONPATH", dist_dir)
        script_dir_dist = project.get_property('dir_dist_scripts')
        _prepend_path(env, "PATH", os.path.join(dist_dir, script_dir_dist))
    else:
        source_dir = project.expand_path("$dir_source_main_python")
        _prepend_path(env, "PYTHONPATH", source_dir)
        script_dir = project.expand_path('$dir_source_main_scripts')
        _prepend_path(env, "PATH", script_dir)

    return_code = execute_command(command_and_arguments,
                                  report_file,
                                  env=env,
                                  error_file_name=report_file)

    if return_code != 0:
        error_str = "Cram tests failed! See %s for full details:\n%s" % (report_file, tail_log(report_file))
        logger.error(error_str)
        raise BuildFailedException(error_str)

    report = read_file(report_file)
    result = report[-1][2:].strip()
    logger.info("Cram tests were fine")
    logger.info(result)
Beispiel #12
0
def analyze(project, logger):
    """ Applies the flake8 script to the sources of the given project. """
    logger.info("Executing flake8 on project sources.")

    verbose = project.get_property("verbose")
    project.set_property_if_unset("flake8_verbose_output", verbose)

    command_and_arguments = ["flake8"]

    flake8_ignore = project.get_property("flake8_ignore")
    if flake8_ignore is not None:
        ignore_option = "--ignore={0}".format(flake8_ignore)
        command_and_arguments.append(ignore_option)

    max_line_length = project.get_property("flake8_max_line_length")
    command_and_arguments.append("--max-line-length={0}".format(max_line_length))

    exclude_patterns = project.get_property("flake8_exclude_patterns")
    if exclude_patterns:
        command_and_arguments.append("--exclude={0}".format(exclude_patterns))

    include_test_sources = project.get_property("flake8_include_test_sources")

    execution_result = execute_tool_on_source_files(project=project,
                                                    name="flake8",
                                                    command_and_arguments=command_and_arguments,
                                                    logger=logger,
                                                    include_test_sources=include_test_sources)

    report_file = execution_result[1]
    report_lines = read_file(report_file)
    count_of_warnings = len(report_lines)

    if count_of_warnings > 0:
        if project.get_property("flake8_break_build"):
            error_message = "flake8 found {0} warning(s)".format(count_of_warnings)
            raise BuildFailedException(error_message)
        else:
            logger.warn("flake8 found %d warning(s).", count_of_warnings)
Beispiel #13
0
def _install_external_plugin(name, version, logger, plugin_module_name):
    if not name.startswith(PYPI_PLUGIN_PROTOCOL) and not name.startswith(VCS_PLUGIN_PROTOCOL):
        message = "Only plugins starting with '{0}' are currently supported"
        raise MissingPluginException(name, message.format((PYPI_PLUGIN_PROTOCOL, VCS_PLUGIN_PROTOCOL)))

    if name.startswith(PYPI_PLUGIN_PROTOCOL):
        pip_package = name.replace(PYPI_PLUGIN_PROTOCOL, "")
        if version:
            pip_package += str(version)
    elif name.startswith(VCS_PLUGIN_PROTOCOL):
        pip_package = name.replace(VCS_PLUGIN_PROTOCOL, "")

    with tempfile.NamedTemporaryFile(delete=False) as log_file:
        log_file_name = log_file.name
        install_cmd = ['pip', 'install', '--upgrade', pip_package]
        result = execute_command(install_cmd,
                                 log_file_name,
                                 error_file_name=log_file_name,
                                 cwd=".",
                                 shell=False)
        if result != 0:
            logger.error("The following pip error was encountered:\n" + "".join(read_file(log_file_name)))
            message = "Failed to install plugin from {0}".format(pip_package)
            raise MissingPluginException(name, message)
Beispiel #14
0
def filter_resource(absolute_file_name, relative_file_name, dictionary,
                    logger):
    logger.debug("Filtering resource %s", absolute_file_name)
    content = "".join(read_file(absolute_file_name))
    filtered = string.Template(content).safe_substitute(dictionary)
    write_file(absolute_file_name, filtered)
Beispiel #15
0
def publish_coverage(project, logger):

    coverage_file = _coverage_file(project)
    coverage_json = read_file(coverage_file)
    coverage = json.loads(''.join(coverage_json))['overall_coverage']
    logger.info('Overall coverage: {0}'.format(coverage))
def execute_pylint(project, logger):
    """
    Collect all source files and Pylint options according properties.
    Call Pylint after that and parse statistic.

    :param project: PyBuilder project object
    :param logger: PyBuilder project logger
    """
    logger.info("Executing pylint on project sources.")
    project.set_property_if_unset("pylint_verbose_output",
                                  project.get_property("verbose"))
    # add max line length
    pylint_args = ["--max-line-length=%s"
                   % project.get_property("pylint_max_line_length")]
    # add ignored messages
    if project.get_property("pylint_ignore"):
        for ignore in project.get_property("pylint_ignore"):
            pylint_args.append("--disable=%s" % ignore)
    # add ignore pattern
    if project.get_property("pylint_exclude_patterns"):
        pylint_args.append(
            "--ignore-patterns=%s"
            % project.get_property("pylint_exclude_patterns"))
    # add extra arguments
    pylint_args.extend(project.get_property("pylint_extra_args"))
    # collect files list
    files = python_plugin_helper.discover_affected_files(
        project.get_property("pylint_include_test_sources"),
        project.get_property("pylint_include_scripts"), project)
    # collect additionally included files
    included_files = [project.expand_path(file_name)
                      for file_name
                      in project.get_property("pylint_include_files")]
    # add files to arguments
    pylint_args = ([file_name for file_name in files]
                   + included_files + pylint_args)
    logger.debug("Calling pylint with: %s", pylint_args)
    # replace stdout/stderr with report files
    prev_stdout, prev_stderr = sys.stdout, sys.stderr
    report_file = project.expand_path("$dir_reports/pylint")
    sys.stdout = open(report_file, 'w')
    sys.stderr = open(project.expand_path("$dir_reports/pylint.err"), 'w')
    score, fatal, error, warning, refactor, convention = _run_pylint(
        pylint_args)
    # return original stdout/stderr
    sys.stdout.close()
    sys.stderr.close()
    sys.stdout, sys.stderr = prev_stdout, prev_stderr
    # write result to logger
    logger.info("Pylint results: score:%s, fatal:%s, error:%s, "
                "warning:%s, refactor:%s, convention:%s" %
                (score, fatal, error, warning, refactor, convention))
    if (project.get_property("pylint_verbose_output")
            and fatal + error + warning + refactor + convention > 0):
        pybuilder_python_plugin_helper.log_report(
            logger, "pylint", utils.read_file(report_file))
    # (C) convention, for programming standard violation
    # (R) refactor, for bad code smell
    # (W) warning, for python specific problems
    # (E) error, for much probably bugs in the code
    # (F) fatal, if an error occurred which prevented pylint from doing
    # errors are errors: break build in any case
    if fatal + error > 0:
        raise BuildFailedException(
            "pylint found %s fatal(s) and %s error(s)" % (fatal, error))
    # work with other types
    if (warning + refactor + convention > 0 and
            project.get_property("pylint_break_build")):
        raise BuildFailedException(
            "pylint found %s warning(s), %s refactor(s) and %s convention(s)"
            % (warning, refactor, convention))
    if (project.get_property("pylint_score_threshold")
            and score < project.get_property("pylint_score_threshold")):
        raise BuildFailedException(
            "pylint current score %s less then threshold %s"
            % (score, project.get_property("pylint_score_threshold")))
def filter_resource(absolute_file_name, relative_file_name, dict, logger):
    logger.debug("Filtering resource %s", absolute_file_name)
    content = "".join(read_file(absolute_file_name))
    filtered = string.Template(content).safe_substitute(dict)
    write_file(absolute_file_name, filtered)
def run_single_test(logger,
                    project,
                    reactor,
                    reports_dir,
                    test,
                    output_test_names=True):
    additional_integrationtest_commandline_text = project.get_property(
        "integrationtest_additional_commandline")

    if additional_integrationtest_commandline_text:
        additional_integrationtest_commandline = tuple(
            additional_integrationtest_commandline_text.split(" "))
    else:
        additional_integrationtest_commandline = ()

    name, _ = os.path.splitext(os.path.basename(test))

    if output_test_names:
        logger.info("Running integration test %s", name)

    python_env = reactor.python_env_registry[project.get_property(
        "integrationtest_python_env")]
    env = prepare_environment(project)
    command_and_arguments = python_env.executable + [test]
    command_and_arguments += additional_integrationtest_commandline

    report_file_name = os.path.join(reports_dir, name)
    error_file_name = report_file_name + ".err"

    test_time = Timer.start()
    return_code = python_env.execute_command(
        command_and_arguments,
        report_file_name,
        env,
        error_file_name=error_file_name,
        inherit_env=project.get_property(
            "integrationtest_inherit_environment"))
    test_time.stop()
    report_item = {
        "test": name,
        "test_file": test,
        "time": test_time.get_millis(),
        "success": True
    }
    if return_code != 0:
        logger.error("Integration test failed: %s, exit code %d", test,
                     return_code)
        report_item["success"] = False

        if project.get_property("verbose") or project.get_property(
                "integrationtest_always_verbose"):
            print_file_content(report_file_name)
            print_text_line()
            print_file_content(error_file_name)
            report_item["exception"] = ''.join(
                read_file(error_file_name)).replace('\'', '')
    elif project.get_property("integrationtest_always_verbose"):
        print_file_content(report_file_name)
        print_text_line()
        print_file_content(error_file_name)

    return report_item