def run(self, outfile_name): error_file_name = "{0}.err".format(outfile_name) return_code = execute_command(self.parts, outfile_name) error_file_lines = read_file(error_file_name) outfile_lines = read_file(outfile_name) return ExternalCommandResult(return_code, outfile_name, outfile_lines, error_file_name, error_file_lines)
def run_on_production_source_files(self, logger, include_test_sources=False, include_scripts=False): execution_result = execute_tool_on_source_files(project=self.project, name=self.command_name, command_and_arguments=self.parts, include_test_sources=include_test_sources, include_scripts=include_scripts, logger=logger) exit_code, report_file = execution_result report_lines = read_file(report_file) error_report_file = '{0}.err'.format(report_file) # TODO @mriehl not dry, execute_tool... should return this error_report_lines = read_file(error_report_file) return ExternalCommandResult(exit_code, report_file, report_lines, error_report_file, error_report_lines)
def run_single_test(logger, project, reports_dir, test, output_test_names=True): name, _ = os.path.splitext(os.path.basename(test)) if output_test_names: logger.info("Running integration test %s", name) env = prepare_environment(project) test_time = Timer.start() command_and_arguments = (sys.executable, test) report_file_name = os.path.join(reports_dir, name) error_file_name = report_file_name + ".err" return_code = execute_command( command_and_arguments, report_file_name, env, error_file_name=error_file_name) test_time.stop() report_item = { "test": name, "test_file": test, "time": test_time.get_millis(), "success": True } if return_code != 0: logger.error("Integration test failed: %s", test) report_item["success"] = False if project.get_property("verbose"): print_file_content(report_file_name) print_text_line() print_file_content(error_file_name) report_item['exception'] = ''.join(read_file(error_file_name)).replace('\'', '') return report_item
def _install_external_plugin(project, name, version, logger, plugin_module_name, upgrade=False, force_reinstall=False): if not name.startswith(PYPI_PLUGIN_PROTOCOL) and not name.startswith(VCS_PLUGIN_PROTOCOL): message = "Only plugins starting with '{0}' are currently supported" raise MissingPluginException(name, message.format((PYPI_PLUGIN_PROTOCOL, VCS_PLUGIN_PROTOCOL))) if name.startswith(PYPI_PLUGIN_PROTOCOL): pip_package = name.replace(PYPI_PLUGIN_PROTOCOL, "") if version: pip_package += str(version) upgrade = True elif name.startswith(VCS_PLUGIN_PROTOCOL): pip_package = name.replace(VCS_PLUGIN_PROTOCOL, "") force_reinstall = True with tempfile.NamedTemporaryFile(delete=True) as log_file: log_file_name = log_file.name result = pip_install(pip_package, index_url=project.get_property("install_dependencies_index_url"), extra_index_url=project.get_property("install_dependencies_extra_index_url"), trusted_host=project.get_property("install_dependencies_trusted_host"), upgrade=upgrade, force_reinstall=force_reinstall, logger=logger, outfile_name=log_file_name, error_file_name=log_file_name, cwd=".") if result != 0: logger.error("The following pip error was encountered:\n" + "".join(read_file(log_file_name))) message = "Failed to install plugin from {0}".format(pip_package) raise MissingPluginException(name, message)
def run_cram_tests(project, logger): logger.info("Running Cram command line tests") command_and_arguments = _cram_command_for(project) command_and_arguments.extend(_find_files(project)) report_file = _report_file(project) env = os.environ.copy() source_dir = project.expand_path("$dir_source_main_python") _prepend_path(env, "PYTHONPATH", source_dir) script_dir = project.expand_path('$dir_source_main_scripts') _prepend_path(env, "PATH", script_dir) return_code = execute_command(command_and_arguments, report_file, env=env, error_file_name=report_file) report = read_file(report_file) result = report[-1][2:].strip() if return_code != 0: logger.error("Cram tests failed!") if project.get_property("verbose"): for line in report: logger.error(line.rstrip()) else: logger.error(result) logger.error("See: '{0}' for details".format(report_file)) raise BuildFailedException("Cram tests failed!") logger.info("Cram tests were fine") logger.info(result)
def analyze(project, logger): logger.info("Executing pep8 on project sources") _, report_file = execute_tool_on_source_files(project, "pep8", ["pep8"]) reports = read_file(report_file) if len(reports) > 0: logger.warn("Found %d warning%s produced by pep8", len(reports), "" if len(reports) == 1 else "s")
def _install_external_plugin(name, logger): if not name.startswith(PYPI_PLUGIN_PROTOCOL): message = "Only plugins starting with '{0}' are currently supported" raise MissingPluginException(name, message.format(PYPI_PLUGIN_PROTOCOL)) plugin_name_on_pypi = name.replace(PYPI_PLUGIN_PROTOCOL, "") log_file = tempfile.NamedTemporaryFile(delete=False).name result = execute_command( "pip install {0}".format(plugin_name_on_pypi), log_file, error_file_name=log_file, shell=True ) if result != 0: logger.error("The following pip error was encountered:\n" + "".join(read_file(log_file))) message = "Failed to install from PyPI".format(plugin_name_on_pypi) raise MissingPluginException(name, message)
def execute_tool_on_source_files(project, name, command_and_arguments, logger=None, include_test_sources=False): files = discover_affected_files(include_test_sources, project) command = as_list(command_and_arguments) + [f for f in files] report_file = project.expand_path("$dir_reports/{0}".format(name)) execution_result = execute_command(command, report_file), report_file report_file = execution_result[1] report_lines = read_file(report_file) if project.get_property(name + "_verbose_output") and logger: log_report(logger, name, report_lines) return execution_result
def execute_pylint(project, logger): logger.info("Executing pylint on project sources") command_and_arguments = ["pylint"] + project.get_property("pylint_options") result_tuple = execute_tool_on_modules(project, "pylint", command_and_arguments, True) if project.get_property("pylint_break_build"): report_file = result_tuple[1] # access by position to avoid changing mocking behaviour warnings = [line for line in read_file(report_file) if line.find('.py:') >= 0] warning_count = len(warnings) if warning_count > 0: message = "Pylint found {} warning(s).".format(warning_count) logger.error(message) raise BuildFailedException(message)
def execute_pychecker(project, logger): command_line = build_command_line(project) logger.info("Executing pychecker on project sources: %s" % (' '.join(command_line))) _, report_file = execute_tool_on_modules(project, "pychecker", command_line, True) warnings = read_file(report_file) report = parse_pychecker_output(project, warnings) project.write_report("pychecker.json", render_report(report.to_json_dict())) if len(warnings) != 0: logger.warn("Found %d warning%s produced by pychecker. See %s for details.", len(warnings), "s" if len(warnings) != 1 else "", report_file) threshold = project.get_property("pychecker_break_build_threshold") if project.get_property("pychecker_break_build") and len(warnings) > threshold: raise BuildFailedException("Found warnings produced by pychecker")
def run_cram_tests(project, logger): logger.info("Running Cram command line tests") cram_tests = list(_find_files(project)) if not cram_tests or len(cram_tests) == 0: if project.get_property("cram_fail_if_no_tests"): raise BuildFailedException("No Cram tests found!") else: return command_and_arguments = _cram_command_for(project) command_and_arguments.extend(cram_tests) report_file = _report_file(project) env = os.environ.copy() if project.get_property('cram_run_test_from_target'): dist_dir = project.expand_path("$dir_dist") _prepend_path(env, "PYTHONPATH", dist_dir) script_dir_dist = project.get_property('dir_dist_scripts') _prepend_path(env, "PATH", os.path.join(dist_dir, script_dir_dist)) else: source_dir = project.expand_path("$dir_source_main_python") _prepend_path(env, "PYTHONPATH", source_dir) script_dir = project.expand_path('$dir_source_main_scripts') _prepend_path(env, "PATH", script_dir) return_code = execute_command(command_and_arguments, report_file, env=env, error_file_name=report_file) if return_code != 0: error_str = "Cram tests failed! See %s for full details:\n%s" % (report_file, tail_log(report_file)) logger.error(error_str) raise BuildFailedException(error_str) report = read_file(report_file) result = report[-1][2:].strip() logger.info("Cram tests were fine") logger.info(result)
def analyze(project, logger): """ Applies the flake8 script to the sources of the given project. """ logger.info("Executing flake8 on project sources.") verbose = project.get_property("verbose") project.set_property_if_unset("flake8_verbose_output", verbose) command_and_arguments = ["flake8"] flake8_ignore = project.get_property("flake8_ignore") if flake8_ignore is not None: ignore_option = "--ignore={0}".format(flake8_ignore) command_and_arguments.append(ignore_option) max_line_length = project.get_property("flake8_max_line_length") command_and_arguments.append("--max-line-length={0}".format(max_line_length)) exclude_patterns = project.get_property("flake8_exclude_patterns") if exclude_patterns: command_and_arguments.append("--exclude={0}".format(exclude_patterns)) include_test_sources = project.get_property("flake8_include_test_sources") execution_result = execute_tool_on_source_files(project=project, name="flake8", command_and_arguments=command_and_arguments, logger=logger, include_test_sources=include_test_sources) report_file = execution_result[1] report_lines = read_file(report_file) count_of_warnings = len(report_lines) if count_of_warnings > 0: if project.get_property("flake8_break_build"): error_message = "flake8 found {0} warning(s)".format(count_of_warnings) raise BuildFailedException(error_message) else: logger.warn("flake8 found %d warning(s).", count_of_warnings)
def _install_external_plugin(name, version, logger, plugin_module_name): if not name.startswith(PYPI_PLUGIN_PROTOCOL) and not name.startswith(VCS_PLUGIN_PROTOCOL): message = "Only plugins starting with '{0}' are currently supported" raise MissingPluginException(name, message.format((PYPI_PLUGIN_PROTOCOL, VCS_PLUGIN_PROTOCOL))) if name.startswith(PYPI_PLUGIN_PROTOCOL): pip_package = name.replace(PYPI_PLUGIN_PROTOCOL, "") if version: pip_package += str(version) elif name.startswith(VCS_PLUGIN_PROTOCOL): pip_package = name.replace(VCS_PLUGIN_PROTOCOL, "") with tempfile.NamedTemporaryFile(delete=False) as log_file: log_file_name = log_file.name install_cmd = ['pip', 'install', '--upgrade', pip_package] result = execute_command(install_cmd, log_file_name, error_file_name=log_file_name, cwd=".", shell=False) if result != 0: logger.error("The following pip error was encountered:\n" + "".join(read_file(log_file_name))) message = "Failed to install plugin from {0}".format(pip_package) raise MissingPluginException(name, message)
def filter_resource(absolute_file_name, relative_file_name, dictionary, logger): logger.debug("Filtering resource %s", absolute_file_name) content = "".join(read_file(absolute_file_name)) filtered = string.Template(content).safe_substitute(dictionary) write_file(absolute_file_name, filtered)
def publish_coverage(project, logger): coverage_file = _coverage_file(project) coverage_json = read_file(coverage_file) coverage = json.loads(''.join(coverage_json))['overall_coverage'] logger.info('Overall coverage: {0}'.format(coverage))
def execute_pylint(project, logger): """ Collect all source files and Pylint options according properties. Call Pylint after that and parse statistic. :param project: PyBuilder project object :param logger: PyBuilder project logger """ logger.info("Executing pylint on project sources.") project.set_property_if_unset("pylint_verbose_output", project.get_property("verbose")) # add max line length pylint_args = ["--max-line-length=%s" % project.get_property("pylint_max_line_length")] # add ignored messages if project.get_property("pylint_ignore"): for ignore in project.get_property("pylint_ignore"): pylint_args.append("--disable=%s" % ignore) # add ignore pattern if project.get_property("pylint_exclude_patterns"): pylint_args.append( "--ignore-patterns=%s" % project.get_property("pylint_exclude_patterns")) # add extra arguments pylint_args.extend(project.get_property("pylint_extra_args")) # collect files list files = python_plugin_helper.discover_affected_files( project.get_property("pylint_include_test_sources"), project.get_property("pylint_include_scripts"), project) # collect additionally included files included_files = [project.expand_path(file_name) for file_name in project.get_property("pylint_include_files")] # add files to arguments pylint_args = ([file_name for file_name in files] + included_files + pylint_args) logger.debug("Calling pylint with: %s", pylint_args) # replace stdout/stderr with report files prev_stdout, prev_stderr = sys.stdout, sys.stderr report_file = project.expand_path("$dir_reports/pylint") sys.stdout = open(report_file, 'w') sys.stderr = open(project.expand_path("$dir_reports/pylint.err"), 'w') score, fatal, error, warning, refactor, convention = _run_pylint( pylint_args) # return original stdout/stderr sys.stdout.close() sys.stderr.close() sys.stdout, sys.stderr = prev_stdout, prev_stderr # write result to logger logger.info("Pylint results: score:%s, fatal:%s, error:%s, " "warning:%s, refactor:%s, convention:%s" % (score, fatal, error, warning, refactor, convention)) if (project.get_property("pylint_verbose_output") and fatal + error + warning + refactor + convention > 0): pybuilder_python_plugin_helper.log_report( logger, "pylint", utils.read_file(report_file)) # (C) convention, for programming standard violation # (R) refactor, for bad code smell # (W) warning, for python specific problems # (E) error, for much probably bugs in the code # (F) fatal, if an error occurred which prevented pylint from doing # errors are errors: break build in any case if fatal + error > 0: raise BuildFailedException( "pylint found %s fatal(s) and %s error(s)" % (fatal, error)) # work with other types if (warning + refactor + convention > 0 and project.get_property("pylint_break_build")): raise BuildFailedException( "pylint found %s warning(s), %s refactor(s) and %s convention(s)" % (warning, refactor, convention)) if (project.get_property("pylint_score_threshold") and score < project.get_property("pylint_score_threshold")): raise BuildFailedException( "pylint current score %s less then threshold %s" % (score, project.get_property("pylint_score_threshold")))
def filter_resource(absolute_file_name, relative_file_name, dict, logger): logger.debug("Filtering resource %s", absolute_file_name) content = "".join(read_file(absolute_file_name)) filtered = string.Template(content).safe_substitute(dict) write_file(absolute_file_name, filtered)
def run_single_test(logger, project, reactor, reports_dir, test, output_test_names=True): additional_integrationtest_commandline_text = project.get_property( "integrationtest_additional_commandline") if additional_integrationtest_commandline_text: additional_integrationtest_commandline = tuple( additional_integrationtest_commandline_text.split(" ")) else: additional_integrationtest_commandline = () name, _ = os.path.splitext(os.path.basename(test)) if output_test_names: logger.info("Running integration test %s", name) python_env = reactor.python_env_registry[project.get_property( "integrationtest_python_env")] env = prepare_environment(project) command_and_arguments = python_env.executable + [test] command_and_arguments += additional_integrationtest_commandline report_file_name = os.path.join(reports_dir, name) error_file_name = report_file_name + ".err" test_time = Timer.start() return_code = python_env.execute_command( command_and_arguments, report_file_name, env, error_file_name=error_file_name, inherit_env=project.get_property( "integrationtest_inherit_environment")) test_time.stop() report_item = { "test": name, "test_file": test, "time": test_time.get_millis(), "success": True } if return_code != 0: logger.error("Integration test failed: %s, exit code %d", test, return_code) report_item["success"] = False if project.get_property("verbose") or project.get_property( "integrationtest_always_verbose"): print_file_content(report_file_name) print_text_line() print_file_content(error_file_name) report_item["exception"] = ''.join( read_file(error_file_name)).replace('\'', '') elif project.get_property("integrationtest_always_verbose"): print_file_content(report_file_name) print_text_line() print_file_content(error_file_name) return report_item