def run(self, outfile_name):
error_file_name = "{0}.err".format(outfile_name)
return_code = execute_command(self.parts, outfile_name)
error_file_lines = read_file(error_file_name)
outfile_lines = read_file(outfile_name)
return ExternalCommandResult(return_code, outfile_name, outfile_lines, error_file_name, error_file_lines)
def run_on_production_source_files(self, logger, include_test_sources=False, include_scripts=False):
execution_result = execute_tool_on_source_files(project=self.project,
name=self.command_name,
command_and_arguments=self.parts,
include_test_sources=include_test_sources,
include_scripts=include_scripts,
logger=logger)
exit_code, report_file = execution_result
report_lines = read_file(report_file)
error_report_file = '{0}.err'.format(report_file) # TODO @mriehl not dry, execute_tool... should return this
error_report_lines = read_file(error_report_file)
return ExternalCommandResult(exit_code, report_file, report_lines, error_report_file, error_report_lines)
def run_single_test(logger, project, reports_dir, test, output_test_names=True):
name, _ = os.path.splitext(os.path.basename(test))
if output_test_names:
logger.info("Running integration test %s", name)
env = prepare_environment(project)
test_time = Timer.start()
command_and_arguments = (sys.executable, test)
report_file_name = os.path.join(reports_dir, name)
error_file_name = report_file_name + ".err"
return_code = execute_command(
command_and_arguments, report_file_name, env, error_file_name=error_file_name)
test_time.stop()
report_item = {
"test": name,
"test_file": test,
"time": test_time.get_millis(),
"success": True
}
if return_code != 0:
logger.error("Integration test failed: %s", test)
report_item["success"] = False
if project.get_property("verbose"):
print_file_content(report_file_name)
print_text_line()
print_file_content(error_file_name)
report_item['exception'] = ''.join(read_file(error_file_name)).replace('\'', '')
return report_item
def _install_external_plugin(project, name, version, logger, plugin_module_name, upgrade=False, force_reinstall=False):
if not name.startswith(PYPI_PLUGIN_PROTOCOL) and not name.startswith(VCS_PLUGIN_PROTOCOL):
message = "Only plugins starting with '{0}' are currently supported"
raise MissingPluginException(name, message.format((PYPI_PLUGIN_PROTOCOL, VCS_PLUGIN_PROTOCOL)))
if name.startswith(PYPI_PLUGIN_PROTOCOL):
pip_package = name.replace(PYPI_PLUGIN_PROTOCOL, "")
if version:
pip_package += str(version)
upgrade = True
elif name.startswith(VCS_PLUGIN_PROTOCOL):
pip_package = name.replace(VCS_PLUGIN_PROTOCOL, "")
force_reinstall = True
with tempfile.NamedTemporaryFile(delete=True) as log_file:
log_file_name = log_file.name
result = pip_install(pip_package,
index_url=project.get_property("install_dependencies_index_url"),
extra_index_url=project.get_property("install_dependencies_extra_index_url"),
trusted_host=project.get_property("install_dependencies_trusted_host"),
upgrade=upgrade,
force_reinstall=force_reinstall,
logger=logger,
outfile_name=log_file_name,
error_file_name=log_file_name,
cwd=".")
if result != 0:
logger.error("The following pip error was encountered:\n" + "".join(read_file(log_file_name)))
message = "Failed to install plugin from {0}".format(pip_package)
raise MissingPluginException(name, message)
def run_cram_tests(project, logger):
logger.info("Running Cram command line tests")
command_and_arguments = _cram_command_for(project)
command_and_arguments.extend(_find_files(project))
report_file = _report_file(project)
env = os.environ.copy()
source_dir = project.expand_path("$dir_source_main_python")
_prepend_path(env, "PYTHONPATH", source_dir)
script_dir = project.expand_path('$dir_source_main_scripts')
_prepend_path(env, "PATH", script_dir)
return_code = execute_command(command_and_arguments,
report_file,
env=env,
error_file_name=report_file)
report = read_file(report_file)
result = report[-1][2:].strip()
if return_code != 0:
logger.error("Cram tests failed!")
if project.get_property("verbose"):
for line in report:
logger.error(line.rstrip())
else:
logger.error(result)
logger.error("See: '{0}' for details".format(report_file))
raise BuildFailedException("Cram tests failed!")
logger.info("Cram tests were fine")
logger.info(result)
def analyze(project, logger):
logger.info("Executing pep8 on project sources")
_, report_file = execute_tool_on_source_files(project, "pep8", ["pep8"])
reports = read_file(report_file)
if len(reports) > 0:
logger.warn("Found %d warning%s produced by pep8",
len(reports), "" if len(reports) == 1 else "s")
def _install_external_plugin(name, logger):
if not name.startswith(PYPI_PLUGIN_PROTOCOL):
message = "Only plugins starting with '{0}' are currently supported"
raise MissingPluginException(name, message.format(PYPI_PLUGIN_PROTOCOL))
plugin_name_on_pypi = name.replace(PYPI_PLUGIN_PROTOCOL, "")
log_file = tempfile.NamedTemporaryFile(delete=False).name
result = execute_command(
"pip install {0}".format(plugin_name_on_pypi), log_file, error_file_name=log_file, shell=True
)
if result != 0:
logger.error("The following pip error was encountered:\n" + "".join(read_file(log_file)))
message = "Failed to install from PyPI".format(plugin_name_on_pypi)
raise MissingPluginException(name, message)
def execute_tool_on_source_files(project, name, command_and_arguments, logger=None, include_test_sources=False):
files = discover_affected_files(include_test_sources, project)
command = as_list(command_and_arguments) + [f for f in files]
report_file = project.expand_path("$dir_reports/{0}".format(name))
execution_result = execute_command(command, report_file), report_file
report_file = execution_result[1]
report_lines = read_file(report_file)
if project.get_property(name + "_verbose_output") and logger:
log_report(logger, name, report_lines)
return execution_result
def execute_pylint(project, logger):
logger.info("Executing pylint on project sources")
command_and_arguments = ["pylint"] + project.get_property("pylint_options")
result_tuple = execute_tool_on_modules(project, "pylint", command_and_arguments, True)
if project.get_property("pylint_break_build"):
report_file = result_tuple[1] # access by position to avoid changing mocking behaviour
warnings = [line
for line in read_file(report_file)
if line.find('.py:') >= 0]
warning_count = len(warnings)
if warning_count > 0:
message = "Pylint found {} warning(s).".format(warning_count)
logger.error(message)
raise BuildFailedException(message)
def execute_pychecker(project, logger):
command_line = build_command_line(project)
logger.info("Executing pychecker on project sources: %s" % (' '.join(command_line)))
_, report_file = execute_tool_on_modules(project, "pychecker", command_line, True)
warnings = read_file(report_file)
report = parse_pychecker_output(project, warnings)
project.write_report("pychecker.json", render_report(report.to_json_dict()))
if len(warnings) != 0:
logger.warn("Found %d warning%s produced by pychecker. See %s for details.",
len(warnings),
"s" if len(warnings) != 1 else "",
report_file)
threshold = project.get_property("pychecker_break_build_threshold")
if project.get_property("pychecker_break_build") and len(warnings) > threshold:
raise BuildFailedException("Found warnings produced by pychecker")
def run_cram_tests(project, logger):
logger.info("Running Cram command line tests")
cram_tests = list(_find_files(project))
if not cram_tests or len(cram_tests) == 0:
if project.get_property("cram_fail_if_no_tests"):
raise BuildFailedException("No Cram tests found!")
else:
return
command_and_arguments = _cram_command_for(project)
command_and_arguments.extend(cram_tests)
report_file = _report_file(project)
env = os.environ.copy()
if project.get_property('cram_run_test_from_target'):
dist_dir = project.expand_path("$dir_dist")
_prepend_path(env, "PYTHONPATH", dist_dir)
script_dir_dist = project.get_property('dir_dist_scripts')
_prepend_path(env, "PATH", os.path.join(dist_dir, script_dir_dist))
else:
source_dir = project.expand_path("$dir_source_main_python")
_prepend_path(env, "PYTHONPATH", source_dir)
script_dir = project.expand_path('$dir_source_main_scripts')
_prepend_path(env, "PATH", script_dir)
return_code = execute_command(command_and_arguments,
report_file,
env=env,
error_file_name=report_file)
if return_code != 0:
error_str = "Cram tests failed! See %s for full details:\n%s" % (report_file, tail_log(report_file))
logger.error(error_str)
raise BuildFailedException(error_str)
report = read_file(report_file)
result = report[-1][2:].strip()
logger.info("Cram tests were fine")
logger.info(result)
def analyze(project, logger):
""" Applies the flake8 script to the sources of the given project. """
logger.info("Executing flake8 on project sources.")
verbose = project.get_property("verbose")
project.set_property_if_unset("flake8_verbose_output", verbose)
command_and_arguments = ["flake8"]
flake8_ignore = project.get_property("flake8_ignore")
if flake8_ignore is not None:
ignore_option = "--ignore={0}".format(flake8_ignore)
command_and_arguments.append(ignore_option)
max_line_length = project.get_property("flake8_max_line_length")
command_and_arguments.append("--max-line-length={0}".format(max_line_length))
exclude_patterns = project.get_property("flake8_exclude_patterns")
if exclude_patterns:
command_and_arguments.append("--exclude={0}".format(exclude_patterns))
include_test_sources = project.get_property("flake8_include_test_sources")
execution_result = execute_tool_on_source_files(project=project,
name="flake8",
command_and_arguments=command_and_arguments,
logger=logger,
include_test_sources=include_test_sources)
report_file = execution_result[1]
report_lines = read_file(report_file)
count_of_warnings = len(report_lines)
if count_of_warnings > 0:
if project.get_property("flake8_break_build"):
error_message = "flake8 found {0} warning(s)".format(count_of_warnings)
raise BuildFailedException(error_message)
else:
logger.warn("flake8 found %d warning(s).", count_of_warnings)
def _install_external_plugin(name, version, logger, plugin_module_name):
if not name.startswith(PYPI_PLUGIN_PROTOCOL) and not name.startswith(VCS_PLUGIN_PROTOCOL):
message = "Only plugins starting with '{0}' are currently supported"
raise MissingPluginException(name, message.format((PYPI_PLUGIN_PROTOCOL, VCS_PLUGIN_PROTOCOL)))
if name.startswith(PYPI_PLUGIN_PROTOCOL):
pip_package = name.replace(PYPI_PLUGIN_PROTOCOL, "")
if version:
pip_package += str(version)
elif name.startswith(VCS_PLUGIN_PROTOCOL):
pip_package = name.replace(VCS_PLUGIN_PROTOCOL, "")
with tempfile.NamedTemporaryFile(delete=False) as log_file:
log_file_name = log_file.name
install_cmd = ['pip', 'install', '--upgrade', pip_package]
result = execute_command(install_cmd,
log_file_name,
error_file_name=log_file_name,
cwd=".",
shell=False)
if result != 0:
logger.error("The following pip error was encountered:\n" + "".join(read_file(log_file_name)))
message = "Failed to install plugin from {0}".format(pip_package)
raise MissingPluginException(name, message)
def filter_resource(absolute_file_name, relative_file_name, dictionary,
logger):
logger.debug("Filtering resource %s", absolute_file_name)
content = "".join(read_file(absolute_file_name))
filtered = string.Template(content).safe_substitute(dictionary)
write_file(absolute_file_name, filtered)
def publish_coverage(project, logger):
coverage_file = _coverage_file(project)
coverage_json = read_file(coverage_file)
coverage = json.loads(''.join(coverage_json))['overall_coverage']
logger.info('Overall coverage: {0}'.format(coverage))
def execute_pylint(project, logger):
"""
Collect all source files and Pylint options according properties.
Call Pylint after that and parse statistic.
:param project: PyBuilder project object
:param logger: PyBuilder project logger
"""
logger.info("Executing pylint on project sources.")
project.set_property_if_unset("pylint_verbose_output",
project.get_property("verbose"))
# add max line length
pylint_args = ["--max-line-length=%s"
% project.get_property("pylint_max_line_length")]
# add ignored messages
if project.get_property("pylint_ignore"):
for ignore in project.get_property("pylint_ignore"):
pylint_args.append("--disable=%s" % ignore)
# add ignore pattern
if project.get_property("pylint_exclude_patterns"):
pylint_args.append(
"--ignore-patterns=%s"
% project.get_property("pylint_exclude_patterns"))
# add extra arguments
pylint_args.extend(project.get_property("pylint_extra_args"))
# collect files list
files = python_plugin_helper.discover_affected_files(
project.get_property("pylint_include_test_sources"),
project.get_property("pylint_include_scripts"), project)
# collect additionally included files
included_files = [project.expand_path(file_name)
for file_name
in project.get_property("pylint_include_files")]
# add files to arguments
pylint_args = ([file_name for file_name in files]
+ included_files + pylint_args)
logger.debug("Calling pylint with: %s", pylint_args)
# replace stdout/stderr with report files
prev_stdout, prev_stderr = sys.stdout, sys.stderr
report_file = project.expand_path("$dir_reports/pylint")
sys.stdout = open(report_file, 'w')
sys.stderr = open(project.expand_path("$dir_reports/pylint.err"), 'w')
score, fatal, error, warning, refactor, convention = _run_pylint(
pylint_args)
# return original stdout/stderr
sys.stdout.close()
sys.stderr.close()
sys.stdout, sys.stderr = prev_stdout, prev_stderr
# write result to logger
logger.info("Pylint results: score:%s, fatal:%s, error:%s, "
"warning:%s, refactor:%s, convention:%s" %
(score, fatal, error, warning, refactor, convention))
if (project.get_property("pylint_verbose_output")
and fatal + error + warning + refactor + convention > 0):
pybuilder_python_plugin_helper.log_report(
logger, "pylint", utils.read_file(report_file))
# (C) convention, for programming standard violation
# (R) refactor, for bad code smell
# (W) warning, for python specific problems
# (E) error, for much probably bugs in the code
# (F) fatal, if an error occurred which prevented pylint from doing
# errors are errors: break build in any case
if fatal + error > 0:
raise BuildFailedException(
"pylint found %s fatal(s) and %s error(s)" % (fatal, error))
# work with other types
if (warning + refactor + convention > 0 and
project.get_property("pylint_break_build")):
raise BuildFailedException(
"pylint found %s warning(s), %s refactor(s) and %s convention(s)"
% (warning, refactor, convention))
if (project.get_property("pylint_score_threshold")
and score < project.get_property("pylint_score_threshold")):
raise BuildFailedException(
"pylint current score %s less then threshold %s"
% (score, project.get_property("pylint_score_threshold")))
def filter_resource(absolute_file_name, relative_file_name, dict, logger):
logger.debug("Filtering resource %s", absolute_file_name)
content = "".join(read_file(absolute_file_name))
filtered = string.Template(content).safe_substitute(dict)
write_file(absolute_file_name, filtered)
def run_single_test(logger,
project,
reactor,
reports_dir,
test,
output_test_names=True):
additional_integrationtest_commandline_text = project.get_property(
"integrationtest_additional_commandline")
if additional_integrationtest_commandline_text:
additional_integrationtest_commandline = tuple(
additional_integrationtest_commandline_text.split(" "))
else:
additional_integrationtest_commandline = ()
name, _ = os.path.splitext(os.path.basename(test))
if output_test_names:
logger.info("Running integration test %s", name)
python_env = reactor.python_env_registry[project.get_property(
"integrationtest_python_env")]
env = prepare_environment(project)
command_and_arguments = python_env.executable + [test]
command_and_arguments += additional_integrationtest_commandline
report_file_name = os.path.join(reports_dir, name)
error_file_name = report_file_name + ".err"
test_time = Timer.start()
return_code = python_env.execute_command(
command_and_arguments,
report_file_name,
env,
error_file_name=error_file_name,
inherit_env=project.get_property(
"integrationtest_inherit_environment"))
test_time.stop()
report_item = {
"test": name,
"test_file": test,
"time": test_time.get_millis(),
"success": True
}
if return_code != 0:
logger.error("Integration test failed: %s, exit code %d", test,
return_code)
report_item["success"] = False
if project.get_property("verbose") or project.get_property(
"integrationtest_always_verbose"):
print_file_content(report_file_name)
print_text_line()
print_file_content(error_file_name)
report_item["exception"] = ''.join(
read_file(error_file_name)).replace('\'', '')
elif project.get_property("integrationtest_always_verbose"):
print_file_content(report_file_name)
print_text_line()
print_file_content(error_file_name)
return report_item