def analyze(project, logger):
""" Applies the flake8 script to the sources of the given project. """
logger.info("Executing flake8 on project sources.")
verbose = project.get_property("verbose")
project.set_property_if_unset("flake8_verbose_output", verbose)
command = ExternalCommandBuilder('flake8', project)
command.use_argument('--ignore={0}').formatted_with_truthy_property('flake8_ignore')
command.use_argument('--max-line-length={0}').formatted_with_property('flake8_max_line_length')
command.use_argument('--filename={0}').formatted_with_truthy_property('flake8_include_patterns')
command.use_argument('--exclude={0}').formatted_with_truthy_property('flake8_exclude_patterns')
command.use_argument('--max-complexity={0}').formatted_with_truthy_property('flake8_max_complexity')
include_test_sources = project.get_property("flake8_include_test_sources")
include_scripts = project.get_property("flake8_include_scripts")
result = command.run_on_production_source_files(logger,
include_test_sources=include_test_sources,
include_scripts=include_scripts,
include_dirs_only=True)
count_of_warnings = len(result.report_lines)
count_of_errors = len(result.error_report_lines)
if count_of_errors > 0:
logger.error('Errors while running flake8. See %s for full details:\n%s' % (
result.error_report_file, tail_log(result.error_report_file)))
if count_of_warnings > 0:
if project.get_property("flake8_break_build"):
error_message = "flake8 found {0} warning(s)".format(count_of_warnings)
raise BuildFailedException(error_message)
else:
logger.warn("flake8 found %d warning(s).", count_of_warnings)
def _do_install_dependency(logger, project, dependency, upgrade, eager_upgrade,
force_reinstall, constraint_file, target_dir, log_file):
batch = isinstance(dependency, collections.Iterable)
exit_code = pip_utils.pip_install(
install_targets=dependency,
index_url=project.get_property("install_dependencies_index_url"),
extra_index_url=project.get_property("install_dependencies_extra_index_url"),
upgrade=upgrade,
insecure_installs=project.get_property("install_dependencies_insecure_installation"),
force_reinstall=force_reinstall,
target_dir=target_dir,
verbose=project.get_property("verbose"),
trusted_host=project.get_property("install_dependencies_trusted_host"),
constraint_file=constraint_file,
eager_upgrade=eager_upgrade,
logger=logger,
outfile_name=log_file)
if exit_code != 0:
if batch:
dependency_name = " batch dependencies."
else:
dependency_name = " dependency '%s'." % dependency.name
raise BuildFailedException("Unable to install%s See %s for full details:\n%s",
dependency_name,
log_file,
tail_log(log_file))
def analyze(project, logger):
""" Applies the frosted script to the sources of the given project. """
logger.info("Executing frosted on project sources.")
verbose = project.get_property("verbose")
project.set_property_if_unset("frosted_verbose_output", verbose)
command = ExternalCommandBuilder("frosted", project)
for ignored_error_code in project.get_property("frosted_ignore", []):
command.use_argument("--ignore={0}".format(ignored_error_code))
include_test_sources = project.get_property("frosted_include_test_sources")
include_scripts = project.get_property("frosted_include_scripts")
result = command.run_on_production_source_files(
logger,
include_test_sources=include_test_sources,
include_scripts=include_scripts)
count_of_warnings = len(result.report_lines)
count_of_errors = len(result.error_report_lines)
if count_of_errors > 0:
logger.error(
"Errors while running frosted. See %s for full details:\n%s" %
(result.error_report_file, tail_log(result.error_report_file)))
if count_of_warnings > 0:
if project.get_property("frosted_break_build"):
error_message = "frosted found {0} warning(s)".format(
count_of_warnings)
raise BuildFailedException(error_message)
else:
logger.warn("frosted found %d warning(s).", count_of_warnings)
def execute_distutils(project, logger, distutils_commands, clean=False):
reports_dir = _prepare_reports_dir(project)
setup_script = project.expand_path("$dir_dist", "setup.py")
for command in distutils_commands:
logger.debug("Executing distutils command %s", command)
if is_string(command):
out_file = os.path.join(reports_dir, safe_log_file_name(command))
else:
out_file = os.path.join(reports_dir, safe_log_file_name("__".join(command)))
with open(out_file, "w") as out_f:
commands = [sys.executable, setup_script]
if project.get_property("verbose"):
commands.append("-v")
if clean:
commands.extend(["clean", "--all"])
if is_string(command):
commands.extend(command.split())
else:
commands.extend(command)
return_code = _run_process_and_wait(commands, project.expand_path("$dir_dist"), out_f)
if return_code != 0:
raise BuildFailedException(
"Error while executing setup command %s. See %s for full details:\n%s",
command, out_file, tail_log(out_file))
def pdoc_compile_docs(project, logger, reactor):
logger.info("Generating PDoc documentation")
if not project.get_property("pdoc_module_name"):
raise BuildFailedException("'pdoc_module_name' must be specified")
pdoc_command_args = project.get_property("pdoc_command_args", [])
pdoc_output_dir = project.expand_path("$pdoc_output_dir")
command_and_arguments = ["pdoc"] + pdoc_command_args
if "--html" in pdoc_command_args:
command_and_arguments += ["--html-dir", pdoc_output_dir]
command_and_arguments += [project.get_property("pdoc_module_name")]
source_directory = project.expand_path("$pdoc_source")
environment = {
"PYTHONPATH": source_directory,
"PATH": reactor.pybuilder_venv.environ["PATH"]
}
report_file = project.expand_path("$dir_reports", "pdoc.err")
logger.debug("Executing PDoc as: %s", command_and_arguments)
return_code = reactor.pybuilder_venv.execute_command(
command_and_arguments,
outfile_name=project.expand_path("$dir_reports", "pdoc"),
error_file_name=report_file,
env=environment,
cwd=pdoc_output_dir)
if return_code:
error_str = "PDoc failed! See %s for full details:\n%s" % (
report_file, tail_log(report_file))
logger.error(error_str)
raise BuildFailedException(error_str)
def analyze(project, logger):
""" Applies the frosted script to the sources of the given project. """
logger.info("Executing frosted on project sources.")
verbose = project.get_property("verbose")
project.set_property_if_unset("frosted_verbose_output", verbose)
command = ExternalCommandBuilder("frosted", project)
for ignored_error_code in project.get_property("frosted_ignore", []):
command.use_argument("--ignore={0}".format(ignored_error_code))
include_test_sources = project.get_property("frosted_include_test_sources")
include_scripts = project.get_property("frosted_include_scripts")
result = command.run_on_production_source_files(logger,
include_test_sources=include_test_sources,
include_scripts=include_scripts)
count_of_warnings = len(result.report_lines)
count_of_errors = len(result.error_report_lines)
if count_of_errors > 0:
logger.error("Errors while running frosted. See %s for full details:\n%s" % (
result.error_report_file, tail_log(result.error_report_file)))
if count_of_warnings > 0:
if project.get_property("frosted_break_build"):
error_message = "frosted found {0} warning(s)".format(count_of_warnings)
raise BuildFailedException(error_message)
else:
logger.warn("frosted found %d warning(s).", count_of_warnings)
def _do_install_dependency(logger, project, dependency, upgrade, eager_upgrade,
force_reinstall, constraint_file, target_dir,
log_file):
batch = isinstance(dependency, collections.Iterable)
exit_code = pip_utils.pip_install(
install_targets=dependency,
index_url=project.get_property("install_dependencies_index_url"),
extra_index_url=project.get_property(
"install_dependencies_extra_index_url"),
upgrade=upgrade,
insecure_installs=project.get_property(
"install_dependencies_insecure_installation"),
force_reinstall=force_reinstall,
target_dir=target_dir,
verbose=project.get_property("verbose"),
trusted_host=project.get_property("install_dependencies_trusted_host"),
constraint_file=constraint_file,
eager_upgrade=eager_upgrade,
logger=logger,
outfile_name=log_file)
if exit_code != 0:
if batch:
dependency_name = " batch dependencies."
else:
dependency_name = " dependency '%s'." % dependency.name
raise BuildFailedException(
"Unable to install%s See %s for full details:\n%s",
dependency_name, log_file, tail_log(log_file))
def execute_distutils(project, logger, distutils_commands, clean=False):
reports_dir = _prepare_reports_dir(project)
setup_script = project.expand_path("$dir_dist", "setup.py")
for command in distutils_commands:
logger.debug("Executing distutils command %s", command)
if is_string(command):
out_file = os.path.join(reports_dir, safe_log_file_name(command))
else:
out_file = os.path.join(reports_dir, safe_log_file_name("__".join(command)))
with open(out_file, "w") as out_f:
commands = [sys.executable, setup_script]
if project.get_property("verbose"):
commands.append("-v")
if clean:
commands.extend(["clean", "--all"])
if is_string(command):
commands.extend(command.split())
else:
commands.extend(command)
return_code = _run_process_and_wait(commands, project.expand_path("$dir_dist"), out_f)
if return_code != 0:
raise BuildFailedException(
"Error while executing setup command %s. See %s for full details:\n%s",
command, out_file, tail_log(out_file))
def _execute_twine(project, logger, command, work_dir, out_file):
logger.debug("Executing Twine %s", command)
with open(out_file, "w") as out_f:
commands = [sys.executable, "-m", "twine"] + command
return_code = _run_process_and_wait(commands, work_dir, out_f)
if return_code != 0:
raise BuildFailedException(
"Error while executing Twine %s. See %s for full details:\n%s",
command, out_file, tail_log(out_file))
def run_sphinx_build(build_command, task_name, logger, project, builder=None):
logger.info("Running %s" % task_name)
log_file = project.expand_path("$dir_target", "reports", task_name)
build_command = [sys.executable, "-m"] + build_command
if project.get_property("verbose"):
logger.debug(build_command)
exit_code = execute_command(build_command, log_file, shell=False)
if exit_code != 0:
raise BuildFailedException("Sphinx build command failed. See %s for full details:\n%s",
log_file,
tail_log(log_file))
def run_sphinx_build(build_command, task_name, logger, project, builder=None):
logger.info("Running %s" % task_name)
log_file = project.expand_path("$dir_target", "reports", task_name)
build_command = [sys.executable, "-m"] + build_command
if project.get_property("verbose"):
logger.debug(build_command)
exit_code = execute_command(build_command, log_file, shell=False)
if exit_code != 0:
raise BuildFailedException(
"Sphinx build command failed. See %s for full details:\n%s",
log_file, tail_log(log_file))
def analyze(project, logger, reactor):
""" Applies the flake8 script to the sources of the given project. """
logger.info("Executing flake8 on project sources.")
verbose = project.get_property("verbose")
project.set_property_if_unset("flake8_verbose_output", verbose)
command = ExternalCommandBuilder("flake8", project, reactor)
command.use_argument("--ignore={0}").formatted_with_truthy_property(
"flake8_ignore")
command.use_argument("--extend-ignore={0}").formatted_with_truthy_property(
"flake8_extend_ignore")
command.use_argument("--max-line-length={0}").formatted_with_property(
"flake8_max_line_length")
command.use_argument("--filename={0}").formatted_with_truthy_property(
"flake8_include_patterns")
command.use_argument("--exclude={0}").formatted_with_truthy_property(
"flake8_exclude_patterns")
command.use_argument(
"--max-complexity={0}").formatted_with_truthy_property(
"flake8_max_complexity")
include_test_sources = project.get_property("flake8_include_test_sources")
include_scripts = project.get_property("flake8_include_scripts")
result = command.run_on_production_source_files(
logger,
include_test_sources=include_test_sources,
include_scripts=include_scripts,
include_dirs_only=True)
count_of_warnings = len(result.report_lines)
count_of_errors = len(result.error_report_lines)
if count_of_errors > 0:
logger.error(
"Errors while running flake8. See %s for full details:\n%s" %
(result.error_report_file, tail_log(result.error_report_file)))
if count_of_warnings > 0:
if project.get_property("flake8_break_build"):
error_message = "flake8 found {0} warning(s)".format(
count_of_warnings)
raise BuildFailedException(error_message)
else:
logger.warn("flake8 found %d warning(s).", count_of_warnings)
def run_sphinx_build(build_command,
task_name,
logger,
project,
reactor,
builder=None):
logger.info("Running %s" % task_name)
log_file = project.expand_path("$dir_target", "reports", task_name)
build_command = reactor.pybuilder_venv.executable + ["-c"] + build_command
exit_code = reactor.pybuilder_venv.execute_command(build_command,
log_file,
shell=False)
if exit_code != 0:
raise BuildFailedException(
"Sphinx build command failed. See %s for full details:\n%s",
log_file, tail_log(log_file))
def run_cram_tests(project, logger, reactor):
logger.info("Running Cram command line tests")
cram_tests = list(_find_files(project))
if not cram_tests or len(cram_tests) == 0:
if project.get_property("cram_fail_if_no_tests"):
raise BuildFailedException("No Cram tests found!")
else:
return
pyb_venv = reactor.pybuilder_venv
command_and_arguments = pyb_venv.executable + _cram_command_for(project)
command_and_arguments.extend(cram_tests)
report_file = _report_file(project)
pyb_environ = pyb_venv.environ
if project.get_property("cram_run_test_from_target"):
dist_dir = project.expand_path("$dir_dist")
_prepend_path(pyb_environ, "PYTHONPATH", dist_dir)
script_dir_dist = project.get_property("dir_dist_scripts")
_prepend_path(pyb_environ, "PATH",
os.path.join(dist_dir, script_dir_dist))
else:
source_dir = project.expand_path("$dir_source_main_python")
_prepend_path(pyb_environ, "PYTHONPATH", source_dir)
script_dir = project.expand_path("$dir_source_main_scripts")
_prepend_path(pyb_environ, "PATH", script_dir)
return_code = pyb_venv.execute_command(command_and_arguments,
report_file,
env=pyb_environ,
error_file_name=report_file)
if return_code != 0:
error_str = "Cram tests failed! See %s for full details:\n%s" % (
report_file, tail_log(report_file))
logger.error(error_str)
raise BuildFailedException(error_str)
report = read_file(report_file)
result = report[-1][2:].strip()
logger.info("Cram tests were fine")
logger.info(result)
def _execute_twine(project, logger, command, work_dir, out_file):
logger.debug("Executing Twine %s", command)
with open(out_file, "w") as out_f:
commands = [sys.executable, "-m", "twine"] + command
return_code = _run_process_and_wait(commands, work_dir, out_f)
if return_code != 0:
raise BuildFailedException(
"Error while executing Twine %s. See %s for full details:\n%s", command, out_file, tail_log(out_file))
def run_py2dsc_deb_build(build_command, task_name, logger, project):
logger.info("Running %s" % task_name)
log_file = project.expand_path("$dir_target", "reports", task_name)
if project.get_property("verbose"):
logger.info(build_command)
exit_code = execute_command(build_command, log_file, shell=True)
if exit_code != 0:
raise BuildFailedException(
"py2dsc_deb build command failed. See %s for full details:\n%s", log_file, tail_log(log_file))
def run_py2dsc_deb_build(python_env, build_command, task_name, logger, project):
logger.info("Running %s" % task_name)
log_file = project.expand_path("$dir_target", "reports", task_name)
if project.get_property("verbose"):
logger.info(build_command)
exit_code = python_env.execute_command(build_command, log_file, shell=True)
if exit_code != 0:
raise BuildFailedException(
"py2dsc_deb build command failed. See %s for full details:\n%s", log_file, tail_log(log_file))
def run_cram_tests(project, logger):
logger.info("Running Cram command line tests")
cram_tests = list(_find_files(project))
if not cram_tests or len(cram_tests) == 0:
if project.get_property("cram_fail_if_no_tests"):
raise BuildFailedException("No Cram tests found!")
else:
return
command_and_arguments = _cram_command_for(project)
command_and_arguments.extend(cram_tests)
report_file = _report_file(project)
env = os.environ.copy()
if project.get_property('cram_run_test_from_target'):
dist_dir = project.expand_path("$dir_dist")
_prepend_path(env, "PYTHONPATH", dist_dir)
script_dir_dist = project.get_property('dir_dist_scripts')
_prepend_path(env, "PATH", os.path.join(dist_dir, script_dir_dist))
else:
source_dir = project.expand_path("$dir_source_main_python")
_prepend_path(env, "PYTHONPATH", source_dir)
script_dir = project.expand_path('$dir_source_main_scripts')
_prepend_path(env, "PATH", script_dir)
return_code = execute_command(command_and_arguments,
report_file,
env=env,
error_file_name=report_file)
if return_code != 0:
error_str = "Cram tests failed! See %s for full details:\n%s" % (report_file, tail_log(report_file))
logger.error(error_str)
raise BuildFailedException(error_str)
report = read_file(report_file)
result = report[-1][2:].strip()
logger.info("Cram tests were fine")
logger.info(result)
def install_dependencies(logger, project, dependencies, python_env,
log_file_name,
local_mapping=None,
constraints_file_name=None,
log_file_mode="ab",
package_type="dependency",
target_dir=None,
ignore_installed=False,
):
entry_paths = target_dir or python_env.site_paths
dependencies_to_install, orig_installed_pkgs, dependency_constraints = _filter_dependencies(logger,
project,
dependencies,
entry_paths,
ignore_installed)
constraints_file = None
if constraints_file_name:
constraints_file = np(jp(python_env.env_dir, constraints_file_name))
create_constraint_file(constraints_file, dependency_constraints)
if not local_mapping:
local_mapping = {}
install_batch = []
for dependency in dependencies_to_install:
url = getattr(dependency, "url", None)
install_options = {}
if should_update_package(dependency.version) and not getattr(dependency, "version_not_a_spec", False):
install_options["upgrade"] = True
if dependency.name in local_mapping or url:
install_options["force_reinstall"] = bool(url)
if not target_dir and dependency.name in local_mapping:
install_options["target_dir"] = local_mapping[dependency.name]
install_batch.append((as_pip_install_target(dependency), install_options))
logger.info("Processing %s packages '%s%s'%s to be installed with %s", package_type, dependency.name,
dependency.version if dependency.version else "",
" from %s" % url if url else "", install_options)
if install_batch:
pip_env = {"PIP_NO_INPUT": "1"}
if project.offline:
pip_env["PIP_NO_INDEX"] = "1"
logger.warn("PIP will be operating in the offline mode")
with open(np(log_file_name), log_file_mode) as log_file:
results = pip_install_batches(install_batch,
python_env,
index_url=project.get_property("install_dependencies_index_url"),
extra_index_url=project.get_property("install_dependencies_extra_index_url"),
trusted_host=project.get_property("install_dependencies_trusted_host"),
insecure_installs=project.get_property(
"install_dependencies_insecure_installation"),
verbose=project.get_property("pip_verbose"),
constraint_file=constraints_file,
logger=logger,
outfile_name=log_file,
error_file_name=log_file,
target_dir=target_dir,
ignore_installed=ignore_installed)
for result in results:
if result:
raise BuildFailedException("Unable to install %s packages into %s. "
"Please see '%s' for full details:\n%s",
package_type,
python_env.env_dir,
log_file_name,
tail_log(log_file_name))
return dependencies_to_install
