def analyze(project, logger): """ Applies the flake8 script to the sources of the given project. """ logger.info("Executing flake8 on project sources.") verbose = project.get_property("verbose") project.set_property_if_unset("flake8_verbose_output", verbose) command = ExternalCommandBuilder('flake8', project) command.use_argument('--ignore={0}').formatted_with_truthy_property('flake8_ignore') command.use_argument('--max-line-length={0}').formatted_with_property('flake8_max_line_length') command.use_argument('--filename={0}').formatted_with_truthy_property('flake8_include_patterns') command.use_argument('--exclude={0}').formatted_with_truthy_property('flake8_exclude_patterns') command.use_argument('--max-complexity={0}').formatted_with_truthy_property('flake8_max_complexity') include_test_sources = project.get_property("flake8_include_test_sources") include_scripts = project.get_property("flake8_include_scripts") result = command.run_on_production_source_files(logger, include_test_sources=include_test_sources, include_scripts=include_scripts, include_dirs_only=True) count_of_warnings = len(result.report_lines) count_of_errors = len(result.error_report_lines) if count_of_errors > 0: logger.error('Errors while running flake8. See %s for full details:\n%s' % ( result.error_report_file, tail_log(result.error_report_file))) if count_of_warnings > 0: if project.get_property("flake8_break_build"): error_message = "flake8 found {0} warning(s)".format(count_of_warnings) raise BuildFailedException(error_message) else: logger.warn("flake8 found %d warning(s).", count_of_warnings)
def _do_install_dependency(logger, project, dependency, upgrade, eager_upgrade, force_reinstall, constraint_file, target_dir, log_file): batch = isinstance(dependency, collections.Iterable) exit_code = pip_utils.pip_install( install_targets=dependency, index_url=project.get_property("install_dependencies_index_url"), extra_index_url=project.get_property("install_dependencies_extra_index_url"), upgrade=upgrade, insecure_installs=project.get_property("install_dependencies_insecure_installation"), force_reinstall=force_reinstall, target_dir=target_dir, verbose=project.get_property("verbose"), trusted_host=project.get_property("install_dependencies_trusted_host"), constraint_file=constraint_file, eager_upgrade=eager_upgrade, logger=logger, outfile_name=log_file) if exit_code != 0: if batch: dependency_name = " batch dependencies." else: dependency_name = " dependency '%s'." % dependency.name raise BuildFailedException("Unable to install%s See %s for full details:\n%s", dependency_name, log_file, tail_log(log_file))
def analyze(project, logger): """ Applies the frosted script to the sources of the given project. """ logger.info("Executing frosted on project sources.") verbose = project.get_property("verbose") project.set_property_if_unset("frosted_verbose_output", verbose) command = ExternalCommandBuilder("frosted", project) for ignored_error_code in project.get_property("frosted_ignore", []): command.use_argument("--ignore={0}".format(ignored_error_code)) include_test_sources = project.get_property("frosted_include_test_sources") include_scripts = project.get_property("frosted_include_scripts") result = command.run_on_production_source_files( logger, include_test_sources=include_test_sources, include_scripts=include_scripts) count_of_warnings = len(result.report_lines) count_of_errors = len(result.error_report_lines) if count_of_errors > 0: logger.error( "Errors while running frosted. See %s for full details:\n%s" % (result.error_report_file, tail_log(result.error_report_file))) if count_of_warnings > 0: if project.get_property("frosted_break_build"): error_message = "frosted found {0} warning(s)".format( count_of_warnings) raise BuildFailedException(error_message) else: logger.warn("frosted found %d warning(s).", count_of_warnings)
def execute_distutils(project, logger, distutils_commands, clean=False): reports_dir = _prepare_reports_dir(project) setup_script = project.expand_path("$dir_dist", "setup.py") for command in distutils_commands: logger.debug("Executing distutils command %s", command) if is_string(command): out_file = os.path.join(reports_dir, safe_log_file_name(command)) else: out_file = os.path.join(reports_dir, safe_log_file_name("__".join(command))) with open(out_file, "w") as out_f: commands = [sys.executable, setup_script] if project.get_property("verbose"): commands.append("-v") if clean: commands.extend(["clean", "--all"]) if is_string(command): commands.extend(command.split()) else: commands.extend(command) return_code = _run_process_and_wait(commands, project.expand_path("$dir_dist"), out_f) if return_code != 0: raise BuildFailedException( "Error while executing setup command %s. See %s for full details:\n%s", command, out_file, tail_log(out_file))
def pdoc_compile_docs(project, logger, reactor): logger.info("Generating PDoc documentation") if not project.get_property("pdoc_module_name"): raise BuildFailedException("'pdoc_module_name' must be specified") pdoc_command_args = project.get_property("pdoc_command_args", []) pdoc_output_dir = project.expand_path("$pdoc_output_dir") command_and_arguments = ["pdoc"] + pdoc_command_args if "--html" in pdoc_command_args: command_and_arguments += ["--html-dir", pdoc_output_dir] command_and_arguments += [project.get_property("pdoc_module_name")] source_directory = project.expand_path("$pdoc_source") environment = { "PYTHONPATH": source_directory, "PATH": reactor.pybuilder_venv.environ["PATH"] } report_file = project.expand_path("$dir_reports", "pdoc.err") logger.debug("Executing PDoc as: %s", command_and_arguments) return_code = reactor.pybuilder_venv.execute_command( command_and_arguments, outfile_name=project.expand_path("$dir_reports", "pdoc"), error_file_name=report_file, env=environment, cwd=pdoc_output_dir) if return_code: error_str = "PDoc failed! See %s for full details:\n%s" % ( report_file, tail_log(report_file)) logger.error(error_str) raise BuildFailedException(error_str)
def analyze(project, logger): """ Applies the frosted script to the sources of the given project. """ logger.info("Executing frosted on project sources.") verbose = project.get_property("verbose") project.set_property_if_unset("frosted_verbose_output", verbose) command = ExternalCommandBuilder("frosted", project) for ignored_error_code in project.get_property("frosted_ignore", []): command.use_argument("--ignore={0}".format(ignored_error_code)) include_test_sources = project.get_property("frosted_include_test_sources") include_scripts = project.get_property("frosted_include_scripts") result = command.run_on_production_source_files(logger, include_test_sources=include_test_sources, include_scripts=include_scripts) count_of_warnings = len(result.report_lines) count_of_errors = len(result.error_report_lines) if count_of_errors > 0: logger.error("Errors while running frosted. See %s for full details:\n%s" % ( result.error_report_file, tail_log(result.error_report_file))) if count_of_warnings > 0: if project.get_property("frosted_break_build"): error_message = "frosted found {0} warning(s)".format(count_of_warnings) raise BuildFailedException(error_message) else: logger.warn("frosted found %d warning(s).", count_of_warnings)
def _do_install_dependency(logger, project, dependency, upgrade, eager_upgrade, force_reinstall, constraint_file, target_dir, log_file): batch = isinstance(dependency, collections.Iterable) exit_code = pip_utils.pip_install( install_targets=dependency, index_url=project.get_property("install_dependencies_index_url"), extra_index_url=project.get_property( "install_dependencies_extra_index_url"), upgrade=upgrade, insecure_installs=project.get_property( "install_dependencies_insecure_installation"), force_reinstall=force_reinstall, target_dir=target_dir, verbose=project.get_property("verbose"), trusted_host=project.get_property("install_dependencies_trusted_host"), constraint_file=constraint_file, eager_upgrade=eager_upgrade, logger=logger, outfile_name=log_file) if exit_code != 0: if batch: dependency_name = " batch dependencies." else: dependency_name = " dependency '%s'." % dependency.name raise BuildFailedException( "Unable to install%s See %s for full details:\n%s", dependency_name, log_file, tail_log(log_file))
def execute_distutils(project, logger, distutils_commands, clean=False): reports_dir = _prepare_reports_dir(project) setup_script = project.expand_path("$dir_dist", "setup.py") for command in distutils_commands: logger.debug("Executing distutils command %s", command) if is_string(command): out_file = os.path.join(reports_dir, safe_log_file_name(command)) else: out_file = os.path.join(reports_dir, safe_log_file_name("__".join(command))) with open(out_file, "w") as out_f: commands = [sys.executable, setup_script] if project.get_property("verbose"): commands.append("-v") if clean: commands.extend(["clean", "--all"]) if is_string(command): commands.extend(command.split()) else: commands.extend(command) return_code = _run_process_and_wait(commands, project.expand_path("$dir_dist"), out_f) if return_code != 0: raise BuildFailedException( "Error while executing setup command %s. See %s for full details:\n%s", command, out_file, tail_log(out_file))
def _execute_twine(project, logger, command, work_dir, out_file): logger.debug("Executing Twine %s", command) with open(out_file, "w") as out_f: commands = [sys.executable, "-m", "twine"] + command return_code = _run_process_and_wait(commands, work_dir, out_f) if return_code != 0: raise BuildFailedException( "Error while executing Twine %s. See %s for full details:\n%s", command, out_file, tail_log(out_file))
def run_sphinx_build(build_command, task_name, logger, project, builder=None): logger.info("Running %s" % task_name) log_file = project.expand_path("$dir_target", "reports", task_name) build_command = [sys.executable, "-m"] + build_command if project.get_property("verbose"): logger.debug(build_command) exit_code = execute_command(build_command, log_file, shell=False) if exit_code != 0: raise BuildFailedException("Sphinx build command failed. See %s for full details:\n%s", log_file, tail_log(log_file))
def run_sphinx_build(build_command, task_name, logger, project, builder=None): logger.info("Running %s" % task_name) log_file = project.expand_path("$dir_target", "reports", task_name) build_command = [sys.executable, "-m"] + build_command if project.get_property("verbose"): logger.debug(build_command) exit_code = execute_command(build_command, log_file, shell=False) if exit_code != 0: raise BuildFailedException( "Sphinx build command failed. See %s for full details:\n%s", log_file, tail_log(log_file))
def analyze(project, logger, reactor): """ Applies the flake8 script to the sources of the given project. """ logger.info("Executing flake8 on project sources.") verbose = project.get_property("verbose") project.set_property_if_unset("flake8_verbose_output", verbose) command = ExternalCommandBuilder("flake8", project, reactor) command.use_argument("--ignore={0}").formatted_with_truthy_property( "flake8_ignore") command.use_argument("--extend-ignore={0}").formatted_with_truthy_property( "flake8_extend_ignore") command.use_argument("--max-line-length={0}").formatted_with_property( "flake8_max_line_length") command.use_argument("--filename={0}").formatted_with_truthy_property( "flake8_include_patterns") command.use_argument("--exclude={0}").formatted_with_truthy_property( "flake8_exclude_patterns") command.use_argument( "--max-complexity={0}").formatted_with_truthy_property( "flake8_max_complexity") include_test_sources = project.get_property("flake8_include_test_sources") include_scripts = project.get_property("flake8_include_scripts") result = command.run_on_production_source_files( logger, include_test_sources=include_test_sources, include_scripts=include_scripts, include_dirs_only=True) count_of_warnings = len(result.report_lines) count_of_errors = len(result.error_report_lines) if count_of_errors > 0: logger.error( "Errors while running flake8. See %s for full details:\n%s" % (result.error_report_file, tail_log(result.error_report_file))) if count_of_warnings > 0: if project.get_property("flake8_break_build"): error_message = "flake8 found {0} warning(s)".format( count_of_warnings) raise BuildFailedException(error_message) else: logger.warn("flake8 found %d warning(s).", count_of_warnings)
def run_sphinx_build(build_command, task_name, logger, project, reactor, builder=None): logger.info("Running %s" % task_name) log_file = project.expand_path("$dir_target", "reports", task_name) build_command = reactor.pybuilder_venv.executable + ["-c"] + build_command exit_code = reactor.pybuilder_venv.execute_command(build_command, log_file, shell=False) if exit_code != 0: raise BuildFailedException( "Sphinx build command failed. See %s for full details:\n%s", log_file, tail_log(log_file))
def run_cram_tests(project, logger, reactor): logger.info("Running Cram command line tests") cram_tests = list(_find_files(project)) if not cram_tests or len(cram_tests) == 0: if project.get_property("cram_fail_if_no_tests"): raise BuildFailedException("No Cram tests found!") else: return pyb_venv = reactor.pybuilder_venv command_and_arguments = pyb_venv.executable + _cram_command_for(project) command_and_arguments.extend(cram_tests) report_file = _report_file(project) pyb_environ = pyb_venv.environ if project.get_property("cram_run_test_from_target"): dist_dir = project.expand_path("$dir_dist") _prepend_path(pyb_environ, "PYTHONPATH", dist_dir) script_dir_dist = project.get_property("dir_dist_scripts") _prepend_path(pyb_environ, "PATH", os.path.join(dist_dir, script_dir_dist)) else: source_dir = project.expand_path("$dir_source_main_python") _prepend_path(pyb_environ, "PYTHONPATH", source_dir) script_dir = project.expand_path("$dir_source_main_scripts") _prepend_path(pyb_environ, "PATH", script_dir) return_code = pyb_venv.execute_command(command_and_arguments, report_file, env=pyb_environ, error_file_name=report_file) if return_code != 0: error_str = "Cram tests failed! See %s for full details:\n%s" % ( report_file, tail_log(report_file)) logger.error(error_str) raise BuildFailedException(error_str) report = read_file(report_file) result = report[-1][2:].strip() logger.info("Cram tests were fine") logger.info(result)
def _execute_twine(project, logger, command, work_dir, out_file): logger.debug("Executing Twine %s", command) with open(out_file, "w") as out_f: commands = [sys.executable, "-m", "twine"] + command return_code = _run_process_and_wait(commands, work_dir, out_f) if return_code != 0: raise BuildFailedException( "Error while executing Twine %s. See %s for full details:\n%s", command, out_file, tail_log(out_file))
def run_py2dsc_deb_build(build_command, task_name, logger, project): logger.info("Running %s" % task_name) log_file = project.expand_path("$dir_target", "reports", task_name) if project.get_property("verbose"): logger.info(build_command) exit_code = execute_command(build_command, log_file, shell=True) if exit_code != 0: raise BuildFailedException( "py2dsc_deb build command failed. See %s for full details:\n%s", log_file, tail_log(log_file))
def run_py2dsc_deb_build(python_env, build_command, task_name, logger, project): logger.info("Running %s" % task_name) log_file = project.expand_path("$dir_target", "reports", task_name) if project.get_property("verbose"): logger.info(build_command) exit_code = python_env.execute_command(build_command, log_file, shell=True) if exit_code != 0: raise BuildFailedException( "py2dsc_deb build command failed. See %s for full details:\n%s", log_file, tail_log(log_file))
def run_cram_tests(project, logger): logger.info("Running Cram command line tests") cram_tests = list(_find_files(project)) if not cram_tests or len(cram_tests) == 0: if project.get_property("cram_fail_if_no_tests"): raise BuildFailedException("No Cram tests found!") else: return command_and_arguments = _cram_command_for(project) command_and_arguments.extend(cram_tests) report_file = _report_file(project) env = os.environ.copy() if project.get_property('cram_run_test_from_target'): dist_dir = project.expand_path("$dir_dist") _prepend_path(env, "PYTHONPATH", dist_dir) script_dir_dist = project.get_property('dir_dist_scripts') _prepend_path(env, "PATH", os.path.join(dist_dir, script_dir_dist)) else: source_dir = project.expand_path("$dir_source_main_python") _prepend_path(env, "PYTHONPATH", source_dir) script_dir = project.expand_path('$dir_source_main_scripts') _prepend_path(env, "PATH", script_dir) return_code = execute_command(command_and_arguments, report_file, env=env, error_file_name=report_file) if return_code != 0: error_str = "Cram tests failed! See %s for full details:\n%s" % (report_file, tail_log(report_file)) logger.error(error_str) raise BuildFailedException(error_str) report = read_file(report_file) result = report[-1][2:].strip() logger.info("Cram tests were fine") logger.info(result)
def install_dependencies(logger, project, dependencies, python_env, log_file_name, local_mapping=None, constraints_file_name=None, log_file_mode="ab", package_type="dependency", target_dir=None, ignore_installed=False, ): entry_paths = target_dir or python_env.site_paths dependencies_to_install, orig_installed_pkgs, dependency_constraints = _filter_dependencies(logger, project, dependencies, entry_paths, ignore_installed) constraints_file = None if constraints_file_name: constraints_file = np(jp(python_env.env_dir, constraints_file_name)) create_constraint_file(constraints_file, dependency_constraints) if not local_mapping: local_mapping = {} install_batch = [] for dependency in dependencies_to_install: url = getattr(dependency, "url", None) install_options = {} if should_update_package(dependency.version) and not getattr(dependency, "version_not_a_spec", False): install_options["upgrade"] = True if dependency.name in local_mapping or url: install_options["force_reinstall"] = bool(url) if not target_dir and dependency.name in local_mapping: install_options["target_dir"] = local_mapping[dependency.name] install_batch.append((as_pip_install_target(dependency), install_options)) logger.info("Processing %s packages '%s%s'%s to be installed with %s", package_type, dependency.name, dependency.version if dependency.version else "", " from %s" % url if url else "", install_options) if install_batch: pip_env = {"PIP_NO_INPUT": "1"} if project.offline: pip_env["PIP_NO_INDEX"] = "1" logger.warn("PIP will be operating in the offline mode") with open(np(log_file_name), log_file_mode) as log_file: results = pip_install_batches(install_batch, python_env, index_url=project.get_property("install_dependencies_index_url"), extra_index_url=project.get_property("install_dependencies_extra_index_url"), trusted_host=project.get_property("install_dependencies_trusted_host"), insecure_installs=project.get_property( "install_dependencies_insecure_installation"), verbose=project.get_property("pip_verbose"), constraint_file=constraints_file, logger=logger, outfile_name=log_file, error_file_name=log_file, target_dir=target_dir, ignore_installed=ignore_installed) for result in results: if result: raise BuildFailedException("Unable to install %s packages into %s. " "Please see '%s' for full details:\n%s", package_type, python_env.env_dir, log_file_name, tail_log(log_file_name)) return dependencies_to_install