def test_execute_command(self, popen, _):
popen.return_value = Mock()
popen.return_value.wait.return_value = 0
self.assertEquals(execute_command(["test", "commands"]), 0)
self.assertEquals(execute_command(["test", "commands"], outfile_name="test.out"), 0)
self.assertEquals(
execute_command(["test", "commands"], outfile_name="test.out", error_file_name="test.out.err"), 0)
def test_execute_command(self, popen, _):
popen.return_value = Mock()
popen.return_value.wait.return_value = 0
self.assertEquals(execute_command(["test", "commands"]), 0)
self.assertEquals(execute_command(["test", "commands"], outfile_name="test.out"), 0)
self.assertEquals(
execute_command(["test", "commands"], outfile_name="test.out", error_file_name="test.out.err"), 0)
def collect_dependencies(project, internal_report_file, external_report_file):
source_dir = project.expand_path("$dir_source_main_python")
internal_command = ["sfood", "--internal"]
external_command = ["sfood", "--external"]
execute_command(internal_command, internal_report_file, cwd=source_dir)
execute_command(external_command, external_report_file, cwd=source_dir)
def collect_dependencies(project, internal_report_file, external_report_file):
source_dir = project.expand_path("$dir_source_main_python")
internal_command = ["sfood", "--internal"]
external_command = ["sfood", "--external"]
execute_command(internal_command, internal_report_file, cwd=source_dir)
execute_command(external_command, external_report_file, cwd=source_dir)
def generate_manpages(project, logger):
"""
Uses the ronn script to convert a markdown source to a gzipped manpage.
"""
logger.info('Generating manpages')
try:
os.makedirs(project.get_property('dir_manpages'))
except OSError as e:
if e.errno != errno.EEXIST:
raise
ronn_report_file = project.expand_path("$dir_reports/{0}".format('generate_manpage'))
generate_manpages_command = build_generate_manpages_command(project)
execute_command(generate_manpages_command, ronn_report_file, shell=True)
def generate_manpages(project, logger):
"""
Uses the ronn script to convert a markdown source to a gzipped manpage.
"""
logger.info('Generating manpages')
try:
os.makedirs(project.get_property('dir_manpages'))
except OSError as e:
if e.errno != errno.EEXIST:
raise
ronn_report_file = project.expand_path(
"$dir_reports/{0}".format('generate_manpage'))
generate_manpages_command = build_generate_manpages_command(project)
execute_command(generate_manpages_command, ronn_report_file, shell=True)
def pip_install(install_targets, index_url=None, extra_index_url=None, upgrade=False, insecure_installs=None,
force_reinstall=False, target_dir=None, verbose=False, trusted_host=None, constraint_file=None,
eager_upgrade=False,
logger=None, outfile_name=None, error_file_name=None, env=None, cwd=None):
pip_command_line = list()
pip_command_line.extend(PIP_EXEC_STANZA)
pip_command_line.append("install")
pip_command_line.extend(build_pip_install_options(index_url=index_url,
extra_index_url=extra_index_url,
upgrade=upgrade,
insecure_installs=insecure_installs,
force_reinstall=force_reinstall,
target_dir=target_dir,
verbose=verbose,
trusted_host=trusted_host,
constraint_file=constraint_file,
eager_upgrade=eager_upgrade
))
for install_target in as_list(install_targets):
pip_command_line.extend(as_pip_install_target(install_target))
if env is None:
env = os.environ
if logger:
logger.debug("Invoking pip: %s", pip_command_line)
return execute_command(pip_command_line, outfile_name=outfile_name, env=env, cwd=cwd,
error_file_name=error_file_name, shell=False)
def _do_install_dependency(logger, project, dependency, upgrade,
force_reinstall, target_dir, log_file):
batch = isinstance(dependency, collections.Iterable)
pip_command_line = list()
pip_command_line.extend(PIP_EXEC_STANZA)
pip_command_line.append("install")
pip_command_line.extend(
build_pip_install_options(
project.get_property("install_dependencies_index_url"),
project.get_property("install_dependencies_extra_index_url"),
upgrade,
project.get_property("install_dependencies_insecure_installation"),
force_reinstall, target_dir, project.get_property("verbose"),
project.get_property("install_dependencies_trusted_host")))
pip_command_line.extend(as_pip_install_target(dependency))
logger.debug("Invoking pip: %s", pip_command_line)
exit_code = execute_command(pip_command_line,
log_file,
env=os.environ,
shell=False)
if exit_code != 0:
if batch:
dependency_name = " batch dependencies."
else:
dependency_name = " dependency '%s'." % dependency.name
if project.get_property("verbose"):
print_file_content(log_file)
raise BuildFailedException("Unable to install%s" % dependency_name)
else:
raise BuildFailedException(
"Unable to install%s See %s for details.", dependency_name,
log_file)
def pip_install(install_targets,
index_url=None,
extra_index_url=None,
upgrade=False,
insecure_installs=None,
force_reinstall=False,
target_dir=None,
verbose=False,
logger=None,
outfile_name=None,
error_file_name=None,
env=None,
cwd=None,
trusted_host=None):
pip_command_line = list()
pip_command_line.extend(PIP_EXEC_STANZA)
pip_command_line.append("install")
pip_command_line.extend(
build_pip_install_options(index_url, extra_index_url, upgrade,
insecure_installs, force_reinstall,
target_dir, verbose, trusted_host))
for install_target in as_list(install_targets):
pip_command_line.extend(as_pip_install_target(install_target))
if env is None:
env = os.environ
if logger:
logger.debug("Invoking pip: %s", pip_command_line)
return execute_command(pip_command_line,
outfile_name=outfile_name,
env=env,
cwd=cwd,
error_file_name=error_file_name,
shell=False)
def run_single_test(logger, project, reports_dir, test, ):
name, _ = os.path.splitext(os.path.basename(test))
logger.info("Running acceptance test %s", name)
env = prepare_environment(project)
test_time = Timer.start()
command_and_arguments = (sys.executable, test)
report_file_name = os.path.join(reports_dir, name)
error_file_name = report_file_name + ".err"
return_code = execute_command(
command_and_arguments, report_file_name, env, error_file_name=error_file_name)
test_time.stop()
report_item = {
"test": name,
"test_file": test,
"time": test_time.get_millis(),
"success": True
}
if return_code != 0:
logger.error("acceptance test failed: %s", test)
report_item["success"] = False
if project.get_property("verbose"):
print_file_content(report_file_name)
print_text_line()
print_file_content(error_file_name)
return report_item
def install_dependency(logger, project, dependency):
url = getattr(dependency, "url", None)
logger.info("Installing dependency '%s'%s", dependency.name,
" from %s" % url if url else "")
log_file = project.expand_path("$dir_install_logs", dependency.name)
log_file = re.sub(r'<|>|=', '_', log_file)
if sys.platform.startswith("win"):
# we can't use quotes on windows
pip_dependency = as_pip_argument(dependency)
else:
# on linux we need quotes because version pinning (>=) would be an IO redirect
pip_dependency = "'{0}'".format(as_pip_argument(dependency))
pip_command_line = "pip install {0}{1}".format(
build_pip_install_options(project, dependency.name), pip_dependency)
exit_code = execute_command(pip_command_line, log_file, shell=True)
if exit_code != 0:
if project.get_property("verbose"):
print_file_content(log_file)
raise BuildFailedException("Unable to install dependency '%s'.",
dependency.name)
else:
raise BuildFailedException(
"Unable to install dependency '%s'. See %s for details.",
getattr(dependency, "name", dependency), log_file)
def execute_tool_on_source_files(project,
name,
command_and_arguments,
logger=None,
include_test_sources=False,
include_scripts=False,
include_dirs_only=False):
if include_dirs_only:
files = discover_affected_dirs(include_test_sources, include_scripts,
project)
else:
files = discover_affected_files(include_test_sources, include_scripts,
project)
command = as_list(command_and_arguments) + [f for f in files]
report_file = project.expand_path("$dir_reports/{0}".format(name))
execution_result = execute_command(command, report_file), report_file
report_file = execution_result[1]
report_lines = read_file(report_file)
if project.get_property(name + "_verbose_output") and logger:
log_report(logger, name, report_lines)
return execution_result
def _install_external_plugin(name, version, logger, plugin_module_name):
if not name.startswith(PYPI_PLUGIN_PROTOCOL) and not name.startswith(
VCS_PLUGIN_PROTOCOL):
message = "Only plugins starting with '{0}' are currently supported"
raise MissingPluginException(
name, message.format((PYPI_PLUGIN_PROTOCOL, VCS_PLUGIN_PROTOCOL)))
if name.startswith(PYPI_PLUGIN_PROTOCOL):
pip_package = name.replace(PYPI_PLUGIN_PROTOCOL, "")
if version:
pip_package += str(version)
elif name.startswith(VCS_PLUGIN_PROTOCOL):
pip_package = name.replace(VCS_PLUGIN_PROTOCOL, "")
with tempfile.NamedTemporaryFile(delete=False) as log_file:
log_file_name = log_file.name
install_cmd = ['pip', 'install', '--upgrade', pip_package]
result = execute_command(install_cmd,
log_file_name,
error_file_name=log_file_name,
cwd=".",
shell=False)
if result != 0:
logger.error("The following pip error was encountered:\n" +
"".join(read_file(log_file_name)))
message = "Failed to install plugin from {0}".format(pip_package)
raise MissingPluginException(name, message)
def run(self, outfile_name):
error_file_name = "{0}.err".format(outfile_name)
return_code = execute_command(self.parts, outfile_name)
error_file_lines = read_file(error_file_name)
outfile_lines = read_file(outfile_name)
return ExternalCommandResult(return_code, outfile_name, outfile_lines, error_file_name, error_file_lines)
def _do_install_dependency(logger, project, dependency, upgrade, force_reinstall, target_dir, log_file):
batch = isinstance(dependency, collections.Iterable)
pip_command_line = list()
pip_command_line.extend(PIP_EXEC_STANZA)
pip_command_line.append("install")
pip_command_line.extend(build_pip_install_options(project.get_property("install_dependencies_index_url"),
project.get_property("install_dependencies_extra_index_url"),
upgrade,
project.get_property(
"install_dependencies_insecure_installation"),
force_reinstall,
target_dir,
project.get_property("verbose"),
project.get_property("install_dependencies_trusted_host")
))
pip_command_line.extend(as_pip_install_target(dependency))
logger.debug("Invoking pip: %s", pip_command_line)
exit_code = execute_command(pip_command_line, log_file, env=os.environ, shell=False)
if exit_code != 0:
if batch:
dependency_name = " batch dependencies."
else:
dependency_name = " dependency '%s'." % dependency.name
if project.get_property("verbose"):
print_file_content(log_file)
raise BuildFailedException("Unable to install%s" % dependency_name)
else:
raise BuildFailedException("Unable to install%s See %s for details.",
dependency_name,
log_file)
def run(self, outfile_name):
error_file_name = "{0}.err".format(outfile_name)
return_code = execute_command(self.parts, outfile_name)
error_file_lines = read_file(error_file_name)
outfile_lines = read_file(outfile_name)
return ExternalCommandResult(return_code, outfile_name, outfile_lines,
error_file_name, error_file_lines)
def run_sphinx_build(build_command, task_name, logger, project):
logger.info("Running %s" % task_name)
log_file = project.expand_path("$dir_target/reports/{0}".format(task_name))
if project.get_property("verbose"):
logger.info(build_command)
exit_code = execute_command(build_command, log_file, shell=True)
if exit_code != 0:
raise BuildFailedException(
"Sphinx build command failed. See %s for details.", log_file)
def run_py2dsc_deb_build(build_command, task_name, logger, project):
logger.info("Running %s" % task_name)
log_file = project.expand_path("$dir_target", "reports", task_name)
if project.get_property("verbose"):
logger.info(build_command)
exit_code = execute_command(build_command, log_file, shell=True)
if exit_code != 0:
raise BuildFailedException(
"py2dsc_deb build command failed. See %s for full details:\n%s", log_file, tail_log(log_file))
def execute_pymetrics(project, logger):
logger.info("Executing pymetrics on project sources")
source_dir = project.expand_path("$dir_source_main_python")
files_to_scan = []
for root, _, files in os.walk(source_dir):
for file_name in files:
if file_name.endswith(".py"):
files_to_scan.append(os.path.join(root, file_name))
csv_file = project.expand_path("$dir_reports/pymetrics.csv")
command = ["pymetrics", "--nosql", "-c", csv_file] + files_to_scan
report_file = project.expand_path("$dir_reports/pymetrics")
env = {"PYTHONPATH": source_dir}
execute_command(command, report_file, env=env)
def execute_pymetrics(project, logger):
logger.info("Executing pymetrics on project sources")
source_dir = project.expand_path("$dir_source_main_python")
files_to_scan = []
for root, _, files in os.walk(source_dir):
for file_name in files:
if file_name.endswith(".py"):
files_to_scan.append(os.path.join(root, file_name))
csv_file = project.expand_path("$dir_reports/pymetrics.csv")
command = ["pymetrics", "--nosql", "-c", csv_file] + files_to_scan
report_file = project.expand_path("$dir_reports/pymetrics")
env = {"PYTHONPATH": source_dir}
execute_command(command, report_file, env=env)
def run_sphinx_build(build_command, task_name, logger, project):
logger.info("Running %s" % task_name)
log_file = project.expand_path(
"$dir_target/reports/{0}".format(task_name))
if project.get_property("verbose"):
logger.info(build_command)
exit_code = execute_command(build_command, log_file, shell=True)
if exit_code != 0:
raise BuildFailedException("Sphinx build command failed. See %s for details.", log_file)
def execute_tool_on_modules(project, name, command_and_arguments, extend_pythonpath=True):
source_dir = project.expand_path("$dir_source_main_python")
modules = discover_modules(source_dir)
command = as_list(command_and_arguments) + modules
report_file = project.expand_path("$dir_reports/%s" % name)
env = os.environ
if extend_pythonpath:
env["PYTHONPATH"] = source_dir
return execute_command(command, report_file, env=env), report_file
def execute_tool_on_modules(project, name, command_and_arguments, extend_pythonpath=True):
source_dir = project.expand_path("$dir_source_main_python")
modules = discover_modules(source_dir)
command = as_list(command_and_arguments) + modules
report_file = project.expand_path("$dir_reports/%s" % name)
env = os.environ
if extend_pythonpath:
env["PYTHONPATH"] = source_dir
return execute_command(command, report_file, env=env), report_file
def run_sphinx_build(build_command, task_name, logger, project, builder=None):
logger.info("Running %s" % task_name)
log_file = project.expand_path("$dir_target", "reports", task_name)
build_command = [sys.executable, "-m"] + build_command
if project.get_property("verbose"):
logger.debug(build_command)
exit_code = execute_command(build_command, log_file, shell=False)
if exit_code != 0:
raise BuildFailedException("Sphinx build command failed. See %s for details.", log_file)
def run_single_test(logger,
project,
reports_dir,
test,
output_test_names=True):
additional_integrationtest_commandline_text = project.get_property(
"integrationtest_additional_commandline", "")
if additional_integrationtest_commandline_text:
additional_integrationtest_commandline = tuple(
additional_integrationtest_commandline_text.split(" "))
else:
additional_integrationtest_commandline = ()
name, _ = os.path.splitext(os.path.basename(test))
if output_test_names:
logger.info("Running integration test %s", name)
env = prepare_environment(project)
test_time = Timer.start()
command_and_arguments = (sys.executable, test)
command_and_arguments += additional_integrationtest_commandline
report_file_name = os.path.join(reports_dir, name)
error_file_name = report_file_name + ".err"
return_code = execute_command(command_and_arguments,
report_file_name,
env,
error_file_name=error_file_name)
test_time.stop()
report_item = {
"test": name,
"test_file": test,
"time": test_time.get_millis(),
"success": True
}
if return_code != 0:
logger.error("Integration test failed: %s", test)
report_item["success"] = False
if project.get_property("verbose") or project.get_property(
"integrationtest_always_verbose"):
print_file_content(report_file_name)
print_text_line()
print_file_content(error_file_name)
report_item['exception'] = ''.join(
read_file(error_file_name)).replace('\'', '')
elif project.get_property("integrationtest_always_verbose"):
print_file_content(report_file_name)
print_text_line()
print_file_content(error_file_name)
return report_item
def pdoc_compile_docs(project, logger):
logger.info("Generating pdoc documentation")
if not project.get_property("pdoc_module_name"):
raise BuildFailedException("'pdoc_module_name' must be specified")
pdoc_command_args = project.get_property("pdoc_command_args", [])
pdoc_output_dir = project.expand_path("$pdoc_output_dir")
command_and_arguments = ["pdoc"] + pdoc_command_args
if "--html" in pdoc_command_args:
command_and_arguments += ["--html-dir", pdoc_output_dir]
command_and_arguments += [project.get_property("pdoc_module_name")]
source_directory = project.expand_path("$pdoc_source")
environment = {"PYTHONPATH": source_directory,
"PATH": os.environ["PATH"]}
logger.debug("Executing pdoc as: %s", command_and_arguments)
execute_command(command_and_arguments, outfile_name=project.expand_path("$dir_reports", "pdoc"), env=environment,
cwd=pdoc_output_dir)
def _install_external_plugin(name, logger):
if not name.startswith(PYPI_PLUGIN_PROTOCOL):
message = "Only plugins starting with '{0}' are currently supported"
raise MissingPluginException(name, message.format(PYPI_PLUGIN_PROTOCOL))
plugin_name_on_pypi = name.replace(PYPI_PLUGIN_PROTOCOL, "")
log_file = tempfile.NamedTemporaryFile(delete=False).name
result = execute_command(
"pip install {0}".format(plugin_name_on_pypi), log_file, error_file_name=log_file, shell=True
)
if result != 0:
logger.error("The following pip error was encountered:\n" + "".join(read_file(log_file)))
message = "Failed to install from PyPI".format(plugin_name_on_pypi)
raise MissingPluginException(name, message)
def install_dependency(logger, project, dependency):
logger.info("Installing dependency '%s'%s", dependency.name, " from %s" % dependency.url if dependency.url else "")
log_file = project.expand_path("$dir_install_logs", dependency.name)
pip_command_line = "pip install {0}'{1}'".format(build_pip_install_options(project), as_pip_argument(dependency))
exit_code = execute_command(pip_command_line, log_file, shell=True)
if exit_code != 0:
if project.get_property("verbose"):
print_file_content(log_file)
raise BuildFailedException("Unable to install dependency '%s'.", dependency.name)
else:
raise BuildFailedException("Unable to install dependency '%s'. See %s for details.",
dependency.name,
log_file)
def sphinx_generate(project, logger):
"""Runs sphinx-build against rst sources for the given project.
"""
logger.info("Running sphinx-build")
log_file = project.expand_path("$dir_target/reports/{0}".format("sphinx-build"))
build_command = get_sphinx_build_command(project)
if project.get_property("verbose"):
logger.info(build_command)
exit_code = execute_command(build_command, log_file, shell=True)
if exit_code != 0:
raise BuildFailedException("Sphinx build command failed. See %s for details.", log_file)
def sphinx_generate(project, logger):
"""Runs sphinx-build against rst sources for the given project.
"""
logger.info("Running sphinx-build")
log_file = project.expand_path("$dir_target/reports/{0}".format("sphinx-build"))
build_command = get_sphinx_build_command(project)
if project.get_property("verbose"):
logger.info(build_command)
exit_code = execute_command(build_command, log_file, shell=True)
if exit_code != 0:
raise BuildFailedException("Sphinx build command failed. See %s for details.", log_file)
def _install_external_plugin(name, logger):
if not name.startswith(PYPI_PLUGIN_PROTOCOL):
message = "Only plugins starting with '{0}' are currently supported"
raise MissingPluginException(name,
message.format(PYPI_PLUGIN_PROTOCOL))
plugin_name_on_pypi = name.replace(PYPI_PLUGIN_PROTOCOL, "")
log_file = tempfile.NamedTemporaryFile(delete=False).name
result = execute_command('pip install {0}'.format(plugin_name_on_pypi),
log_file,
error_file_name=log_file,
shell=True)
if result != 0:
logger.error("The following pip error was encountered:\n" +
"".join(read_file(log_file)))
message = "Failed to install from PyPI".format(plugin_name_on_pypi)
raise MissingPluginException(name, message)
def execute_tool_on_source_files(project, name, command_and_arguments, logger=None, include_test_sources=False):
files = discover_affected_files(include_test_sources, project)
command = as_list(command_and_arguments) + [f for f in files]
report_file = project.expand_path("$dir_reports/{0}".format(name))
execution_result = execute_command(command, report_file), report_file
report_file = execution_result[1]
report_lines = read_file(report_file)
if project.get_property(name + "_verbose_output") and logger:
log_report(logger, name, report_lines)
return execution_result
def install_dependency(logger, project, dependency):
logger.info("Installing dependency '%s'%s", dependency.name,
" from %s" % dependency.url if dependency.url else "")
log_file = project.expand_path("$dir_install_logs", dependency.name)
pip_command_line = "pip install {0}'{1}'".format(
build_pip_install_options(project, dependency),
as_pip_argument(dependency))
exit_code = execute_command(pip_command_line, log_file, shell=True)
if exit_code != 0:
if project.get_property("verbose"):
print_file_content(log_file)
raise BuildFailedException("Unable to install dependency '%s'.",
dependency.name)
else:
raise BuildFailedException(
"Unable to install dependency '%s'. See %s for details.",
dependency.name, log_file)
def run_single_test(logger, project, reports_dir, test, output_test_names=True):
additional_integrationtest_commandline_text = project.get_property("integrationtest_additional_commandline", "")
if additional_integrationtest_commandline_text:
additional_integrationtest_commandline = tuple(additional_integrationtest_commandline_text.split(" "))
else:
additional_integrationtest_commandline = ()
name, _ = os.path.splitext(os.path.basename(test))
if output_test_names:
logger.info("Running integration test %s", name)
env = prepare_environment(project)
test_time = Timer.start()
command_and_arguments = (sys.executable, test)
command_and_arguments += additional_integrationtest_commandline
report_file_name = os.path.join(reports_dir, name)
error_file_name = report_file_name + ".err"
return_code = execute_command(
command_and_arguments, report_file_name, env, error_file_name=error_file_name)
test_time.stop()
report_item = {
"test": name,
"test_file": test,
"time": test_time.get_millis(),
"success": True
}
if return_code != 0:
logger.error("Integration test failed: %s", test)
report_item["success"] = False
if project.get_property("verbose") or project.get_property("integrationtest_always_verbose"):
print_file_content(report_file_name)
print_text_line()
print_file_content(error_file_name)
report_item['exception'] = ''.join(read_file(error_file_name)).replace('\'', '')
elif project.get_property("integrationtest_always_verbose"):
print_file_content(report_file_name)
print_text_line()
print_file_content(error_file_name)
return report_item
def install_dependency(logger, project, dependency):
url = getattr(dependency, "url", None)
logger.info("Installing dependency '%s'%s", dependency.name,
" from %s" % url if url else "")
log_file = project.expand_path("$dir_install_logs", dependency.name)
log_file = re.sub(r'<|>|=', '_', log_file)
target_dir = None
try:
target_dir = project.get_property(
"install_dependencies_local_mapping")[dependency.name]
except KeyError:
pass
pip_command_line = list()
pip_command_line.append(PIP_EXECUTABLE)
pip_command_line.append("install")
pip_command_line.extend(
build_pip_install_options(
project.get_property("install_dependencies_index_url"),
project.get_property("install_dependencies_extra_index_url"),
project.get_property("install_dependencies_upgrade"),
project.get_property("install_dependencies_insecure_installation"),
True if url else False, target_dir,
project.get_property("verbose")))
pip_command_line.extend(as_pip_install_target(dependency))
logger.debug("Invoking pip: %s", pip_command_line)
exit_code = execute_command(pip_command_line,
log_file,
env=os.environ,
shell=False)
if exit_code != 0:
if project.get_property("verbose"):
print_file_content(log_file)
raise BuildFailedException("Unable to install dependency '%s'.",
dependency.name)
else:
raise BuildFailedException(
"Unable to install dependency '%s'. See %s for details.",
getattr(dependency, "name", dependency), log_file)
def execute_command(self,
command_and_arguments,
outfile_name=None,
env=None,
cwd=None,
error_file_name=None,
shell=False,
no_path_search=False,
inherit_env=True):
environ = self.environ if inherit_env else {}
if env:
environ.update(env)
return execute_command(command_and_arguments,
outfile_name=outfile_name,
env=environ,
cwd=cwd,
error_file_name=error_file_name,
shell=shell,
no_path_search=no_path_search,
logger=self.logger)
def pip_install(install_targets, index_url=None, extra_index_url=None, upgrade=False, insecure_installs=None,
force_reinstall=False, target_dir=None, verbose=False, logger=None, outfile_name=None,
error_file_name=None, env=None, cwd=None):
pip_command_line = list()
pip_command_line.append(PIP_EXECUTABLE)
pip_command_line.append("install")
pip_command_line.extend(build_pip_install_options(index_url,
extra_index_url,
upgrade,
insecure_installs,
force_reinstall,
target_dir,
verbose
))
for install_target in as_list(install_targets):
pip_command_line.extend(as_pip_install_target(install_target))
if logger:
logger.debug("Invoking pip: %s", pip_command_line)
return execute_command(pip_command_line, outfile_name=outfile_name, env=env, cwd=cwd,
error_file_name=error_file_name, shell=False)
def install_dependency(logger, project, dependency):
url = getattr(dependency, "url", None)
logger.info("Installing dependency '%s'%s", dependency.name,
" from %s" % url if url else "")
log_file = project.expand_path("$dir_install_logs", dependency.name)
if sys.platform.startswith("win"):
# we can't use quotes on windows
pip_dependency = as_pip_argument(dependency)
else:
# on linux we need quotes because version pinning (>=) would be an IO redirect
pip_dependency = "'{0}'".format(as_pip_argument(dependency))
pip_command_line = "pip install {0}{1}".format(build_pip_install_options(project, pip_dependency), pip_dependency)
exit_code = execute_command(pip_command_line, log_file, shell=True)
if exit_code != 0:
if project.get_property("verbose"):
print_file_content(log_file)
raise BuildFailedException("Unable to install dependency '%s'.", dependency.name)
else:
raise BuildFailedException("Unable to install dependency '%s'. See %s for details.",
getattr(dependency, "name", dependency),
log_file)
def install_dependency(logger, project, dependency):
url = getattr(dependency, "url", None)
logger.info("Installing dependency '%s'%s", dependency.name,
" from {0!s}".format(url) if url else "")
log_file = project.expand_path("$dir_install_logs", dependency.name)
log_file = re.sub(r'<|>|=', '_', log_file)
target_dir = None
try:
target_dir = project.get_property("install_dependencies_local_mapping")[dependency.name]
except KeyError:
pass
pip_command_line = list()
pip_command_line.extend(PIP_EXEC_STANZA)
pip_command_line.append("install")
pip_command_line.extend(build_pip_install_options(project.get_property("install_dependencies_index_url"),
project.get_property("install_dependencies_extra_index_url"),
project.get_property("install_dependencies_upgrade"),
project.get_property(
"install_dependencies_insecure_installation"),
True if url else False,
target_dir,
project.get_property("verbose"),
project.get_property("install_dependencies_trusted_host")
))
pip_command_line.extend(as_pip_install_target(dependency))
logger.debug("Invoking pip: %s", pip_command_line)
exit_code = execute_command(pip_command_line, log_file, env=os.environ, shell=False)
if exit_code != 0:
if project.get_property("verbose"):
print_file_content(log_file)
raise BuildFailedException("Unable to install dependency '%s'.", dependency.name)
else:
raise BuildFailedException("Unable to install dependency '%s'. See %s for details.",
getattr(dependency, "name", dependency),
log_file)
def checkstyle(project, logger):
#pylint: disable=R0914
logger.info("Executing pylint on project sources.")
src_dir = project.get_property('dir_source_main_python')
py_files = [e for e in discover_files_matching(src_dir, '*.py')] #pylint: disable=R1721
cmd = as_list('pylint', '--rcfile=.pylintrc', py_files)
out_fname = project.expand_path("$dir_reports/{}".format('pylint'))
err_fname = '{}.err'.format(out_fname)
exit_code = execute_command(cmd, out_fname)
report = read_file(out_fname)
error = read_file(err_fname)
if exit_code != 0 and error:
msg = 'Errors occurred while running pylint, check {}'.format(
err_fname)
logger.error(msg)
raise BuildFailedException(msg)
logger.warn(''.join(report))
try:
# get pylint score from the message
# e.g. "Your code has been rated at 10.00/10 (previous run: 10.00/10, +0.00)"
pattern = r'(\d{0,2}\.\d{2})/10'
pat = re.compile(pattern)
res = re.search(pat, report[-2])
score = float(res.group(1))
except Exception as _:
traceback.print_exc()
raise BuildFailedException('fail to parse pylint score.')
cutoff = 8.5
if project.get_property('break_build') and score < cutoff:
msg = 'Fail to pass (score={:.2f}, cutoff={:.2f})'.format(
score, cutoff)
raise BuildFailedException(msg)
logger.info('Pass (score={:.2f}, cutoff={:.2f})'.format(score, cutoff))
def _install_external_plugin(name, version, logger, plugin_module_name):
if not name.startswith(PYPI_PLUGIN_PROTOCOL) and not name.startswith(VCS_PLUGIN_PROTOCOL):
message = "Only plugins starting with '{0}' are currently supported"
raise MissingPluginException(name, message.format((PYPI_PLUGIN_PROTOCOL, VCS_PLUGIN_PROTOCOL)))
if name.startswith(PYPI_PLUGIN_PROTOCOL):
pip_package = name.replace(PYPI_PLUGIN_PROTOCOL, "")
if version:
pip_package += str(version)
elif name.startswith(VCS_PLUGIN_PROTOCOL):
pip_package = name.replace(VCS_PLUGIN_PROTOCOL, "")
with tempfile.NamedTemporaryFile(delete=False) as log_file:
log_file_name = log_file.name
install_cmd = ['pip', 'install', '--upgrade', pip_package]
result = execute_command(install_cmd,
log_file_name,
error_file_name=log_file_name,
cwd=".",
shell=False)
if result != 0:
logger.error("The following pip error was encountered:\n" + "".join(read_file(log_file_name)))
message = "Failed to install plugin from {0}".format(pip_package)
raise MissingPluginException(name, message)
def debian(project, logger):
log_file = project.expand_path("$dir_target/reports/debian")
exit_code = execute_command("./setup.py --command-packages=stdeb.command sdist_dsc --dist-dir %s bdist_deb" % project.expand_path("$dir_target/deb_dist"), log_file, cwd=project.expand_path("$dir_dist"), shell=True)
if exit_code != 0:
raise BuildFailedException("debian build command failed. See %s for details.", log_file)
pass
def generate_pdf(graph_file, pdf_file):
execute_command(["dot", "-Tpdf", graph_file], pdf_file)
def generate_graph(report_file, graph_file):
execute_command(["sfood-graph", report_file], graph_file)
def generate_graph(report_file, graph_file):
execute_command(["sfood-graph", report_file], graph_file)
def generate_pdf(graph_file, pdf_file):
execute_command(["dot", "-Tpdf", graph_file], pdf_file)