def _venv_python_executable(env_dir, platform):
"""Binary Python executable for a specific virtual environment"""
if is_windows(platform):
candidate = jp(env_dir, "Scripts", _venv_python_exename)
# On Windows python.exe could be in PythonXY/ or venv/Scripts/
if not os.path.exists(candidate):
alternative = jp(env_dir, _venv_python_exename)
if os.path.exists(alternative):
candidate = alternative
else:
candidate = jp(env_dir, "bin", _venv_python_exename)
return np(candidate)
def test_should_generate_sphinx_build_command_builder_dir(self):
self.project.set_property("sphinx_config_path", "docs/")
self.project.set_property("sphinx_source_dir", "docs/")
self.project.set_property("sphinx_output_dir", "docs/_build/")
self.project.set_property("sphinx_doc_builder", ["JSONx", "pdf"])
sphinx_build_command = get_sphinx_build_command(
self.project, Mock(), "JSONx")
self.assertEqual(sphinx_build_command, [
ANY, "-b", "JSONx",
np(jp(self.project.basedir, "docs/")),
np(jp(self.project.basedir, "docs/_build/JSONx"))
])
def test_running_plugin_fails(
self,
read_file_mock,
report_mock,
find_files_mock,
command_mock,
tail_mock,
):
project = Project('.')
project.set_property('verbose', False)
project.set_property('dir_source_main_python', 'python')
project.set_property('dir_source_main_scripts', 'scripts')
logger = Mock()
reactor = Mock()
reactor.python_env_registry = {}
reactor.python_env_registry["pybuilder"] = pyb_env = Mock()
reactor.pybuilder_venv = pyb_env
pyb_env.environ = {}
pyb_env.executable = ["a/b"]
execute_mock = pyb_env.execute_command = Mock()
command_mock.return_value = ['cram']
find_files_mock.return_value = ['test1.cram', 'test2.cram']
report_mock.return_value = 'report_file'
read_file_mock.return_value = ['test failes for file', '# results']
execute_mock.return_value = 1
tail_mock.return_value = "tail data"
self.assertRaises(BuildFailedException, run_cram_tests, project,
logger, reactor)
execute_mock.assert_called_once_with(
['a/b', 'cram', 'test1.cram', 'test2.cram'],
'report_file',
error_file_name='report_file',
env={
'PYTHONPATH': np(jp(project.basedir, 'python')) + pathsep,
'PATH': np(jp(project.basedir, 'scripts')) + pathsep
})
expected_info_calls = [
call('Running Cram command line tests'),
]
expected_error_calls = [
call(
'Cram tests failed! See report_file for full details:\ntail data'
),
]
self.assertEqual(expected_info_calls, logger.info.call_args_list)
self.assertEqual(expected_error_calls, logger.error.call_args_list)
def test_expand_path_should_return_expanded_path_and_additional_parts_when_additional_parts_are_given(
self):
self.project.set_property("spam", "spam")
self.project.set_property("eggs", "eggs")
self.assertEqual(
np(jp(self.project.basedir, "spam", "eggs", "foo", "bar")),
self.project.expand_path("$spam/$eggs", "foo", "bar"))
def _optimize_omit_module_files(module_files, omit_module_files):
# This is a stupid implementation but given the number of entries it'll do (until it won't)
omf_lookup = set(omit_module_files)
result_omit = set()
def has_modules_in_dir_not_omitted(od):
for mf in module_files:
if mf.startswith(od) and mf not in omf_lookup:
return True
for omit in omit_module_files:
already_omitted = False
for ro in result_omit:
if fnmatch(omit, ro):
already_omitted = True
break
if already_omitted:
continue
prev_omit = omit
omit_dir = dirname(omit)
while prev_omit != omit_dir:
if has_modules_in_dir_not_omitted(omit_dir):
result_omit.add(
jp(prev_omit, "*") if prev_omit != omit else prev_omit)
break
prev_omit = omit_dir
omit_dir = dirname(omit_dir)
return sorted(result_omit)
def list_modules(project):
source_path = project.expand_path("$" + PYTHON_SOURCES_PROPERTY)
for potential_module_file in listdir(source_path):
potential_module_path = np(jp(source_path, potential_module_file))
if isfile(potential_module_path) and potential_module_file.endswith(
".py"):
yield potential_module_file[:-len(".py")]
def list_scripts(project):
scripts_dir = project.expand_path("$" + SCRIPTS_SOURCES_PROPERTY)
if not exists(scripts_dir):
return
for script in listdir(scripts_dir):
if isfile(jp(scripts_dir,
script)) and not HIDDEN_FILE_NAME_PATTERN.match(script):
yield np(script)
def _list_top_level_packages(vendorized_path):
vendorized_packages = set()
for name in _list_metadata_dirs(vendorized_path):
top_level_path = jp(vendorized_path, name, "top_level.txt")
if exists(top_level_path):
with open(top_level_path) as f:
for p in filter(None, map(lambda line: line.strip(), f)):
vendorized_packages.add(p)
return vendorized_packages
def test_find_files(self, discover_mock):
project = Project('.')
project.set_property('dir_source_cmdlinetest', np('any/dir'))
project.set_property('cram_test_file_glob', '*.t')
expected = [np(jp(project.basedir, './any/dir/test.cram'))]
discover_mock.return_value = expected
received = _find_files(project)
self.assertEqual(expected, received)
discover_mock.assert_called_once_with(np('any/dir'), '*.t')
def list_modules(project):
source_path = project.expand_path("$" + PYTHON_SOURCES_PROPERTY)
result = []
for potential_module_file in listdir(source_path):
potential_module_path = np(jp(source_path, potential_module_file))
if isfile(potential_module_path) and potential_module_file.endswith(".py"):
result.append(potential_module_file[:-3])
return sorted(result)
def test_should_generate_sphinx_build_command_debug(self):
self.project.set_property("sphinx_config_path", "docs/")
self.project.set_property("sphinx_source_dir", "docs/")
self.project.set_property("sphinx_output_dir", "docs/_build/")
self.project.set_property("sphinx_doc_builder", "JSONx")
logger = Mock()
logger.level = 1
logger.DEBUG = 1
sphinx_build_command = get_sphinx_build_command(
self.project, logger, "JSONx")
self.assertEqual(sphinx_build_command, [
ANY, "-b", "JSONx", "-vvvv",
np(jp(self.project.basedir, "docs/")),
np(jp(self.project.basedir, "docs/_build/"))
])
def vendorize(project, reactor, logger):
target_dir = project.expand_path("$vendorize_target_dir")
packages = [
Dependency(p)
for p in as_list(project.get_property("vendorize_packages"))
]
clean = project.get_property("vendorize_clean_target_dir")
logfile = project.expand_path("$dir_logs", "vendorize.log")
logger.info("Will vendorize packages %r into %r%s", packages, target_dir,
" (cleaning)" if clean else "")
if clean:
rmtree(target_dir, ignore_errors=False)
makedirs(target_dir, exist_ok=True)
reactor.pybuilder_venv.install_dependencies(packages,
install_log_path=logfile,
package_type="vendorized",
target_dir=target_dir,
ignore_installed=True)
# Vendorize
_vendorize(target_dir)
# Cleanup globs
for g in project.get_property("vendorize_cleanup_globs"):
for p in iglob(jp(target_dir, g)):
if isdir(p):
rmtree(p)
else:
unlink(p)
# Populate names after cleanup
cleaned_up_packages = _list_top_level_packages(target_dir)
with open(jp(target_dir, "__init__.py"), "wt") as init_py:
init_py.write("__names__ = %r\n" % sorted(cleaned_up_packages))
# Cleanup metadata
for p in _list_metadata_dirs(target_dir):
if isdir(p):
rmtree(p)
else:
unlink(p)
def include_file(self, package_name, filename):
package_name = package_name or ""
if not filename or filename.strip() == "":
raise ValueError("Missing argument filename.")
full_filename = np(jp(package_name.replace(".", sep), filename))
self._manifest_include(full_filename)
self._add_package_data(package_name, filename)
def list_scripts(project):
scripts_dir = project.expand_path("$" + SCRIPTS_SOURCES_PROPERTY)
result = []
if not exists(scripts_dir):
return result
for script in listdir(scripts_dir):
if isfile(jp(scripts_dir, script)) and not HIDDEN_FILE_NAME_PATTERN.match(script):
result.append(np(script))
return sorted(result)
def test_should_add_pattern_to_list_of_included_filed_for_package_spam(
self, walk):
walk.return_value = [
[
jp(self.project.basedir, "spam"), ("foo", "bar"),
("bacon.eggs", "bacon.noeggs")
],
]
self.project.include_directory("spam", "*.eggs")
self.assertEqual({"spam": ["bacon.eggs"]}, self.project.package_data)
def use_bldsup(build_support_dir="bldsup"):
"""Specify a local build support directory for build specific extensions.
use_plugin(name) and import will look for python modules in BUILD_SUPPORT_DIR.
WARNING: The BUILD_SUPPORT_DIR must exist and must have an __init__.py file in it.
"""
assert isdir(build_support_dir), "use_bldsup('{0}'): The {0} directory must exist!".format(
build_support_dir)
init_file = jp(build_support_dir, "__init__.py")
assert isfile(init_file), "use_bldsup('{0}'): The {1} file must exist!".format(build_support_dir, init_file)
sys.path.insert(0, build_support_dir)
def test_get_sphinx_apidoc_command_enabled(self):
sphinx_mock = Mock()
sys.modules["sphinx"] = sphinx_mock
try:
sphinx_mock.version_info = (1, 2, 3, 4, 5)
self.project.set_property("sphinx_run_apidoc", True)
self.project.set_property("dir_target", "dir_target")
self.project.set_property("dir_source_main_python", "dir_source")
self.project.set_property("sphinx_project_name", "project_name")
self.assertEqual(
get_sphinx_apidoc_command(self.project, self.reactor), [
ANY, "-H", "project_name", "-o",
np(jp(self.project.basedir,
"dir_target/sphinx_pyb/apidoc")),
np(jp(self.project.basedir, "dir_source"))
])
finally:
del sys.modules["sphinx"]
def test_should_generate_sphinx_quickstart_command_with_project_properties(
self):
self.project.set_property("sphinx_doc_author", "bar")
self.project.set_property("sphinx_project_name", "foo")
self.project.set_property("sphinx_project_version", "3")
self.project.set_property("sphinx_source_dir", "docs/")
sphinx_quickstart_command = get_sphinx_quickstart_command(self.project)
self.assertEqual(sphinx_quickstart_command, [
ANY, "-q", "-p", "foo", "-a", "bar", "-v", "3",
np(jp(self.project.basedir, "docs/"))
])
def __init__(self,
env_dir,
reactor,
platform=None,
install_log_name="install.log"):
self._env_dir = env_dir
self._reactor = reactor
self._platform = platform or sys.platform
self._long_desc = "Unpopulated"
self._site_paths = None
self._venv_symlinks = os.name == "posix"
self._install_log_path = jp(self._env_dir, install_log_name)
self._populated = False
def test_running_plugin_cram_from_target(self, read_file_mock, report_mock,
find_files_mock, command_mock):
project = Project('.')
project.set_property('cram_run_test_from_target', True)
project.set_property('dir_dist', 'python')
project.set_property('dir_dist_scripts', 'scripts')
project.set_property('verbose', False)
project._plugin_env = {}
logger = Mock()
reactor = Mock()
reactor.python_env_registry = {}
reactor.python_env_registry["pybuilder"] = pyb_env = Mock()
reactor.pybuilder_venv = pyb_env
pyb_env.environ = {}
pyb_env.executable = ["a/b"]
execute_mock = pyb_env.execute_command = Mock()
command_mock.return_value = ['cram']
find_files_mock.return_value = ['test1.cram', 'test2.cram']
report_mock.return_value = 'report_file'
read_file_mock.return_value = ['test failes for file', '# results']
execute_mock.return_value = 0
run_cram_tests(project, logger, reactor)
execute_mock.assert_called_once_with(
['a/b', 'cram', 'test1.cram', 'test2.cram'],
'report_file',
error_file_name='report_file',
env={
'PYTHONPATH': np(jp(project.basedir, 'python')) + pathsep,
'PATH': np(jp(project.basedir, 'python/scripts')) + pathsep
})
expected_info_calls = [
call('Running Cram command line tests'),
call('Cram tests were fine'),
call('results'),
]
self.assertEqual(expected_info_calls, logger.info.call_args_list)
def test_sphinx_pyb_quickstart_generate(self, symlink, mkdir, exists,
rmtree, open):
exists.return_value = False
self.project.set_property("sphinx_source_dir", "sphinx_source_dir")
self.project.set_property("sphinx_config_path", "sphinx_config_path")
self.project.set_property("dir_target", "dir_target")
self.project.set_property("dir_source_main_python", "dir_source")
self.project.set_property("sphinx_project_name", "project_name")
sphinx_pyb_quickstart_generate(self.project, Mock(), self.reactor)
open().__enter__().write.assert_called_with("""\
# Automatically generated by PyB
import sys
from os.path import normcase as nc, normpath as np, join as jp, dirname, exists
sphinx_pyb_dir = nc(np(jp(dirname(__file__) if __file__ else '.', %r)))
sphinx_pyb_module = 'sphinx_pyb_conf'
sphinx_pyb_module_file = nc(np(jp(sphinx_pyb_dir, sphinx_pyb_module + '.py')))
sys.path.insert(0, sphinx_pyb_dir)
if not exists(sphinx_pyb_module_file):
raise RuntimeError("No PyB-based Sphinx configuration found in " + sphinx_pyb_module_file)
from sphinx_pyb_conf import *
# Overwrite PyB-settings here statically if that's the thing that you want
""" % relpath(np(jp(self.project.basedir, "../dir_target/sphinx_pyb")),
self.project.basedir))
symlink.assert_called_with(relpath(
np(jp(self.project.basedir, "../dir_target/sphinx_pyb/apidoc")),
self.project.basedir),
np(
jp(self.project.basedir,
"sphinx_source_dir/apidoc")),
target_is_directory=True)
def test_iglob(self):
self.assertEqual(list(iglob(jp(self.tmp_dir, "*.py"))),
[jp(self.tmp_dir, "x.py")])
self.assertEqual(
list(iglob(jp(self.tmp_dir, "**", "*.py"), recursive=True)), [
jp(self.tmp_dir, "x.py"),
jp(self.tmp_dir, "a", "y.py"),
jp(self.tmp_dir, "a", "b", "z.py")
])
def _override_python_env_for_coverage(current_python_env, coverage_config,
source_path, omit_patterns):
import coverage as cov_module
cov_parent_dir = ap(jp(dirname(cov_module.__file__), ".."))
new_python_env = copy.copy(current_python_env)
new_python_env.overwrite(
"executable",
tuple(
current_python_env.executable + [
ap(
jp(
dirname(sys.modules[_override_python_env_for_coverage.
__module__].__file__),
"_coverage_shim.py")),
repr(
{
"cov_parent_dir": cov_parent_dir,
"cov_kwargs": coverage_config,
"cov_source_path": source_path,
"cov_omit_patterns": omit_patterns,
}, )
], ))
return new_python_env
def _vendorize(vendorized_path):
vendorized_packages = _list_top_level_packages(vendorized_path)
for root, dirs, files in walk(vendorized_path):
for py_path in files:
if not py_path.endswith(".py"):
continue
py_path = np(jp(root, py_path))
with open(py_path, "rt") as source_file:
source = source_file.read()
parsed_ast = ast.parse(source, filename=py_path)
it = ImportTransformer(py_path, source, vendorized_path,
vendorized_packages, [])
it.visit(parsed_ast)
with open(py_path, "wt") as source_file:
source_file.write(it.transformed_source)
return vendorized_packages
def verify_project_directory(project_directory, project_descriptor):
project_directory = np(project_directory)
if not os.path.exists(project_directory):
raise PyBuilderException("Project directory does not exist: %s", project_directory)
if not os.path.isdir(project_directory):
raise PyBuilderException("Project directory is not a directory: %s", project_directory)
project_descriptor_full_path = jp(project_directory, project_descriptor)
if not os.path.exists(project_descriptor_full_path):
raise PyBuilderException("Project directory does not contain descriptor file: %s",
project_descriptor_full_path)
if not os.path.isfile(project_descriptor_full_path):
raise PyBuilderException("Project descriptor is not a file: %s", project_descriptor_full_path)
return project_directory, project_descriptor_full_path
def _filter_covered_modules(logger, module_names, module_file_suffixes,
modules_exceptions, source_path):
result_module_names = []
result_module_files = []
omit_module_files = []
module_files = []
for idx, module_name in enumerate(module_names):
module_file = nc(jp(source_path, module_file_suffixes[idx]))
module_files.append(module_file)
skip_module = False
for module_exception in modules_exceptions:
if module_exception.endswith("*"):
if module_name.startswith(module_exception[:-1]):
skip_module = True
break
else:
if module_name == module_exception:
skip_module = True
break
if not skip_module:
with open(module_file, "rb") as f:
try:
ast.parse(f.read(), module_file)
except SyntaxError:
logger.warn(
"Unable to parse module %r (file %r) due to syntax error and will be excluded"
% (module_name, module_file))
skip_module = True
if skip_module:
logger.debug("Module %r (file %r) was excluded", module_name,
module_file)
omit_module_files.append(module_file)
else:
result_module_names.append(module_name)
result_module_files.append(module_file)
omit_module_files = _optimize_omit_module_files(module_files,
omit_module_files)
return result_module_names, result_module_files, omit_module_files
def _setup_plugin_directory(self, reset_plugins):
per = self.python_env_registry
system_env = per["system"]
plugin_dir = self._plugin_dir = np(
jp(self.project.basedir, ".pybuilder", "plugins",
system_env.versioned_dir_name))
self.logger.debug("Setting up plugins VEnv at '%s'%s", plugin_dir,
" (resetting)" if reset_plugins else "")
plugin_env = per["pybuilder"] = PythonEnv(
plugin_dir, self).create_venv(with_pip=True,
symlinks=system_env.venv_symlinks,
upgrade=True,
clear=(reset_plugins
or system_env.is_pypy),
offline=self.project.offline)
prepend_env_to_path(plugin_env, sys.path)
patch_mp_pyb_env(plugin_env)
def include_directory(self, package_path, patterns_list, package_root=""):
if not package_path or package_path.strip() == "":
raise ValueError("Missing argument package_path.")
if not patterns_list:
raise ValueError("Missing argument patterns_list.")
patterns_list = as_list(patterns_list)
package_name = PATH_SEP_RE.sub(".", package_path)
self._manifest_include_directory(package_path, patterns_list)
package_full_path = self.expand_path(package_root, package_path)
for root, dirnames, filenames in os.walk(package_full_path):
filenames = list(fnmatch.filter(filenames, pattern) for pattern in patterns_list)
for filename in itertools.chain.from_iterable(filenames):
full_path = np(jp(root, filename))
relative_path = relpath(full_path, package_full_path)
self._add_package_data(package_name, relative_path)
def test_should_write_pycharm_file(self, os, mock_open):
project = Project('basedir', name='pybuilder')
project.set_property('dir_source_main_python', 'src/main/python')
project.set_property('dir_source_unittest_python',
'src/unittest/python')
project.set_property('dir_source_integrationtest_python',
'src/integrationtest/python')
project.set_property('dir_target', 'build')
mock_open.return_value = MagicMock(spec=TYPE_FILE)
os.path.join.side_effect = lambda first, second: first + sep + second
pycharm_generate(project, Mock())
mock_open.assert_called_with(
np(jp(project.basedir, '.idea/pybuilder.iml')), 'w')
metadata_file = mock_open.return_value.__enter__.return_value
metadata_file.write.assert_called_with(
"""<?xml version="1.0" encoding="UTF-8"?>
<!-- This file has been generated by the PyBuilder PyCharm Plugin -->
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src/main/python" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/unittest/python" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/src/integrationtest/python" isTestSource="true" />
<excludeFolder url="file://$MODULE_DIR$/.pybuilder" />
<excludeFolder url="file://$MODULE_DIR$/build" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="PyDocumentationSettings">
<option name="myDocStringFormat" value="Plain" />
</component>
<component name="TestRunnerService">
<option name="projectConfiguration" value="Unittests" />
<option name="PROJECT_TEST_RUNNER" value="Unittests" />
</component>
</module>""")
def test_expand_path_should_return_expanded_path(self):
self.project.set_property("spam", "spam")
self.project.set_property("eggs", "eggs")
self.assertEqual(np(jp(self.project.basedir, "spam", "eggs")),
self.project.expand_path("$spam/$eggs"))