def test_engine_run_args_mode_localsrcs_overload_package(self):
with prepare_workdir() as dir_a, prepare_workdir() as dir_b:
config = {
'local_sources': [
# --local-sources set to a specific path
dir_a,
# overriding path for the `multiple-b` package
'multiple-b@{}'.format(dir_b),
],
}
with prepare_testenv(config=config, template='multiple') as engine:
self.assertTrue(isinstance(engine.opts.local_srcs, dict))
self.assertTrue(GBL_LSRCS in engine.opts.local_srcs)
self.assertEqual(engine.opts.local_srcs[GBL_LSRCS], dir_a)
self.assertTrue('multiple-b' in engine.opts.local_srcs)
self.assertEqual(engine.opts.local_srcs['multiple-b'], dir_b)
def test_pkg_pipeline_pkgenv_generic(self):
pkg_name = 'test-generic'
with prepare_testenv(template='pkg-env') as engine:
pkgs = engine.pkgman.load([pkg_name])
pipeline = RelengPackagePipeline(engine, engine.opts, {})
for pkg in pkgs:
processed = pipeline.process(pkg)
self.assertTrue(processed)
# verify package-specific environment variables were set
self.assertTrue('TEST_GENERIC_BUILD_OUTPUT_DIR' in os.environ)
test_outdir = os.environ['TEST_GENERIC_BUILD_OUTPUT_DIR']
results = os.path.join(test_outdir, 'invoke-env.json')
self.assertTrue(os.path.exists(results))
with open(results, 'r') as f:
data = json.load(f)
# verify that all expected environment variables are set
expected_pkg_env_keys = [
'PKG_BUILD_BASE_DIR',
'PKG_BUILD_DIR',
'PKG_BUILD_OUTPUT_DIR',
'PKG_CACHE_DIR',
'PKG_CACHE_FILE',
'PKG_DEFDIR',
'PKG_NAME',
'PKG_REVISION',
'PKG_SITE',
'PKG_VERSION',
]
self.assertTrue(all(x in data for x in expected_pkg_env_keys))
# verify that all unexpected environment variables are not set
unexpected_pkg_env_keys = [
'PKG_INTERNAL',
]
self.assertTrue(all(x not in data for x in unexpected_pkg_env_keys))
# sanity check the contents of various environment variables
self.assertEqual(data['PKG_BUILD_DIR'], data['PKG_BUILD_BASE_DIR'])
self.assertEqual(data['PKG_NAME'], pkg_name)
self.assertEqual(data['PKG_REVISION'], 'test-revision')
self.assertEqual(data['PKG_SITE'], 'mocked-site')
self.assertEqual(data['PKG_VERSION'], 'test-version')
def test_site_url_fetch_archive_tar_invalid(self):
with httpd_context() as httpd:
host, port = httpd.server_address
site = 'http://{}:{}/test.tgz'.format(host, port)
httpd.rsp.append((200, b'not-an-archive'))
with prepare_testenv(template='minimal') as engine:
root_dir = engine.opts.root_dir
pkg_script = os.path.join(root_dir,
'package', 'minimal', 'minimal')
with open(pkg_script, 'a') as f:
f.write('MINIMAL_SITE="{}"\n'.format(site))
rv = engine.run()
self.assertFalse(rv)
def test_engine_run_args_mode_localsrcs_per_package(self):
with prepare_workdir() as dir_a, prepare_workdir() as dir_b:
config = {
'local_sources': [
# explicit path set for `multiple-a` package
'multiple-a@{}'.format(dir_a),
# explicit path set for `multiple-b` package
'multiple-b@{}'.format(dir_b),
],
}
with prepare_testenv(config=config, template='multiple') as engine:
self.assertTrue(isinstance(engine.opts.local_srcs, dict))
self.assertFalse(GBL_LSRCS in engine.opts.local_srcs)
self.assertTrue('multiple-a' in engine.opts.local_srcs)
self.assertEqual(engine.opts.local_srcs['multiple-a'], dir_a)
self.assertTrue(engine.opts.local_srcs)
self.assertTrue('multiple-b' in engine.opts.local_srcs)
self.assertEqual(engine.opts.local_srcs['multiple-b'], dir_b)
self.assertTrue(engine.opts.local_srcs)
def test_pkg_pipeline_pkgenv_internal(self):
pkg_name = 'test-internal'
with prepare_testenv(template='pkg-env') as engine:
pkgs = engine.pkgman.load([pkg_name])
pipeline = RelengPackagePipeline(engine, engine.opts, {})
for pkg in pkgs:
processed = pipeline.process(pkg)
self.assertTrue(processed)
# verify package-specific environment variables were set
self.assertTrue('TEST_INTERNAL_BUILD_OUTPUT_DIR' in os.environ)
test_outdir = os.environ['TEST_INTERNAL_BUILD_OUTPUT_DIR']
results = os.path.join(test_outdir, 'invoke-env.json')
self.assertTrue(os.path.exists(results))
with open(results, 'r') as f:
data = json.load(f)
# ensure that the internal environment flag is set when processing
# an internal-flagged package
self.assertTrue('PKG_INTERNAL' in data)
def test_pkg_pipeline_pkgenv_empty(self):
pkg_name = 'test-empty'
with prepare_testenv(template='pkg-env') as engine:
pkgs = engine.pkgman.load([pkg_name])
pipeline = RelengPackagePipeline(engine, engine.opts, {})
for pkg in pkgs:
processed = pipeline.process(pkg)
self.assertTrue(processed)
# verify package-specific environment variables were set
self.assertTrue('TEST_EMPTY_BUILD_OUTPUT_DIR' in os.environ)
test_outdir = os.environ['TEST_EMPTY_BUILD_OUTPUT_DIR']
results = os.path.join(test_outdir, 'invoke-env.json')
self.assertTrue(os.path.exists(results))
with open(results, 'r') as f:
data = json.load(f)
# verify that provided environment keys which may be empty if not
# set are indeed empty
self.assertEqual(data['PKG_SITE'], '')
self.assertEqual(data['PKG_VERSION'], '')
def test_site_url_fetch_archive_zip_valid(self):
with httpd_context() as httpd:
host, port = httpd.server_address
site = 'http://{}:{}/test.zip'.format(host, port)
httpd_assets = fetch_unittest_assets_dir('sample-files')
archive = os.path.join(httpd_assets, 'sample-files.zip')
with open(archive, 'rb') as f:
data = f.read()
httpd.rsp.append((200, data))
with prepare_testenv(template='minimal') as engine:
root_dir = engine.opts.root_dir
pkg_script = os.path.join(root_dir,
'package', 'minimal', 'minimal')
with open(pkg_script, 'a') as f:
f.write('MINIMAL_SITE="{}"\n'.format(site))
rv = engine.run()
self.assertTrue(rv)
def test_pkg_pipeline_licenses_multiple(self):
pkg_names = [
'test-a',
'test-b',
'test-c',
]
expected_licenses = [
2, # mocked BSD + MIT
None,
1, # mocked GPL
]
with prepare_testenv(template='licenses') as engine:
pkgs = engine.pkgman.load(pkg_names)
pipeline = RelengPackagePipeline(engine, engine.opts, {})
for pkg in pkgs:
if not pipeline.process(pkg):
break
for pkg, expected in zip(pkg_names, expected_licenses):
if expected is not None:
# a package with license information should be tracked
self.assertTrue(pkg in pipeline.license_files)
# verify we have expected license counts
licenses = pipeline.license_files[pkg]['files']
self.assertEqual(len(licenses), expected)
# verify that the license reference maps to a real file
for license_ in licenses:
self.assertTrue(os.path.exists(license_))
else:
# packages without license data should not provide any
# information
self.assertFalse(pkg in pipeline.license_files)
def test_engine_run_environ_cfg_dl_dir(self):
with prepare_workdir() as dl_dir:
os.environ['RELENG_DL_DIR'] = dl_dir
with prepare_testenv() as engine:
self.assertEqual(engine.opts.dl_dir, dl_dir)
def test_engine_run_environ_cfg_cache_dir(self):
with prepare_workdir() as cache_dir:
os.environ['RELENG_CACHE_DIR'] = cache_dir
with prepare_testenv() as engine:
self.assertEqual(engine.opts.cache_dir, cache_dir)
def test_pkg_pipeline_containedenv(self):
pkg_names = [
'test1',
'test2',
]
expected_prefix = '/my-custom-prefix'
with prepare_testenv(template='contained-env') as engine:
pkgs = engine.pkgman.load(pkg_names)
pipeline = RelengPackagePipeline(engine, engine.opts, {})
for pkg in pkgs:
if not pipeline.process(pkg):
break
# verify package-specific environment variables were set
self.assertTrue('TEST1_BUILD_OUTPUT_DIR' in os.environ)
test1_outdir = os.environ['TEST1_BUILD_OUTPUT_DIR']
results1 = os.path.join(test1_outdir, 'invoke-env.json')
self.assertTrue(os.path.exists(results1))
with open(results1, 'r') as f:
data = json.load(f)
pkg_keys = [
'HOST_INCLUDE_DIR',
'HOST_LIB_DIR',
'NJOBS',
'NJOBSCONF',
'PREFIX',
'PREFIXED_HOST_DIR',
'PREFIXED_STAGING_DIR',
'PREFIXED_TARGET_DIR',
'STAGING_INCLUDE_DIR',
'STAGING_LIB_DIR',
'TARGET_INCLUDE_DIR',
'TARGET_LIB_DIR',
]
self.assertTrue(all(x in data for x in pkg_keys))
self.assertEqual(data['NJOBS'], '42')
self.assertEqual(data['NJOBSCONF'], '42')
self.assertEqual(data['PREFIX'], expected_prefix)
opts = engine.opts
nprefix = os.path.normpath(expected_prefix)
expected_host_pdir = opts.host_dir + nprefix
expected_staging_pdir = opts.staging_dir + nprefix
expected_target_pdir = opts.target_dir + nprefix
J = os.path.join
expected_host_include_dir = J(expected_host_pdir, 'include')
expected_host_lib_dir = J(expected_host_pdir, 'lib')
expected_staging_include_dir = J(expected_staging_pdir, 'include')
expected_staging_lib_dir = J(expected_staging_pdir, 'lib')
expected_target_include_dir = J(expected_target_pdir, 'include')
expected_target_lib_dir = J(expected_target_pdir, 'lib')
self.assertEqual(data['HOST_INCLUDE_DIR'],
expected_host_include_dir)
self.assertEqual(data['HOST_LIB_DIR'], expected_host_lib_dir)
self.assertEqual(data['PREFIXED_HOST_DIR'], expected_host_pdir)
self.assertEqual(data['PREFIXED_STAGING_DIR'],
expected_staging_pdir)
self.assertEqual(data['PREFIXED_TARGET_DIR'], expected_target_pdir)
self.assertEqual(data['STAGING_INCLUDE_DIR'],
expected_staging_include_dir)
self.assertEqual(data['STAGING_LIB_DIR'], expected_staging_lib_dir)
self.assertEqual(data['TARGET_INCLUDE_DIR'],
expected_target_include_dir)
self.assertEqual(data['TARGET_LIB_DIR'], expected_target_lib_dir)
# verify package-specific environment variables were properly
# restored to original counts (leak check)
self.assertTrue('TEST2_BUILD_OUTPUT_DIR' in os.environ)
test2_outdir = os.environ['TEST2_BUILD_OUTPUT_DIR']
results2 = os.path.join(test2_outdir, 'invoke-env.json')
self.assertTrue(os.path.exists(results2))
with open(results2, 'r') as f:
data = json.load(f)
if 'NJOBS' in data:
self.assertNotEqual(data['NJOBS'], '42')
if 'NJOBSCONF' in data:
self.assertNotEqual(data['NJOBSCONF'], '42')
if 'PREFIX' in data:
self.assertNotEqual(data['PREFIX'], '/my-custom-prefix')
self.assertNotEqual(data['HOST_INCLUDE_DIR'],
expected_host_include_dir)
self.assertNotEqual(data['HOST_LIB_DIR'],
expected_host_lib_dir)
self.assertNotEqual(data['PREFIXED_HOST_DIR'],
expected_host_pdir)
self.assertNotEqual(data['PREFIXED_STAGING_DIR'],
expected_staging_pdir)
self.assertNotEqual(data['PREFIXED_TARGET_DIR'],
expected_target_pdir)
self.assertNotEqual(data['STAGING_INCLUDE_DIR'],
expected_staging_include_dir)
self.assertNotEqual(data['STAGING_LIB_DIR'],
expected_staging_lib_dir)
self.assertNotEqual(data['TARGET_INCLUDE_DIR'],
expected_target_include_dir)
self.assertNotEqual(data['TARGET_LIB_DIR'],
expected_target_lib_dir)
def test_engine_run_args_mode_localsrcs_not_configured(self):
with prepare_testenv(template='minimal') as engine:
self.assertFalse(engine.opts.local_srcs)
def test_pkg_pipeline_remote_cfg_override(self):
with prepare_testenv(template='remote-cfg-override') as engine:
engine.run()
self._assertFileFlagExists(engine, 'jobs-2')
def test_pkg_pipeline_remote_scripts_enabled(self):
with prepare_testenv(template='remote-scripts') as engine:
engine.run()
self._assertFileFlag(engine, 'build-remote', True)
self._assertFileFlag(engine, 'configure-remote', True)
self._assertFileFlag(engine, 'install-remote', True)
def test_site_url_fetch_archive_tar_valid_internal_extract(self):
with httpd_context() as httpd:
host, port = httpd.server_address
site = 'http://{}:{}/test.tar'.format(host, port)
httpd_assets = fetch_unittest_assets_dir('sample-files')
archive = os.path.join(httpd_assets, 'sample-files.tar')
with open(archive, 'rb') as f:
data = f.read()
with prepare_testenv(template='minimal') as engine:
root_dir = engine.opts.root_dir
pkg_script = os.path.join(root_dir,
'package', 'minimal', 'minimal')
with open(pkg_script, 'a') as f:
f.write('MINIMAL_SITE="{}"\n'.format(site))
otes = TAR.exists()
try:
# temporarily force a non-external tar command (if needed)
RelengTool.detected[TAR_COMMAND] = False
httpd.rsp.append((200, data))
rv = engine.run()
finally:
RelengTool.detected[TAR_COMMAND] = otes
self.assertTrue(rv)
outdir = os.environ['MINIMAL_BUILD_DIR']
stripped_file = os.path.join(outdir, 'tar-file-container.txt')
self.assertTrue(os.path.exists(stripped_file))
with prepare_testenv(template='minimal') as engine:
root_dir = engine.opts.root_dir
pkg_script = os.path.join(root_dir,
'package', 'minimal', 'minimal')
# redo tar fetch but with a no strip option
with open(pkg_script, 'a') as f:
f.write('MINIMAL_SITE="{}"\n'.format(site))
f.write('MINIMAL_STRIP_COUNT=0\n')
try:
RelengTool.detected[TAR_COMMAND] = False
httpd.rsp.append((200, data))
rv = engine.run()
finally:
RelengTool.detected[TAR_COMMAND] = otes
self.assertTrue(rv)
outdir = os.environ['MINIMAL_BUILD_DIR']
sample_root_file = os.path.join(outdir, 'tar-file-root')
container_dir = os.path.join(outdir, 'container')
sample_container_file = os.path.join(
container_dir, 'tar-file-container.txt')
self.assertTrue(os.path.exists(sample_root_file))
self.assertTrue(os.path.exists(sample_container_file))
def test_engine_run_defaults_options_logging(self):
with prepare_testenv() as engine:
self.assertFalse(engine.opts.debug)
self.assertFalse(engine.opts.no_color_out)
self.assertFalse(engine.opts.verbose)
def test_engine_run_bin_paths_project_root(self):
with prepare_testenv(template='hosts-check') as engine:
engine.run()
expected = 'releng-tool-test-script-root'
self._validate_test_script(expected)
def test_engine_run_bin_paths_host_output_default_prefix(self):
with prepare_testenv(template='hosts-check') as engine:
engine.run()
expected = 'releng-tool-test-script-bin'
self._validate_test_script(expected)
def test_pkg_pipeline_remote_cfg_disabled_option(self):
with prepare_testenv(template='remote-cfg-disabled') as engine:
engine.run()
self._assertFileFlagExists(engine, 'jobs-0')
def test_engine_run_defaults_options_modes(self):
with prepare_testenv() as engine:
self.assertFalse(engine.opts.devmode)
self.assertFalse(engine.opts.force)
self.assertFalse(engine.opts.local_srcs)
def test_pkg_pipeline_remote_scripts_disabled_option(self):
with prepare_testenv(template='remote-scripts-disabled') as engine:
engine.run()
self._assertFileFlag(engine, 'build-remote', False)
self._assertFileFlag(engine, 'configure-remote', False)
self._assertFileFlag(engine, 'install-remote', False)
def test_engine_run_defaults_options_jobs(self):
with prepare_testenv() as engine:
self.assertNotEqual(engine.opts.jobs, 0)
self.assertEqual(engine.opts.jobsconf, 0)
def test_pkg_pipeline_remote_cfg_enabled(self):
with prepare_testenv(template='remote-cfg') as engine:
engine.run()
self._assertFileFlagExists(engine, 'jobs-1')
