def execute(self, script, env, values, inputs):
tests = values.get('tests')
package_test_env = values.get('package_test_env')
if not tests:
message = 'No tests for %s' % (script.descriptor.full_name)
self.log_d(message)
return step_result(True, message)
if env.config.skip_tests:
message = '%s: Skipping tests because of --skip-tests' % (
script.descriptor.full_name)
self.blurb(message)
return step_result(True, message)
staged_tarball = inputs.get('staged_tarball', None)
if not staged_tarball or not path.isfile(staged_tarball):
message = 'Missing staged tarball for %s: ' % (str(
script.descriptor))
self.log_d(message)
return step_result(False, message)
for test in tests:
check.check_value_file(test)
config = package_tester.test_config(script, staged_tarball,
env.artifact_manager,
env.tools_manager,
package_test_env)
tester = package_tester(config, test.filename)
result = tester.run()
if not result.success:
return result
return step_result(True, None)
def execute(self, script, env, values, inputs):
if env.config.download_only:
return step_result(
True,
'skipping step_install_requirements because download_only is True'
)
package_desc = script.descriptor
requirements = env.requirement_manager.resolve_deps(
[package_desc.name], env.config.build_target.system,
['BUILD', 'RUN'], False)
build_blurb.blurb(
'rebuild', '%s - requirements: %s' %
(package_desc.name, ' '.join([p.name for p in requirements])))
if not requirements:
message = 'No requirements for %s' % (script.descriptor.full_name)
self.log_d(message)
return step_result(True, message)
script.requirements_manager.install_packages(requirements,
script.build_target,
['BUILD', 'RUN'])
return step_result(True, None)
def _make_standalone_python(self, program, script, env):
src_basename = path.basename(program.filename)
dst_basename = path.basename(program.dst_filename)
if dst_basename.endswith('.py'):
return step_result(
False,
'dst program should not end in .py: %s' % (dst_basename))
src_program = variable.substitute(program.filename,
script.substitutions)
if not path.isabs(src_program):
src_program = path.join(script.build_dir, src_program)
if not path.isfile(src_program):
return step_result(False,
'src program not found: %s' % (src_program))
tmp_src_program = path.join(script.build_dir, dst_basename + '.py')
file_util.copy(src_program, tmp_src_program)
dst_program = path.join(script.build_dir, 'dist', dst_basename)
cmd = 'pyinstaller --log INFO -F %s' % (tmp_src_program)
rv = self.call_shell(cmd, script, env)
if not rv.success:
return rv
if not path.isfile(dst_program):
return step_result(False,
'dst program not found: %s' % (dst_program))
installed_program = path.join(script.staged_files_dir,
program.dst_filename)
file_util.mkdir(path.dirname(installed_program))
file_util.copy(dst_program, installed_program)
os.chmod(installed_program, 0o755)
return step_result(True, None)
def execute(self, script, env, values, inputs):
install_files = values.get('install_files')
if not install_files:
message = 'No install_files for %s' % (script.descriptor.full_name)
self.log_d(message)
return step_result(True, message)
check.check_value_install_file_seq(install_files)
for install_file in install_files:
src = variable.substitute(install_file.filename,
script.substitutions)
if not path.isfile(src):
return step_result(
False, 'File not found at %s: %s' %
(str(install_file.origin), path.relpath(src)))
dst = path.join(script.staged_files_dir, install_file.dst_filename)
dst_dir = path.dirname(dst)
mode = file_util.mode(src)
self.blurb('Installing file %s in %s (%s)' % (src, dst_dir, mode))
file_util.mkdir(path.dirname(dst))
shutil.copy(src, dst)
os.chmod(dst, mode)
return step_result(True, None)
def execute(self, script, env, values, inputs):
bt = script.build_target
if not bt.system == build_system.LINUX:
return step_result(True)
return step_result(True)
new_path = path.join(script.staged_files_lib_dir, 'x86_64-linux-gnu')
if not path.exists(new_path):
cmd = 'mkdir x86_64-linux-gnu && mv perl x86_64-linux-gnu'
execute.execute(cmd, cwd=script.staged_files_lib_dir, shell=True)
return step_result(True)
def execute(self, script, env, values, inputs):
if not script.has_staged_files_dir():
return step_result(
True,
script.format_message(
'No droppings to cleanup in {staged_files_dir}'))
droppings = file_find.find_fnmatch(script.staged_files_dir, ['*.bak'],
relative=False)
file_util.remove(droppings)
return step_result(True, None)
def execute(self, script, env, values, inputs):
if not script.instructions:
message = 'No build instructions for %s' % (
script.descriptor.full_name)
self.log_d(message)
return step_result(True, message)
success, message = script.instructions.save(
script.staged_files_instructions_dir)
if not success:
self.log_d(message)
return step_result(False, message)
return step_result(True, None)
def execute(self, script, env, values, inputs):
delete_files = values.get('delete_files')
ignore_missing = values.get('delete_files_ignore_missing')
if not delete_files:
message = 'No delete_files for %s' % (script.descriptor.full_name)
self.log_d(message)
return step_result(True, message)
delete_files_in_staged_files_dir = [ path.join(script.staged_files_dir, f) for f in delete_files ]
missing_files = [ f for f in delete_files_in_staged_files_dir if not path.exists(f) ]
if missing_files and not ignore_missing:
return step_result(False, 'File(s) to delete not found: %s' % (' '.join(missing_files)))
file_util.remove(delete_files_in_staged_files_dir)
return step_result(True, None)
def execute(self, script, env, values, inputs):
downloaded_tarballs = inputs.get('downloaded_tarballs', None)
if not downloaded_tarballs:
return step_result(True, None)
for t in downloaded_tarballs:
self.blurb('Extracting %s to %s' %
(path.relpath(t.filename), path.relpath(t.dest_dir)))
archiver.extract(t.filename,
t.dest_dir,
base_dir=t.base_dir,
strip_common_ancestor=t.strip_common_ancestor)
return step_result(True, None)
def execute(self, script, env, values, inputs):
if not script.build_target.is_darwin():
return step_result(True, None)
if not path.isdir(script.staged_files_dir):
return step_result(True, None)
binaries = binary_detector.find_strippable_binaries(script.staged_files_dir, format_name = 'macho')
for b in binaries:
deps = [ dep for dep in library.dependencies(b) if dep.startswith(script.staged_files_dir) ]
for dep in deps:
self.blurb('Fixing rpath: %s %s' % (path.relpath(b), path.relpath(dep)))
rpath = library.relative_rpath(b, dep)
new_dep = path.join('@executable_path', rpath)
cmd = 'install_name_tool -change %s %s %s' % (dep, new_dep, b)
execute.execute(cmd)
return step_result(True, None)
def execute(self, script, env, args):
fake_success = args['fake_success']
if fake_success:
fake_message = ''
else:
fake_message = 'step %s failed' % (args['fake_name'])
return step_result(fake_success, fake_message)
def execute(self, script, env, values, inputs):
pc_files = pkg_config.find_pc_files(script.staged_files_dir)
for pc_file in pc_files:
if pkg_config_file.rewrite_cleanup(pc_file, pc_file, backup=False):
self.blurb('Cleaned %s' % (path.relpath(pc_file)))
return step_result(True, None)
def _run_exe_test(clazz, exe_env_name, exe_default, config, test_source):
context = clazz._make_test_context(config, test_source)
build_blurb.blurb_verbose(
'tester', '%s: running shell test %s' %
(context.package_info.name, test_source))
exe = clazz._determine_exe(exe_env_name, exe_default)
cmd = '%s %s' % (exe, context.test_source_with_replacements)
rv = execute.execute(cmd,
env=context.env,
non_blocking=build_blurb.verbose,
stderr_to_stdout=True,
raise_error=False)
os.environ = context.saved_env
if rv.exit_code != 0:
return step_result(rv.exit_code == 0, rv.stdout)
return step_result(True, None)
def execute(self, script, env, values, inputs):
if not path.isdir(script.staged_files_bin_dir):
return step_result(True, None)
if values.get('skip_binary_third_party_prefix'):
return step_result(True, None)
binaries = dir_util.list(script.staged_files_bin_dir)
for b in binaries:
link_src = path.basename(b)
if not link_src.startswith(env.config.third_party_prefix):
link_filename = '%s%s' % (env.config.third_party_prefix,
link_src)
link_dst = path.join(script.staged_files_bin_dir,
link_filename)
file_util.symlink(link_src, link_dst)
return step_result(True, None)
def execute(self, script, env, values, inputs):
if not script.has_staged_files_dir():
return step_result(
True, 'No files to check in %s' %
(path.relpath(script.staged_files_dir)))
replacements = {
script.staged_files_dir:
'${REBUILD_PACKAGE_PREFIX}',
script.requirements_manager.installation_dir:
'${REBUILD_PACKAGE_PREFIX}',
}
file_search.search_replace(script.staged_files_dir,
replacements,
backup=False,
test_func=file_mime.is_text)
return step_result(True, None)
def execute(self, script, env, values, inputs):
env_files = values.get('env_files')
if not env_files:
message = 'No env files for %s' % (script.descriptor.full_name)
self.log_d(message)
return step_result(True, message)
for env_file in env_files:
dst_file = path.join(script.stagged_env_dir,
path.basename(env_file.filename))
file_replace.copy_with_substitute(env_file.filename,
dst_file,
script.substitutions,
backup=False)
return step_result(True, None)
def _run_c_test(clazz, config, test_source, compiler, extra_cflags):
context = clazz._make_test_context(config, test_source)
build_blurb.blurb_verbose(
'tester', '%s: running c test %s with %s' %
(context.package_info.name, path.relpath(test_source), compiler))
package_names_for_pkg_config = [context.package_info.name]
# FIXME: static is hardcoded here (depends on global static mode)
cflags, libs = context.package_manager.compilation_flags(
package_names_for_pkg_config, static=True)
test_exe = path.join(context.test_dir, context.test_name + '.exe')
cflags += extra_cflags
cflags_flat = ' '.join(cflags)
libs_flat = ' '.join(libs)
compilation_cmd = '%s -o %s %s %s %s' % (
compiler, test_exe, context.test_source_with_replacements,
cflags_flat, libs_flat)
build_blurb.blurb_verbose(
'tester', '%s: compilation_cmd=\"%s\"' %
(context.package_info.name, compilation_cmd))
build_blurb.blurb_verbose(
'tester', '%s: cflags=\"%s\" libs=\"%s\"' %
(context.package_info.name, cflags_flat, libs_flat))
# Build the test
rv = execute.execute(compilation_cmd,
env=context.env,
non_blocking=build_blurb.verbose,
stderr_to_stdout=True,
raise_error=False)
if rv.exit_code != 0:
return step_result(rv.exit_code == 0, rv.stdout)
# Run the test
rv = execute.execute(test_exe,
env=context.env,
non_blocking=build_blurb.verbose,
stderr_to_stdout=True,
raise_error=False)
# Restore the environment cause _make_test_context() hacks it
os.environ = context.saved_env
if rv.exit_code != 0:
return step_result(rv.exit_code == 0, rv.stdout)
return step_result(True, None)
def execute(self, script, env, values, inputs):
staged_files_lib_dir = path.join(script.staged_files_dir, 'lib')
if path.isdir(staged_files_lib_dir):
droppings = file_find.find_fnmatch(path.join(
script.staged_files_dir, 'lib'), ['*.la'],
relative=False)
file_util.remove(droppings)
return step_result(True, None)
def _run_python_test(clazz, config, test_source):
context = clazz._make_test_context(config, test_source)
build_blurb.blurb_verbose(
'tester', '%s: running python test %s' %
(context.package_info.name, test_source))
# FIXME: move this to context
context.env['PYTHONPATH'] = context.package_manager.python_lib_dir
# Run the test
cmd = 'python2.7 %s' % (context.test_source_with_replacements)
rv = execute.execute(cmd,
env=context.env,
non_blocking=build_blurb.verbose,
stderr_to_stdout=True,
raise_error=False)
if rv.exit_code != 0:
return step_result(rv.exit_code == 0, rv.stdout)
return step_result(True, None)
def execute(self, script, env, values, inputs):
staged_tarball = inputs.get('staged_tarball', None)
if not staged_tarball or not path.isfile(staged_tarball):
message = 'Missing staged tarball for %s: ' % (str(
script.descriptor))
self.log_d(message)
return step_result(False, message)
metadata = inputs.get('metadata', None)
if not metadata:
message = 'Missing metadata for %s: ' % (str(script.descriptor))
self.log_d(message)
return step_result(False, message)
assert archiver.is_valid(staged_tarball)
published_tarball = env.artifact_manager.publish(
staged_tarball, script.build_target, True, metadata)
self.blurb('published tarball: %s' % (path.relpath(published_tarball)))
return step_result(True,
None,
outputs={'published_tarball': published_tarball})
def _make_standalone_shell_script(self, program, script, env):
src_basename = path.basename(program.filename)
dst_basename = path.basename(program.dst_filename)
if dst_basename.endswith('.sh'):
return step_result(
False,
'dst program should not end in .sh: %s' % (dst_basename))
src_program = variable.substitute(program.filename,
script.substitutions)
if not path.isabs(src_program):
src_program = path.join(script.build_dir, src_program)
if not path.isfile(src_program):
return step_result(False,
'src program not found: %s' % (src_program))
installed_program = path.join(script.staged_files_dir,
program.dst_filename)
file_util.mkdir(path.dirname(installed_program))
file_util.copy(src_program, installed_program)
os.chmod(installed_program, 0o755)
return step_result(True, None)
def execute(self, script, env, values, inputs):
droppings = [
'lib/python/easy-install.pth',
'lib/python/site.py',
'lib/python/site.pyc',
]
droppings = [
path.join(script.staged_files_dir, dropping)
for dropping in droppings
]
file_util.remove(droppings)
return step_result(True, None)
def execute(self, script, env, values, inputs):
standalone_programs = values.get('standalone_programs', [])
if not standalone_programs:
return step_result(True)
file_util.mkdir(script.staged_files_bin_dir)
for program in standalone_programs:
src_program = path.join(script.build_dir, program.filename)
if src_program.lower().endswith('.py'):
rv = self._make_standalone_python(program, script, env)
if not rv.success:
return rv
elif src_program.lower().endswith('.sh'):
rv = self._make_standalone_shell_script(program, script, env)
if not rv.success:
return rv
else:
raise RuntimeError('Unknown standalone program type: %s' %
(src_program))
return step_result(True, None)
def execute(self, script, env, values, inputs):
if env.config.download_only:
return step_result(
True,
'skipping step_setup_install_tool_requirements because download_only is True'
)
package_desc = script.descriptor
tools = script.resolve_deps(['TOOL'], False)
build_blurb.blurb(
'rebuild', '%s - tool requirements: %s' %
(package_desc.name, ' '.join([t.name for t in tools])))
if not tools:
message = 'No tools for %s' % (script.descriptor.full_name)
self.log_d(message)
return step_result(True, message)
env.tools_manager.ensure_tools(tools)
env.tools_manager.expose_env(tools)
return step_result(True, None)
def execute(self, script, env, values, inputs):
patches = values.get('patches')
if not patches:
message = 'No patches for %s' % (script.descriptor.full_name)
self.log_d(message)
return step_result(True, message)
patch_program = values.get('patch_program')
patch_dir = values.get('patch_dir')
for p in patches:
strip = int(p.get_property('strip',
self.DEFAULT_PATCH_STRIP_DEPTH))
self.blurb(
'Applying patch %s (strip=%d) in dir %s' % (path.relpath(
p.filename), strip, path.relpath(patch_dir.filename)))
exit_code, msg = patch.patch(p.filename,
patch_dir.filename,
strip=strip,
backup=True,
posix=True,
program=patch_program)
return step_result(exit_code == 0, msg)
def execute(self, script, env, values, inputs):
pc_files = values.get('pc_files')
pc_file_variables = values.get('pc_file_variables')
if not pc_files:
message = 'No .pc files for %s' % (script.descriptor.full_name)
self.log_d(message)
return step_result(True, message)
replacements = {}
replacements.update(script.substitutions)
if pc_file_variables:
pc_file_variables = pc_file_variables[:]
pc_file_variables.unquote_strings()
replacements.update(pc_file_variables.to_dict())
for src_pc in pc_files:
dst_dir = path.join(script.staged_files_dir, 'lib/pkgconfig')
dst_pc = path.join(dst_dir, path.basename(src_pc.filename))
file_replace.copy_with_substitute(src_pc.filename,
dst_pc,
replacements,
backup=False)
return step_result(True, None)
def execute(self, script, env, values, inputs):
if not script.build_target.is_darwin():
return step_result(True, None)
if values.get('ignore_check_darwin_archs'):
return step_result(True, None)
if path.isdir(script.staged_files_lib_dir):
expected_archs = script.build_target.arch
for lib in library.list_libraries(script.staged_files_lib_dir,
relative=False):
actual_archs = lipo.archs(lib)
if not self.__matches(expected_archs, actual_archs):
expected_label = ','.join(expected_archs)
if actual_archs:
actual_label = ','.join(actual_archs)
else:
actual_label = 'None'
msg = 'Expected archs for %s dont match. Should be \"%s\" instead if \"%s\"' % (
lib, expected_label, actual_label)
return step_result(False, msg)
return step_result(True, None)
def execute(self, script, env, values, inputs):
script_file = values.get('script')
script_env = values.get('script_env')
if not script_file:
message = 'No script for %s' % (script.descriptor.full_name)
self.log_d(message)
return step_result(True, message)
shell_env = script_env[:]
shell_env.extend(script_file.properties)
return self.call_shell(path.abspath(script_file.filename),
script,
env,
shell_env=shell_env)
def execute(self, script, env, values, inputs):
if not script.has_stage_dir():
return step_result(
False,
script.format_message(
'No files to package found in {stage_dir}'))
tarball_name = '%s.tar.gz' % (script.descriptor.full_name)
assert tarball_name == script.descriptor.tarball_filename
output_tarball_path = path.join(script.artifact_dir,
script.descriptor.tarball_filename)
self.blurb('creating tarball %s from %s' %
(path.relpath(output_tarball_path),
path.relpath(script.stage_dir)),
fit=True)
staged_tarball, metadata = package.create_package(
output_tarball_path, script.descriptor, script.build_target,
script.stage_dir)
self.blurb('staged tarball: %s' % (path.relpath(staged_tarball)))
outputs = {
'staged_tarball': staged_tarball,
'metadata': metadata,
}
return step_result(True, None, outputs=outputs)
def execute(self, script, env, values, inputs):
if env.config.download_only:
return step_result(True, None, outputs={'_skip_rest': True})
packages = values.get('packages')
archives = self._filenames(env, packages)
common_files = self._common_files(archives)
if common_files:
return self.result(
False, 'conflicting files found between artifacts: %s' %
(' '.join(common_files)))
for archive in archives:
self.blurb('Extracting %s to %s' %
(path.relpath(archive), path.relpath(script.stage_dir)))
archiver.extract(archive,
script.stage_dir,
exclude=['metadata/metadata.json'])
return self.result(True)
