def __main(clazz):
ap = argparse.ArgumentParser(description = 'My hacky little build system.')
ap.add_argument('-c', '--clean', dest = 'clean', action = 'store_true')
ap.add_argument('-v', '--verbose', dest = 'verbose', action = 'store_true')
ap.add_argument('-C', '--chdir', dest = 'chdir', action = 'store', default = None)
ap.set_defaults(clean = False, verbose = False)
ap.add_argument('scons_args', action = 'append', nargs = argparse.REMAINDER)
args = ap.parse_args()
if args.chdir != None:
if not path.isdir(args.chdir):
return ( 1, 'Not a directory: %s' % (args.chdir) )
else:
os.chdir(args.chdir)
scons_args = args.scons_args #[0]
if args.clean:
cmd = clazz.make_scons_cmd(args.verbose, [ '-c' ])
execute.execute(cmd, non_blocking = True)
cmd = clazz.make_scons_cmd(args.verbose, scons_args)
execute.execute(cmd, non_blocking = True)
return ( 0, None )
def _extract_with_ar(clazz, archive, dest_dir, tools):
'Return the archive contents using ar.'
cmd = [
tools.ar,
'x',
archive,
]
cmd_flat = ' '.join(cmd)
file_util.mkdir(dest_dir)
execute.execute(cmd_flat, cwd = dest_dir)
def _replace_with_ar(clazz, archive, objects, tools):
'Replace an archive using ar.'
cmd = [
tools.ar,
'r',
archive,
] + objects
cmd_flat = ' '.join(cmd)
file_util.mkdir(path.dirname(archive))
execute.execute(cmd_flat)
def _make_temp_archive_dmg(clazz, items, filename, mode):
tmp_dir = temp_file.make_temp_dir()
for item in items:
assert item
assert item.arcname
file_util.save(path.join(tmp_dir, item.arcname), content = item.content)
tmp_dmg = temp_file.make_temp_file()
cmd = 'hdiutil create -srcfolder %s -ov -format UDZO %s' % (tmp_dir, filename)
execute.execute(cmd)
file_util.remove(tmp_dir)
def execute(self, script, env, values, inputs):
bt = script.build_target
if not bt.system == build_system.LINUX:
return step_result(True)
return step_result(True)
new_path = path.join(script.staged_files_lib_dir, 'x86_64-linux-gnu')
if not path.exists(new_path):
cmd = 'mkdir x86_64-linux-gnu && mv perl x86_64-linux-gnu'
execute.execute(cmd, cwd=script.staged_files_lib_dir, shell=True)
return step_result(True)
def create(self, root_dir, base_dir = None,
extra_items = None,
include = None, exclude = None):
self._pre_create()
items = self._find(root_dir, base_dir, extra_items, include, exclude)
tmp_dir = temp_file.make_temp_dir()
for item in items:
file_util.copy(item.filename, path.join(tmp_dir, item.arcname))
cmd = 'hdiutil create -srcfolder %s -ov -format UDZO %s' % (tmp_dir, self.filename)
execute.execute(cmd)
file_util.remove(tmp_dir)
def _make_temp_archive_xz(clazz, items, filename, mode):
tmp_dir = temp_file.make_temp_dir()
for item in items:
assert item
assert item.arcname
file_util.save(path.join(tmp_dir, item.arcname), content = item.content)
tmp_xz = temp_file.make_temp_file()
manifest_content = '\n'.join([ item.arcname for item in items ])
manifest = temp_file.make_temp_file(content = manifest_content)
cmd = 'tar Jcf %s -C %s -T %s' % (filename, tmp_dir, manifest)
execute.execute(cmd)
file_util.remove(tmp_dir)
def _determine_manifest(clazz, stage_dir):
'Return the list of files to package. Maybe could do some filtering here. Using find because its faster that bes.fs.file_find.'
stuff = dir_util.list(stage_dir, relative=True)
rv = execute.execute(['find'] + stuff + ['-type', 'f'], cwd=stage_dir)
files = text_line_parser.parse_lines(rv.stdout,
strip_text=True,
remove_empties=True)
rv = execute.execute(['find'] + stuff + ['-type', 'l'], cwd=stage_dir)
links = text_line_parser.parse_lines(rv.stdout,
strip_text=True,
remove_empties=True)
return sorted(files + links)
def create(self, root_dir, base_dir = None,
extra_items = None,
include = None, exclude = None):
self._pre_create()
items = self._find(root_dir, base_dir, extra_items, include, exclude)
tmp_dir = temp_file.make_temp_dir()
for item in items:
file_util.copy(item.filename, path.join(tmp_dir, item.arcname))
manifest_content = '\n'.join([ item.arcname for item in items ])
manifest = temp_file.make_temp_file(content = manifest_content)
cmd = 'tar Jcf %s -C %s -T %s' % (self.filename, tmp_dir, manifest)
execute.execute(cmd)
file_util.remove(tmp_dir)
def execute(self, script, env, values, inputs):
if not script.build_target.is_linux():
return self.result(True, None)
if not path.isdir(script.staged_files_dir):
return self.result(True, None)
binaries = binary_detector.find_strippable_binaries(script.staged_files_dir, format_name = 'elf')
rpath = values.get('rpath')
for b in binaries:
if not library.is_library(b):
self.blurb('Setting rpath %s for %s' % (rpath, path.relpath(b)))
cmd = 'patchelf --set-rpath %s %s' % (rpath, b)
execute.execute(cmd)
return self.result(True, None)
def extract(self, dest_dir, base_dir = None,
strip_common_ancestor = False, strip_head = None,
include = None, exclude = None):
dest_dir = self._determine_dest_dir(dest_dir, base_dir)
filtered_members = self._filter_for_extract(self.members, include, exclude)
if filtered_members == self.members:
self.extract_all(dest_dir, base_dir = base_dir, strip_common_ancestor = strip_common_ancestor,
strip_head = strip_head)
return
manifest = temp_file.make_temp_file(content = '\n'.join(filtered_members))
cmd = 'tar xf %s -C %s -T %s' % (self.filename, dest_dir, manifest)
execute.execute(cmd)
self._handle_extract_strip_common_ancestor(filtered_members, strip_common_ancestor, strip_head, dest_dir)
def make(clazz, setup_filename):
assert path.isfile(setup_filename)
temp_dir = tempfile.mkdtemp()
src_dir = path.dirname(setup_filename)
assert path.isdir(src_dir)
shutil.rmtree(temp_dir)
shutil.copytree(src_dir, temp_dir, symlinks = True)
cmd = [ 'python', path.basename(setup_filename), 'bdist_egg' ]
env = copy.deepcopy(os.environ)
env['PYTHONDONTWRITEBYTECODE'] = '1'
execute.execute(cmd, shell = False, cwd = temp_dir, env = env)
eggs = glob.glob('%s/dist/*.egg' % (temp_dir))
assert len(eggs) == 1
return eggs[0]
def create(self, root_dir, base_dir = None, extra_items = None,
include = None, exclude = None):
items = self._find(root_dir, base_dir, extra_items, include, exclude)
ext = archive_extension.extension_for_filename(self.filename)
mode = archive_extension.write_format_for_filename(self.filename)
# print('CACA: ext=%s' % (ext))
# print('CACA: mode=%s' % (mode))
tmp_dir = temp_file.make_temp_dir()
for item in items:
file_util.copy(item.filename, path.join(tmp_dir, item.arcname))
manifest_content = '\n'.join([ item.arcname for item in items ])
manifest = temp_file.make_temp_file(content = manifest_content)
cmd = 'tar Jcf %s -C %s -T %s' % (self.filename, tmp_dir, manifest)
execute.execute(cmd)
file_util.remove(tmp_dir)
def execute(self, script, env, values, inputs):
if not script.build_target.is_darwin():
return step_result(True, None)
if not path.isdir(script.staged_files_dir):
return step_result(True, None)
binaries = binary_detector.find_strippable_binaries(script.staged_files_dir, format_name = 'macho')
for b in binaries:
deps = [ dep for dep in library.dependencies(b) if dep.startswith(script.staged_files_dir) ]
for dep in deps:
self.blurb('Fixing rpath: %s %s' % (path.relpath(b), path.relpath(dep)))
rpath = library.relative_rpath(b, dep)
new_dep = path.join('@executable_path', rpath)
cmd = 'install_name_tool -change %s %s %s' % (dep, new_dep, b)
execute.execute(cmd)
return step_result(True, None)
def _replace_darwin(clazz, archive, objects, tools):
'Replace an archive on darwin using libtool.'
cmd = [
tools.libtool,
'-static',
'-o',
archive,
]
if path.exists(archive):
cmd += [ archive ]
cmd += objects
cmd_flat = ' '.join(cmd)
parent_dir = path.dirname(archive)
if parent_dir:
file_util.mkdir(parent_dir)
execute.execute(cmd_flat)
def extract_members(self, members, dest_dir, base_dir = None,
strip_common_ancestor = False, strip_head = None,
include = None, exclude = None):
# Cheat by using a temporary zip file to do the actual work. Super innefecient but
# easy since theres no library to extract just some stuff from dmg files.
tmp_dir = temp_file.make_temp_dir()
cmd = 'tar xf %s -C %s' % (self.filename, tmp_dir)
execute.execute(cmd)
tmp_zip = temp_file.make_temp_file(suffix = '.zip')
az = archive_zip(tmp_zip)
az.create(tmp_dir)
az.extract_members(members, dest_dir, base_dir = base_dir,
strip_common_ancestor = strip_common_ancestor, strip_head = strip_head,
include = include, exclude = exclude)
file_util.remove(tmp_zip)
file_util.remove(tmp_dir)
def _unit_test_runner_builder(target, source, env):
self = env['BES_BUILD']
verbose = env['__BES_VERBOSE']
binary_path = source[0].get_abspath()
stamp_path = target[0].get_abspath()
runner_env = {
'_BES_BUILD_CWD': os.getcwd(),
'_BES_BUILD_SOURCE_DIR': self.source_dir,
}
non_blocking = verbose
rv = execute.execute(binary_path,
raise_error=False,
non_blocking=non_blocking,
stderr_to_stdout=True,
env=runner_env)
if rv.exit_code != 0:
sys.stdout.write(rv.stdout)
stamp_content = str(datetime.now()) + '\n'
open(stamp_path, 'w').write(stamp_content)
return rv.exit_code
def grep(clazz, tarball, pattern):
'Return the output of ag (silver searcher) for an archive.'
tmp_dir = temp_file.make_temp_dir()
archiver.extract(tarball, tmp_dir, strip_common_ancestor = True)
result = execute.execute('ag %s .' % (pattern), cwd = tmp_dir, shell = True, raise_error = False).stdout
file_util.remove(tmp_dir)
return result
def test_update_script(self):
config = '''
[common]
artifacts_dir: /somewhere/not/there
[test1]
description: test1
packages: orange_juice
'''
test = self._setup_test(config)
args = [
'packages', 'update', '--artifacts',
test.artifact_manager.root_dir, '--root-dir', test.tmp_dir,
'--system', 'linux', '--distro', 'ubuntu', '--distro-version',
'18', '--level', 'release', 'test1'
]
rv = self.run_script(args)
self.assertEqual(0, rv.exit_code)
args = [
'packages', 'print', '--root-dir', test.tmp_dir, '--system',
'linux', '--distro', 'ubuntu', '--distro-version', '18', '--level',
'release', 'test1'
]
rv = self.run_script(args)
self.assertEqual([
'citrus', 'fiber', 'fructose', 'fruit', 'orange', 'orange_juice',
'water'
], rv.stdout.split('\n'))
config = '''
[common]
artifacts_dir: /somewhere/not/there
[test1]
description: test1
packages: orange_juice pear_juice
'''
file_util.save(path.join(test.tmp_dir, 'config'), content=config)
cmd = [
path.join(test.tmp_dir,
'update.sh'), '--system', 'linux', '--distro', 'ubuntu',
'--distro-version', '18', '--level', 'release', 'test1'
]
env = os_env.make_clean_env(keep_keys=['PYTHONPATH'],
update={'PATH': path.dirname(self.script)})
rv = execute.execute(cmd,
raise_error=False,
env=env,
stderr_to_stdout=True)
if rv.exit_code != 0 or self.DEBUG:
self.spew(rv.stdout)
self.assertEqual(0, rv.exit_code)
args = [
'packages', 'print', '--root-dir', test.tmp_dir, '--level',
'release', '--system', 'linux', '--distro', 'ubuntu',
'--distro-version', '18', '--level', 'release', 'test1'
]
rv = self.run_script(args)
self.assertEqual([
'citrus', 'fiber', 'fructose', 'fruit', 'orange', 'orange_juice',
'pear', 'pear_juice', 'water'
], rv.stdout.split('\n'))
def test_args_flat_with_shell(self):
script = self.data_path('shell_script_true.sh')
cmd = '%s one two three' % (script)
rv = execute.execute(cmd, shell = True)
self.assertEqual( 0, rv.exit_code )
self.assertEqual( [ 'one', 'two', 'three' ], rv.stdout.strip().split(' ') )
def strip(clazz, build_target, binary):
clazz.check_strippable(binary)
tc = toolchain.get_toolchain(build_target)
ce = tc.compiler_environment()
strip_exe = ce['STRIP']
cmd = [strip_exe, binary]
rv = execute.execute(cmd, raise_error=False)
return rv.exit_code == 0
def _run_c_test(clazz, config, test_source, compiler, extra_cflags):
context = clazz._make_test_context(config, test_source)
build_blurb.blurb_verbose(
'tester', '%s: running c test %s with %s' %
(context.package_info.name, path.relpath(test_source), compiler))
package_names_for_pkg_config = [context.package_info.name]
# FIXME: static is hardcoded here (depends on global static mode)
cflags, libs = context.package_manager.compilation_flags(
package_names_for_pkg_config, static=True)
test_exe = path.join(context.test_dir, context.test_name + '.exe')
cflags += extra_cflags
cflags_flat = ' '.join(cflags)
libs_flat = ' '.join(libs)
compilation_cmd = '%s -o %s %s %s %s' % (
compiler, test_exe, context.test_source_with_replacements,
cflags_flat, libs_flat)
build_blurb.blurb_verbose(
'tester', '%s: compilation_cmd=\"%s\"' %
(context.package_info.name, compilation_cmd))
build_blurb.blurb_verbose(
'tester', '%s: cflags=\"%s\" libs=\"%s\"' %
(context.package_info.name, cflags_flat, libs_flat))
# Build the test
rv = execute.execute(compilation_cmd,
env=context.env,
non_blocking=build_blurb.verbose,
stderr_to_stdout=True,
raise_error=False)
if rv.exit_code != 0:
return step_result(rv.exit_code == 0, rv.stdout)
# Run the test
rv = execute.execute(test_exe,
env=context.env,
non_blocking=build_blurb.verbose,
stderr_to_stdout=True,
raise_error=False)
# Restore the environment cause _make_test_context() hacks it
os.environ = context.saved_env
if rv.exit_code != 0:
return step_result(rv.exit_code == 0, rv.stdout)
return step_result(True, None)
def call_shell(clazz, command, script, env, shell_env = None, save_logs = None, execution_dir = None):
command = execute.listify_command(command)
command = [ part for part in command if part ]
shell_env = shell_env or key_value_list()
save_logs = save_logs or []
check.check_key_value_list(shell_env)
log.log_i('rebuild', 'call_shell(command=%s)' % (command))
build_blurb.blurb_verbose('rebuild', 'call_shell(cmd=%s, shell_env=%s)' % (command, shell_env))
env = clazz.create_command_env(script)
env.update(shell_env)
#clazz.env_dump(env, script.descriptor.name, 'PRE ENVIRONMENT')
clazz._env_substitite(env)
rogue_key = clazz._env_find_roque_dollar_sign(env)
if rogue_key:
raise RuntimeError('Rogue dollar sign (\"$\") found in %s: %s' % (rogue_key, env[rogue_key]))
command = [ variable.substitute(part, env) for part in object_util.listify(command) ]
# Check that no rogue dollar signs are in the command
for part in command:
if variable.has_rogue_dollar_signs(part):
raise RuntimeError('Rogue dollar sign (\"$\") found in: %s' % (part))
build_blurb.blurb('rebuild', '%s - %s' % (script.descriptor.name, ' '.join(command)))
# file_util.mkdir(script.build_dir)
retry_script = clazz._write_retry_script(command, env, script)
#clazz.env_dump(env, script.descriptor.name, clazz.__name__ + ' : ' + 'POST ENVIRONMENT')
for k,v in env.items():
if string_util.is_quoted(v):
env[k] = string_util.unquote(v)
if execution_dir:
cwd = path.join(script.build_dir, execution_dir)
else:
cwd = script.build_dir
rv = execute.execute(command,
cwd = cwd,
env = env,
shell = True,
non_blocking = build_blurb.verbose,
stderr_to_stdout = build_blurb.verbose,
raise_error = False)
message = rv.stdout
if rv.stderr:
message = message + '\n' + rv.stderr
result = step_result(rv.exit_code == 0, message)
clazz.save_build_dir_logs(script, save_logs)
return result
def create(clazz, dst_dir, config, no_filters):
'Create a jail at dst_dir with config.'
if not isinstance(config, config_file):
raise RuntimeError('Not a config_file: %s' % (str(config)))
if path.exists(dst_dir):
if not path.isdir(dst_dir):
raise RuntimeError('Not a directory: %s' % (dst_dir))
if not dir_util.is_empty(dst_dir):
raise RuntimeError('Directory should be empty: %s' % (dst_dir))
else:
file_util.mkdir(dst_dir)
clazz.__install_packages(dst_dir, config, no_filters)
clazz.__install_binaries(dst_dir, config, no_filters)
for post_hook in config.jail.hooks.post or []:
#print "executing: ", post_hook
execute.execute(post_hook)
def _create_package(clazz, tarball_filename, stage_dir, timer):
'Return the list of files to package. Maybe could do some filtering here. Using find because its faster that bes.fs.file_find.'
if timer:
timer.start('create_package - determine manifest')
files_to_package = clazz._determine_manifest(stage_dir)
if timer:
timer.stop()
file_util.mkdir(path.dirname(tarball_filename))
manifest = temp_file.make_temp_file(
content='\n'.join(files_to_package))
tar_cmd = [
'tar', 'zcf', tarball_filename, '-C', stage_dir, '-T', manifest
]
if timer:
timer.start('create_package - execute %s' % (' '.join(tar_cmd)))
execute.execute(tar_cmd)
if timer:
timer.stop()
def autoconf_help(clazz, tarball):
'Return the output of configure --help for an autoconf archive.'
tmp_dir = temp_file.make_temp_dir()
archiver.extract(tarball, tmp_dir, strip_common_ancestor = True)
confiugure_path = path.join(tmp_dir, 'configure')
if not path.exists(confiugure_path):
raise RuntimeError('No configure script found in %s' % (tarball))
help = execute.execute('./configure --help', cwd = tmp_dir, shell = True, raise_error = False).stdout
file_util.remove(tmp_dir)
return help
def _contents_with_ar(clazz, archive, tools):
'Return the archive contents using ar.'
cmd = [
tools.ar,
't',
archive,
]
cmd_flat = ' '.join(cmd)
rv = execute.execute(cmd_flat)
return rv.stdout.strip().split('\n')
def caca_dependencies(clazz, filename, python_exe):
'Return list of files filename depends on or None if snakefood is not found.'
rv = execute.execute('%s -v %s' % (python_exe, filename), raise_error = False)
parser = text_line_parser(rv.stderr)
parser.remove_empties()
for line in parser:
f = clazz._IMPORT_PATTERN.findall(line.text)
if f:
print(f[0][2])
return None
def _run_test(self, script_name):
script = path.join(path.dirname(__file__), script_name)
print("python: running: %s" % (script))
env = self._make_env()
rv = execute.execute(script,
shell = False,
stderr_to_stdout = True,
raise_error = False,
env = env)
print(rv.stdout)
return rv.exit_code
def extract(self, root_dir, stuff_dir_basename, env_dir_basename):
tmp_dir = temp_file.make_temp_dir(prefix='package.extract.',
suffix='.dir',
dir=root_dir)
dst_stuff_dir = path.join(root_dir, stuff_dir_basename)
dst_env_dir = path.join(root_dir, env_dir_basename)
file_util.mkdir(dst_stuff_dir)
file_util.mkdir(dst_env_dir)
# tar cmd is 10x faster than archiver. need to fix archiver
tar_cmd = ['tar', 'xf', self.tarball, '-C', tmp_dir]
execute.execute(tar_cmd)
#archiver.extract_all(self.tarball, tmp_dir)
src_stuff_dir = path.join(tmp_dir, self.FILES_DIR)
src_env_dir = path.join(tmp_dir, self.ENV_DIR)
if path.isdir(src_stuff_dir):
dir_util.move_files(src_stuff_dir, dst_stuff_dir)
self._post_install_hooks(dst_stuff_dir)
if path.isdir(src_env_dir):
dir_util.move_files(src_env_dir, dst_env_dir)
self._variable_substitution_hook(dst_env_dir, dst_stuff_dir)
file_util.remove(tmp_dir)
def __load_config(self):
if not path.isfile(self._args.config):
raise RuntimeError('File not found: %s' % (self._args.config))
variables = {
'root': self._args.location,
'source_dir': path.dirname(self._args.config),
'username': user.USERNAME,
}
if host.SYSTEM == host.MACOS:
variables['DARWIN_USER_CACHE_DIR'] = execute.execute(
'getconf DARWIN_USER_CACHE_DIR').stdout.strip()
cf = config_file(self._args.config, variables)
return cf
def make_template_tarball(root, template_name, template_version):
tmp_dir = temp_file.make_temp_dir(delete = not DEBUG)
if DEBUG:
print('DEBUG2: tmp_dir=%s' % (tmp_dir))
full_name = '%s-%s' % (template_name, template_version)
template_dir = path.join(root, 'template')
tar_util.copy_tree_with_tar(template_dir, tmp_dir)
working_dir = path.join(tmp_dir, full_name)
tarball_filename = '%s.tar.gz' % (full_name)
command = [
'cd %s' % (working_dir),
'automake -a',
'autoconf',
'./configure',
'make dist',
'cp %s %s' % (tarball_filename, root),
]
env = os_env.make_clean_env(keep_keys = [ 'PATH', 'PKG_CONFIG_PATH' ])
env['GZIP'] = '-n'
execute.execute(' && '.join(command), shell = True, non_blocking = True, env = env)
result = path.join(root, '%s-%s.tar.gz' % (template_name, template_version))
assert path.isfile(result)
return result
def test_args(self):
script = self.data_path('shell_script_true.sh')
cmd = [
script,
'one',
'two',
'three',
]
rv = execute.execute(cmd)
self.assertEqual( 0, rv.exit_code )
self.assertEqual( [ 'one', 'two', 'three' ], rv.stdout.strip().split(' ') )
def dependencies(clazz, filename):
'Return a list of dependencies for filename (executable or shared lib) or None if not applicable.'
types = [
binary_format_elf.FILE_TYPE_EXECUTABLE,
binary_format_elf.FILE_TYPE_SHARED_LIB
]
if not binary_format_elf().file_is_of_type(filename, types):
return None
cmd = ['ldd', filename]
rv = execute.execute(cmd)
assert rv.stdout.find('not a dynamic executable') == -1
lines = rv.stdout.split('\n')
deps = [clazz.__parse_ldd_line(line) for line in lines]
deps = [d for d in deps if d]
return sorted(deps)
def patch_make(clazz, working_dir):
'Create a patch out of 2 directories one ending in .orig.'
dirs = dir_util.list(working_dir, relative = True)
if len(dirs) != 2:
raise RuntimeError('Found more than 2 directories in %s' % (working_dir))
if not dirs[1].endswith(clazz.ORIGINAL_DIR_TAIL):
raise RuntimeError('Dir 2 should end in .orig instead it is %s' % (dir2))
base_dir = file_util.remove_tail(dirs[1], clazz.ORIGINAL_DIR_TAIL)
if string_util.remove_tail(dirs[1], clazz.ORIGINAL_DIR_TAIL) != dirs[0]:
raise RuntimeError('Dir 1 and 2 dont have the same name: %s %s' % (dirs[1], dirs[0]))
cmd = 'diff -ur %s %s --exclude="*~" --exclude=".#*" --exclude="#*"' % (dirs[1], dirs[0])
patch = execute.execute(cmd, cwd = working_dir, raise_error = False).stdout
return patch.strip()
def diff(clazz, archive1, archive2, strip_common_ancestor = False):
'Return the output of diffing the contents of 2 archives.'
members1 = archiver.members(archive1)
members2 = archiver.members(archive2)
content1 = '\n'.join(members1)
content2 = '\n'.join(members2)
tmp_file1 = temp_file.make_temp_file(content = content1)
tmp_file2 = temp_file.make_temp_file(content = content2)
rv = execute.execute('diff -u %s %s' % (tmp_file1, tmp_file2), raise_error = False)
return rv.stdout
def __command_create(self):
if not path.isfile(self._args.config):
raise RuntimeError('File not found: %s' % (self._args.config))
variables = {
'root': self._args.location,
'source_dir': path.dirname(self._args.config),
'username': user.USERNAME,
}
if host.SYSTEM == host.MACOS:
variables['DARWIN_USER_CACHE_DIR'] = execute.execute(
'getconf DARWIN_USER_CACHE_DIR').stdout.strip()
cf = config_file(self._args.config, variables)
if self._args.wipe:
file_util.remove(self._args.location)
jail.create(self._args.location, self._config, self._args.no_filters)
return 0
def _run_exe_test(clazz, exe_env_name, exe_default, config, test_source):
context = clazz._make_test_context(config, test_source)
build_blurb.blurb_verbose(
'tester', '%s: running shell test %s' %
(context.package_info.name, test_source))
exe = clazz._determine_exe(exe_env_name, exe_default)
cmd = '%s %s' % (exe, context.test_source_with_replacements)
rv = execute.execute(cmd,
env=context.env,
non_blocking=build_blurb.verbose,
stderr_to_stdout=True,
raise_error=False)
os.environ = context.saved_env
if rv.exit_code != 0:
return step_result(rv.exit_code == 0, rv.stdout)
return step_result(True, None)
def _run_python_test(clazz, config, test_source):
context = clazz._make_test_context(config, test_source)
build_blurb.blurb_verbose(
'tester', '%s: running python test %s' %
(context.package_info.name, test_source))
# FIXME: move this to context
context.env['PYTHONPATH'] = context.package_manager.python_lib_dir
# Run the test
cmd = 'python2.7 %s' % (context.test_source_with_replacements)
rv = execute.execute(cmd,
env=context.env,
non_blocking=build_blurb.verbose,
stderr_to_stdout=True,
raise_error=False)
if rv.exit_code != 0:
return step_result(rv.exit_code == 0, rv.stdout)
return step_result(True, None)
def _call_pkg_config(clazz,
args,
PKG_CONFIG_LIBDIR=[],
PKG_CONFIG_PATH=[]):
check.check_string_seq(PKG_CONFIG_PATH)
cmd = [clazz._PKG_CONFIG_EXE] + object_util.listify(args)
env = {
'PKG_CONFIG_DEBUG_SPEW': '1',
'PKG_CONFIG_LIBDIR': ':'.join(PKG_CONFIG_LIBDIR),
'PKG_CONFIG_PATH': ':'.join(PKG_CONFIG_PATH),
# 'PATH': os_env_var('PATH').value,
}
for p in PKG_CONFIG_PATH:
file_util.mkdir(p)
#build_blurb.blurb_verbose('pkg_config', '_call_pkg_config() cmd=%s' % (str(cmd)))
#print('pkg_config', '_call_pkg_config() cmd=%s; env=%s' % (str(cmd), str(env)))
#print('pkg_config', '_call_pkg_config() cmd=%s' % (str(cmd)))
rv = execute.execute(cmd, env=env)
return rv
def dependencies(clazz, filename):
'Return list of files filename depends on or None if snakefood is not found.'
try:
sfood_exe = file_path.which('sfood')
except RuntimeError as ex:
return None
rv = execute.execute('%s %s' % (sfood_exe, filename))
parser = text_line_parser(rv.stdout)
parser.remove_empties()
parser.append(',')
code = '[ %s ]' % (str(parser))
v = eval(code)
result = []
for item in v:
p1, p2 = item[1]
if p1 and p2:
filename = path.join(p1, p2)
if path.isfile(filename):
result.append(filename)
return result
def dependencies(clazz, filename):
'Return a list of dependencies for filename (executable or shared lib) or None if not applicable.'
filename = path.abspath(filename)
types = [
binary_format_macho.FILE_TYPE_EXECUTABLE,
binary_format_macho.FILE_TYPE_SHARED_LIB
]
if not binary_format_macho().file_is_of_type(filename, types):
return None
cmd = ['otool', '-L', filename]
rv = execute.execute(cmd)
assert rv.stdout.find('is not an object file') == -1
lines = rv.stdout.split('\n')
if len(lines) < 2:
return None
deps = [l.partition(' ')[0].strip() for l in lines[1:]]
deps = [l for l in deps if l]
if filename in deps:
deps.remove(filename)
return sorted(algorithm.unique(deps))
def instructions(self, env):
buf = StringIO()
buf.write('#!/bin/bash\n')
buf.write('echo "----1----"\n')
buf.write('declare -px\n')
buf.write('echo "----2----"\n')
for f in self.files_abs:
buf.write('source \"%s\"\n' % (f))
buf.write('echo "----3----"\n')
buf.write('declare -px\n')
buf.write('echo "----4----"\n')
script = temp_file.make_temp_file(content = buf.getvalue(), delete = not self.DEBUG)
if self.DEBUG:
print('env_dir: script=%s' % (script))
os.chmod(script, 0o755)
try:
rv = execute.execute(script, raise_error = True, shell = True, env = env)
finally:
if not self.DEBUG:
file_util.remove(script)
parser = text_line_parser(rv.stdout)
if rv.stderr:
raise RuntimeError(rv.stderr)
env1 = self._parse_env_lines(parser.cut_lines('----1----', '----2----'))
env2 = self._parse_env_lines(parser.cut_lines('----3----', '----4----'))
delta = self._env_delta(env1, env2)
instructions = []
for key in delta.added:
instructions.append(instruction(key, env2[key], action.SET))
for key in delta.removed:
instructions.append(instruction(key, None, action.UNSET))
for key in delta.changed:
value1 = env1[key]
value2 = env2[key]
for inst in self._determine_change_instructions(key, value1, value2):
instructions.append(inst)
return sorted(instructions, key = lambda x: ( x.key, x.value ) )
def _execute_cmd(clazz, *args):
cmd = ' '.join(args)
clazz.log_i('executing: "%s"' % (cmd))
return execute.execute(cmd)
def test_true_with_shell(self):
script = self.data_path('shell_script_true.sh')
rv = execute.execute(script, shell = True)
self.assertEqual( 0, rv.exit_code )
def tty(clazz):
'Return the current tty for this process.'
return execute.execute('tty').stdout.strip()
def supported_formats(clazz): 'Return a list of names of the formats the tar command line util supports' tmp_dir = temp_file.make_temp_dir() file_util.save(path.join(tmp_dir, 'x.txt', content = 'x')) cmd = 'tar Jcf foo.tar -C %s x.txt' % (tmp_dir) rv = execute.execute(cmd, cwd = tmp_dir, raise_error = False)
def extract_all(self, dest_dir, base_dir = None,
strip_common_ancestor = False, strip_head = None):
dest_dir = self._determine_dest_dir(dest_dir, base_dir)
cmd = 'tar xf %s -C %s' % (self.filename, dest_dir)
execute.execute(cmd)
self._handle_extract_strip_common_ancestor(self.members, strip_common_ancestor, strip_head, dest_dir)
def extract(clazz, filename, dest_dir):
execute.execute('tar xf {filename} -C {dest_dir}'.format(filename = filename, dest_dir = dest_dir))
def members(clazz, filename):
cmd = 'tar tf %s' % (filename)
rv = execute.execute(cmd)
return [ i for i in rv.stdout.split('\n') if i ]
def xcrun(clazz, args):
cmd = [clazz.XCRUN_EXE] + args
return execute.execute(cmd, shell=False)
def test_false_with_shell(self):
script = self.data_path('shell_script_false.sh')
rv = execute.execute(script, raise_error = False, shell = True)
self.assertEqual( 1, rv.exit_code )
def mime_type(clazz, filename):
cmd = 'file --brief --mime %s' % (filename)
rv = execute.execute(cmd, raise_error = False)
if rv.exit_code != 0:
return ''
return rv.stdout.strip()
