def test_substitute(self):
self.assertEqual( 'X and Y', variable.substitute('$foo and $bar', { 'foo': 'X', 'bar': 'Y' }) )
# for some dumb reason this is broken in python 2.7
#self.assertEqual( 'XY', variable.substitute('$foo$bar', { 'foo': 'X', 'bar': 'Y' }) )
self.assertEqual( 'X', variable.substitute('$foo', { 'foo': 'X', 'bar': 'Y' }) )
self.assertEqual( 'Y', variable.substitute('$bar', { 'foo': 'X', 'bar': 'Y' }) )
self.assertEqual( '$not', variable.substitute('$not', { 'foo': 'X', 'bar': 'Y' }) )
self.assertEqual( '', variable.substitute('$foo$bar$foo$bar', { 'foo': '', 'bar': '' }) )
def substitute(self, variables):
unixpath = []
pythonpath = []
requires = set()
for p in self.unixpath:
unixpath.append(variable.substitute(p, variables))
for p in self.pythonpath:
pythonpath.append(variable.substitute(p, variables))
for p in self.requires:
requires.add(variable.substitute(p, variables))
return config_data(self.name, unixpath, pythonpath, requires)
def make_static_lib(self, lib, objects, arflags = None): assert objects objects = object_util.listify(objects) cmd = '$AR $AR_FLAGS %s %s' % (lib, ' '.join(objects)) cmd = variable.substitute(cmd, self.variables) self._execute_cmd(cmd) return lib
def execute(self, script, env, values, inputs):
install_files = values.get('install_files')
if not install_files:
message = 'No install_files for %s' % (script.descriptor.full_name)
self.log_d(message)
return step_result(True, message)
check.check_value_install_file_seq(install_files)
for install_file in install_files:
src = variable.substitute(install_file.filename,
script.substitutions)
if not path.isfile(src):
return step_result(
False, 'File not found at %s: %s' %
(str(install_file.origin), path.relpath(src)))
dst = path.join(script.staged_files_dir, install_file.dst_filename)
dst_dir = path.dirname(dst)
mode = file_util.mode(src)
self.blurb('Installing file %s in %s (%s)' % (src, dst_dir, mode))
file_util.mkdir(path.dirname(dst))
shutil.copy(src, dst)
os.chmod(dst, mode)
return step_result(True, None)
def _make_standalone_python(self, program, script, env):
src_basename = path.basename(program.filename)
dst_basename = path.basename(program.dst_filename)
if dst_basename.endswith('.py'):
return step_result(
False,
'dst program should not end in .py: %s' % (dst_basename))
src_program = variable.substitute(program.filename,
script.substitutions)
if not path.isabs(src_program):
src_program = path.join(script.build_dir, src_program)
if not path.isfile(src_program):
return step_result(False,
'src program not found: %s' % (src_program))
tmp_src_program = path.join(script.build_dir, dst_basename + '.py')
file_util.copy(src_program, tmp_src_program)
dst_program = path.join(script.build_dir, 'dist', dst_basename)
cmd = 'pyinstaller --log INFO -F %s' % (tmp_src_program)
rv = self.call_shell(cmd, script, env)
if not rv.success:
return rv
if not path.isfile(dst_program):
return step_result(False,
'dst program not found: %s' % (dst_program))
installed_program = path.join(script.staged_files_dir,
program.dst_filename)
file_util.mkdir(path.dirname(installed_program))
file_util.copy(dst_program, installed_program)
os.chmod(installed_program, 0o755)
return step_result(True, None)
def make_shared_lib(self, lib, objects, ldflags = None):
assert objects
ldflags = ldflags or []
objects = object_util.listify(objects)
cmd = '$CC -shared $LDFLAGS %s %s -o %s' % (' '.join(ldflags), ' '.join(objects), lib)
cmd = variable.substitute(cmd, self.variables)
self._execute_cmd(cmd)
return lib
def link_exe(self, exe, objects, ldflags = None):
assert objects
ldflags = ldflags or []
objects = object_util.listify(objects)
cmd = '$CC %s -o %s $LDFLAGS %s' % (' '.join(objects), exe, ' '.join(ldflags))
cmd = variable.substitute(cmd, self.variables)
self._execute_cmd(cmd)
return exe
def call_shell(clazz, command, script, env, shell_env = None, save_logs = None, execution_dir = None):
command = execute.listify_command(command)
command = [ part for part in command if part ]
shell_env = shell_env or key_value_list()
save_logs = save_logs or []
check.check_key_value_list(shell_env)
log.log_i('rebuild', 'call_shell(command=%s)' % (command))
build_blurb.blurb_verbose('rebuild', 'call_shell(cmd=%s, shell_env=%s)' % (command, shell_env))
env = clazz.create_command_env(script)
env.update(shell_env)
#clazz.env_dump(env, script.descriptor.name, 'PRE ENVIRONMENT')
clazz._env_substitite(env)
rogue_key = clazz._env_find_roque_dollar_sign(env)
if rogue_key:
raise RuntimeError('Rogue dollar sign (\"$\") found in %s: %s' % (rogue_key, env[rogue_key]))
command = [ variable.substitute(part, env) for part in object_util.listify(command) ]
# Check that no rogue dollar signs are in the command
for part in command:
if variable.has_rogue_dollar_signs(part):
raise RuntimeError('Rogue dollar sign (\"$\") found in: %s' % (part))
build_blurb.blurb('rebuild', '%s - %s' % (script.descriptor.name, ' '.join(command)))
# file_util.mkdir(script.build_dir)
retry_script = clazz._write_retry_script(command, env, script)
#clazz.env_dump(env, script.descriptor.name, clazz.__name__ + ' : ' + 'POST ENVIRONMENT')
for k,v in env.items():
if string_util.is_quoted(v):
env[k] = string_util.unquote(v)
if execution_dir:
cwd = path.join(script.build_dir, execution_dir)
else:
cwd = script.build_dir
rv = execute.execute(command,
cwd = cwd,
env = env,
shell = True,
non_blocking = build_blurb.verbose,
stderr_to_stdout = build_blurb.verbose,
raise_error = False)
message = rv.stdout
if rv.stderr:
message = message + '\n' + rv.stderr
result = step_result(rv.exit_code == 0, message)
clazz.save_build_dir_logs(script, save_logs)
return result
def make_default_replacements(r, name):
result = {}
s = {
'template': name,
'TEMPLATE': name.upper(),
}
for key, value in r.items():
result[key] = variable.substitute(value, s)
print('BEFORE: %s' % (value))
print(' AFTER: %s' % (result[key]))
return result
def _sources(self, all_scripts):
'Return a list of all script and dependency sources for this script.'
sources = self._script_sources() + self._dep_sources(all_scripts)
result = []
for source in sources:
s = variable.substitute(source, self.substitutions)
if path.isfile(s):
result.append(s)
else:
pass #print('you suck so bad because file not found: %s' % (s))
return result
def test_substitute_parenthesis(self):
self.assertEqual( 'X and Y', variable.substitute('$(foo) and $(bar)', { 'foo': 'X', 'bar': 'Y' }) )
self.assertEqual( 'XY', variable.substitute('$(foo)$(bar)', { 'foo': 'X', 'bar': 'Y' }) )
self.assertEqual( 'X', variable.substitute('$(foo)', { 'foo': 'X', 'bar': 'Y' }) )
self.assertEqual( 'Y', variable.substitute('$(bar)', { 'foo': 'X', 'bar': 'Y' }) )
self.assertEqual( '$(not)', variable.substitute('$(not)', { 'foo': 'X', 'bar': 'Y' }) )
self.assertEqual( '', variable.substitute('$(foo)$(bar)$(foo)$(bar)', { 'foo': '', 'bar': '' }) )
def test_substitute_brackets(self):
self.assertEqual( 'X and Y', variable.substitute('${foo} and ${bar}', { 'foo': 'X', 'bar': 'Y' }) )
self.assertEqual( 'XY', variable.substitute('${foo}${bar}', { 'foo': 'X', 'bar': 'Y' }) )
self.assertEqual( 'X', variable.substitute('${foo}', { 'foo': 'X', 'bar': 'Y' }) )
self.assertEqual( 'Y', variable.substitute('${bar}', { 'foo': 'X', 'bar': 'Y' }) )
self.assertEqual( '${not}', variable.substitute('${not}', { 'foo': 'X', 'bar': 'Y' }) )
self.assertEqual( '', variable.substitute('${foo}${bar}${foo}${bar}', { 'foo': '', 'bar': '' }) )
def test_substitute_at_sign(self):
self.assertEqual( 'X and Y', variable.substitute('@foo@ and @bar@', { 'foo': 'X', 'bar': 'Y' }) )
self.assertEqual( 'XY', variable.substitute('@foo@@bar@', { 'foo': 'X', 'bar': 'Y' }) )
self.assertEqual( 'X', variable.substitute('@foo@', { 'foo': 'X', 'bar': 'Y' }) )
self.assertEqual( 'Y', variable.substitute('@bar@', { 'foo': 'X', 'bar': 'Y' }) )
self.assertEqual( '@not@', variable.substitute('@not@', { 'foo': 'X', 'bar': 'Y' }) )
self.assertEqual( '', variable.substitute('@foo@@bar@@foo@@bar@', { 'foo': '', 'bar': '' }) )
def copy_with_substitute(clazz, src, dst, replacements, backup = True):
assert isinstance(replacements, dict)
content = file_util.read(src, 'utf-8')
new_content = variable.substitute(content, replacements)
old_content = None
if path.exists(dst):
old_content = file_util.read(dst, 'utf-8')
if old_content == new_content:
return False
if backup:
file_util.backup(dst)
file_util.save(dst, content = new_content, mode = file_util.mode(src))
return True
def compile_c(self, sources, objects = None, cflags = None):
assert sources
cflags = cflags or []
sources = object_util.listify(sources)
if objects:
objects = object_util.listify(objects)
targets = self._make_targets(sources, objects)
for src, obj in targets:
cmd = '$CC $CFLAGS %s -c $(SRC) -o $(OBJ)' % (' '.join(cflags))
variables = copy.deepcopy(self.variables)
variables['SRC'] = src
variables['OBJ'] = obj
cmd = variable.substitute(cmd, variables)
print(cmd)
self._execute_cmd(cmd)
return targets
def _make_standalone_shell_script(self, program, script, env):
src_basename = path.basename(program.filename)
dst_basename = path.basename(program.dst_filename)
if dst_basename.endswith('.sh'):
return step_result(
False,
'dst program should not end in .sh: %s' % (dst_basename))
src_program = variable.substitute(program.filename,
script.substitutions)
if not path.isabs(src_program):
src_program = path.join(script.build_dir, src_program)
if not path.isfile(src_program):
return step_result(False,
'src program not found: %s' % (src_program))
installed_program = path.join(script.staged_files_dir,
program.dst_filename)
file_util.mkdir(path.dirname(installed_program))
file_util.copy(src_program, installed_program)
os.chmod(installed_program, 0o755)
return step_result(True, None)
def parse_expression(self, expression):
variables = {
'system': self.system,
'arch': self._arch_to_string(self.arch),
'level': self.level,
'distro': self.distro or 'None',
}
dict_util.quote_strings(variables)
exp_with_vars = variable.substitute(expression, variables)
constants = {
'MACOS': 'macos',
'LINUX': 'linux',
'RELEASE': 'release',
'DEBUG': 'debug',
'RASPBIAN': 'raspbian',
}
dict_util.quote_strings(constants)
exp_with_consts = string_util.replace(exp_with_vars,
constants,
word_boundary=True)
return eval(exp_with_consts)
def substitute(self, text):
return variable.substitute(text, self._substitutions)
def _env_substitite(clazz, env):
for key in sorted(env.keys()):
env[key] = variable.substitute(str(env[key]), env)
def __parse_list(clazz, s, variables):
result = string_util.split_by_white_space(s, strip=True)
return [variable.substitute(x, variables) for x in result]
def __parse_list_lines(clazz, s, variables):
result = [x.strip() for x in s.split('\n')]
return [variable.substitute(x, variables) for x in result if x]
def test_substitute_nested(self):
self.assertEqual( 'X and ZY and Z', variable.substitute('${foo} and ${bar} and ${kiwi}', { 'foo': 'X', 'bar': '${kiwi}Y', 'kiwi': 'Z' }) )
def test_invalid_key(self):
with self.assertRaises(TypeError) as context:
variable.substitute('$foo and $bar', { 'foo': [ 'x' ], 'bar': 'Y' })
def test_substitute_word_boundary(self):
self.assertEqual( 'X and Y and Z', variable.substitute('$foo and $foo_bar and $foo_bar_baz',
{ 'foo': 'X', 'foo_bar': 'Y', 'foo_bar_baz': 'Z' }) )
def test_substitute_mixed(self):
self.assertEqual( 'X and Y and Z and A', variable.substitute('$foo and ${bar} and ${baz} and @PIP@',
{ 'foo': 'X', 'bar': 'Y', 'baz': 'Z', 'PIP': 'A' }) )
def replace(self, replacements):
self.value = variable.substitute(self.value, replacements)
def _make_test_context(clazz, config, test_source):
timer = debug_timer('am', 'error', disabled=True)
timer.start('_make_test_context()')
test_name = path.splitext(path.basename(test_source))[0]
test_root_dir = path.join(config.script.test_dir, test_name)
pm_root_dir = path.join(test_root_dir, 'requirements')
pm = package_manager(pm_root_dir, config.artifact_manager)
timer.start('load tarball')
package = pm.load_tarball(config.package_tarball,
config.script.build_target)
timer.stop()
pd = package.package_descriptor
timer.start('resolve_deps')
deps_packages = config.script.resolve_deps(['RUN', 'TEST'], False)
timer.stop()
all_packages = deps_packages + [pd]
all_packages_names = [p.name for p in all_packages]
timer.start('install packages')
pm.install_packages(deps_packages, config.script.build_target,
['RUN', 'TEST'])
timer.stop()
timer.start('install tarball')
pm.install_tarball(config.package_tarball, ['RUN', 'TEST'])
timer.stop()
tool_reqs = pd.requirements.filter_by(
['TOOL'], config.script.env.config.host_build_target.system)
tool_reqs_names = tool_reqs.names()
timer.start('resolve tools deps')
resolved_tool_reqs = config.artifact_manager.resolve_deps(
tool_reqs_names, config.script.env.config.host_build_target,
['TOOL'], True)
timer.stop()
timer.start('tools update')
config.tools_manager.ensure_tools(resolved_tool_reqs)
timer.stop()
saved_env = os_env.clone_current_env()
shell_env = os_env.make_clean_env()
shell_env = config.tools_manager.transform_env(shell_env,
resolved_tool_reqs)
shell_env = pm.transform_env(shell_env, all_packages_names)
test_source_with_replacements = path.join(test_root_dir,
path.basename(test_source))
substitutions = {}
substitutions.update(config.script.substitutions)
substitutions.update({
'REBUILD_TEST_NAME': test_name,
'REBUILD_SHELL_FRAMEWORK_DIR': pm.shell_framework_dir,
'_BES_DEV_ROOT': pm.shell_framework_dir,
})
for kv in config.extra_env:
shell_env[kv.key] = variable.substitute(kv.value, substitutions)
shell_env.update(substitutions)
file_replace.copy_with_substitute(test_source,
test_source_with_replacements,
substitutions,
backup=False)
timer.stop()
return clazz._test_context(package.package_descriptor, shell_env,
saved_env, test_root_dir, test_name,
test_source_with_replacements, pm)
def substitute_variables(self, d, word_boundary = True):
for i, kv in enumerate(self._values):
self._values[i] = key_value(kv.key, variable.substitute(kv.value, d, word_boundary = word_boundary))
