def test_substitute(self): self.assertEqual( 'X and Y', variable.substitute('$foo and $bar', { 'foo': 'X', 'bar': 'Y' }) ) # for some dumb reason this is broken in python 2.7 #self.assertEqual( 'XY', variable.substitute('$foo$bar', { 'foo': 'X', 'bar': 'Y' }) ) self.assertEqual( 'X', variable.substitute('$foo', { 'foo': 'X', 'bar': 'Y' }) ) self.assertEqual( 'Y', variable.substitute('$bar', { 'foo': 'X', 'bar': 'Y' }) ) self.assertEqual( '$not', variable.substitute('$not', { 'foo': 'X', 'bar': 'Y' }) ) self.assertEqual( '', variable.substitute('$foo$bar$foo$bar', { 'foo': '', 'bar': '' }) )
def substitute(self, variables): unixpath = [] pythonpath = [] requires = set() for p in self.unixpath: unixpath.append(variable.substitute(p, variables)) for p in self.pythonpath: pythonpath.append(variable.substitute(p, variables)) for p in self.requires: requires.add(variable.substitute(p, variables)) return config_data(self.name, unixpath, pythonpath, requires)
def make_static_lib(self, lib, objects, arflags = None): assert objects objects = object_util.listify(objects) cmd = '$AR $AR_FLAGS %s %s' % (lib, ' '.join(objects)) cmd = variable.substitute(cmd, self.variables) self._execute_cmd(cmd) return lib
def execute(self, script, env, values, inputs): install_files = values.get('install_files') if not install_files: message = 'No install_files for %s' % (script.descriptor.full_name) self.log_d(message) return step_result(True, message) check.check_value_install_file_seq(install_files) for install_file in install_files: src = variable.substitute(install_file.filename, script.substitutions) if not path.isfile(src): return step_result( False, 'File not found at %s: %s' % (str(install_file.origin), path.relpath(src))) dst = path.join(script.staged_files_dir, install_file.dst_filename) dst_dir = path.dirname(dst) mode = file_util.mode(src) self.blurb('Installing file %s in %s (%s)' % (src, dst_dir, mode)) file_util.mkdir(path.dirname(dst)) shutil.copy(src, dst) os.chmod(dst, mode) return step_result(True, None)
def _make_standalone_python(self, program, script, env): src_basename = path.basename(program.filename) dst_basename = path.basename(program.dst_filename) if dst_basename.endswith('.py'): return step_result( False, 'dst program should not end in .py: %s' % (dst_basename)) src_program = variable.substitute(program.filename, script.substitutions) if not path.isabs(src_program): src_program = path.join(script.build_dir, src_program) if not path.isfile(src_program): return step_result(False, 'src program not found: %s' % (src_program)) tmp_src_program = path.join(script.build_dir, dst_basename + '.py') file_util.copy(src_program, tmp_src_program) dst_program = path.join(script.build_dir, 'dist', dst_basename) cmd = 'pyinstaller --log INFO -F %s' % (tmp_src_program) rv = self.call_shell(cmd, script, env) if not rv.success: return rv if not path.isfile(dst_program): return step_result(False, 'dst program not found: %s' % (dst_program)) installed_program = path.join(script.staged_files_dir, program.dst_filename) file_util.mkdir(path.dirname(installed_program)) file_util.copy(dst_program, installed_program) os.chmod(installed_program, 0o755) return step_result(True, None)
def make_shared_lib(self, lib, objects, ldflags = None): assert objects ldflags = ldflags or [] objects = object_util.listify(objects) cmd = '$CC -shared $LDFLAGS %s %s -o %s' % (' '.join(ldflags), ' '.join(objects), lib) cmd = variable.substitute(cmd, self.variables) self._execute_cmd(cmd) return lib
def link_exe(self, exe, objects, ldflags = None): assert objects ldflags = ldflags or [] objects = object_util.listify(objects) cmd = '$CC %s -o %s $LDFLAGS %s' % (' '.join(objects), exe, ' '.join(ldflags)) cmd = variable.substitute(cmd, self.variables) self._execute_cmd(cmd) return exe
def call_shell(clazz, command, script, env, shell_env = None, save_logs = None, execution_dir = None): command = execute.listify_command(command) command = [ part for part in command if part ] shell_env = shell_env or key_value_list() save_logs = save_logs or [] check.check_key_value_list(shell_env) log.log_i('rebuild', 'call_shell(command=%s)' % (command)) build_blurb.blurb_verbose('rebuild', 'call_shell(cmd=%s, shell_env=%s)' % (command, shell_env)) env = clazz.create_command_env(script) env.update(shell_env) #clazz.env_dump(env, script.descriptor.name, 'PRE ENVIRONMENT') clazz._env_substitite(env) rogue_key = clazz._env_find_roque_dollar_sign(env) if rogue_key: raise RuntimeError('Rogue dollar sign (\"$\") found in %s: %s' % (rogue_key, env[rogue_key])) command = [ variable.substitute(part, env) for part in object_util.listify(command) ] # Check that no rogue dollar signs are in the command for part in command: if variable.has_rogue_dollar_signs(part): raise RuntimeError('Rogue dollar sign (\"$\") found in: %s' % (part)) build_blurb.blurb('rebuild', '%s - %s' % (script.descriptor.name, ' '.join(command))) # file_util.mkdir(script.build_dir) retry_script = clazz._write_retry_script(command, env, script) #clazz.env_dump(env, script.descriptor.name, clazz.__name__ + ' : ' + 'POST ENVIRONMENT') for k,v in env.items(): if string_util.is_quoted(v): env[k] = string_util.unquote(v) if execution_dir: cwd = path.join(script.build_dir, execution_dir) else: cwd = script.build_dir rv = execute.execute(command, cwd = cwd, env = env, shell = True, non_blocking = build_blurb.verbose, stderr_to_stdout = build_blurb.verbose, raise_error = False) message = rv.stdout if rv.stderr: message = message + '\n' + rv.stderr result = step_result(rv.exit_code == 0, message) clazz.save_build_dir_logs(script, save_logs) return result
def make_default_replacements(r, name): result = {} s = { 'template': name, 'TEMPLATE': name.upper(), } for key, value in r.items(): result[key] = variable.substitute(value, s) print('BEFORE: %s' % (value)) print(' AFTER: %s' % (result[key])) return result
def _sources(self, all_scripts): 'Return a list of all script and dependency sources for this script.' sources = self._script_sources() + self._dep_sources(all_scripts) result = [] for source in sources: s = variable.substitute(source, self.substitutions) if path.isfile(s): result.append(s) else: pass #print('you suck so bad because file not found: %s' % (s)) return result
def test_substitute_parenthesis(self): self.assertEqual( 'X and Y', variable.substitute('$(foo) and $(bar)', { 'foo': 'X', 'bar': 'Y' }) ) self.assertEqual( 'XY', variable.substitute('$(foo)$(bar)', { 'foo': 'X', 'bar': 'Y' }) ) self.assertEqual( 'X', variable.substitute('$(foo)', { 'foo': 'X', 'bar': 'Y' }) ) self.assertEqual( 'Y', variable.substitute('$(bar)', { 'foo': 'X', 'bar': 'Y' }) ) self.assertEqual( '$(not)', variable.substitute('$(not)', { 'foo': 'X', 'bar': 'Y' }) ) self.assertEqual( '', variable.substitute('$(foo)$(bar)$(foo)$(bar)', { 'foo': '', 'bar': '' }) )
def test_substitute_brackets(self): self.assertEqual( 'X and Y', variable.substitute('${foo} and ${bar}', { 'foo': 'X', 'bar': 'Y' }) ) self.assertEqual( 'XY', variable.substitute('${foo}${bar}', { 'foo': 'X', 'bar': 'Y' }) ) self.assertEqual( 'X', variable.substitute('${foo}', { 'foo': 'X', 'bar': 'Y' }) ) self.assertEqual( 'Y', variable.substitute('${bar}', { 'foo': 'X', 'bar': 'Y' }) ) self.assertEqual( '${not}', variable.substitute('${not}', { 'foo': 'X', 'bar': 'Y' }) ) self.assertEqual( '', variable.substitute('${foo}${bar}${foo}${bar}', { 'foo': '', 'bar': '' }) )
def test_substitute_at_sign(self): self.assertEqual( 'X and Y', variable.substitute('@foo@ and @bar@', { 'foo': 'X', 'bar': 'Y' }) ) self.assertEqual( 'XY', variable.substitute('@foo@@bar@', { 'foo': 'X', 'bar': 'Y' }) ) self.assertEqual( 'X', variable.substitute('@foo@', { 'foo': 'X', 'bar': 'Y' }) ) self.assertEqual( 'Y', variable.substitute('@bar@', { 'foo': 'X', 'bar': 'Y' }) ) self.assertEqual( '@not@', variable.substitute('@not@', { 'foo': 'X', 'bar': 'Y' }) ) self.assertEqual( '', variable.substitute('@foo@@bar@@foo@@bar@', { 'foo': '', 'bar': '' }) )
def copy_with_substitute(clazz, src, dst, replacements, backup = True): assert isinstance(replacements, dict) content = file_util.read(src, 'utf-8') new_content = variable.substitute(content, replacements) old_content = None if path.exists(dst): old_content = file_util.read(dst, 'utf-8') if old_content == new_content: return False if backup: file_util.backup(dst) file_util.save(dst, content = new_content, mode = file_util.mode(src)) return True
def compile_c(self, sources, objects = None, cflags = None): assert sources cflags = cflags or [] sources = object_util.listify(sources) if objects: objects = object_util.listify(objects) targets = self._make_targets(sources, objects) for src, obj in targets: cmd = '$CC $CFLAGS %s -c $(SRC) -o $(OBJ)' % (' '.join(cflags)) variables = copy.deepcopy(self.variables) variables['SRC'] = src variables['OBJ'] = obj cmd = variable.substitute(cmd, variables) print(cmd) self._execute_cmd(cmd) return targets
def _make_standalone_shell_script(self, program, script, env): src_basename = path.basename(program.filename) dst_basename = path.basename(program.dst_filename) if dst_basename.endswith('.sh'): return step_result( False, 'dst program should not end in .sh: %s' % (dst_basename)) src_program = variable.substitute(program.filename, script.substitutions) if not path.isabs(src_program): src_program = path.join(script.build_dir, src_program) if not path.isfile(src_program): return step_result(False, 'src program not found: %s' % (src_program)) installed_program = path.join(script.staged_files_dir, program.dst_filename) file_util.mkdir(path.dirname(installed_program)) file_util.copy(src_program, installed_program) os.chmod(installed_program, 0o755) return step_result(True, None)
def parse_expression(self, expression): variables = { 'system': self.system, 'arch': self._arch_to_string(self.arch), 'level': self.level, 'distro': self.distro or 'None', } dict_util.quote_strings(variables) exp_with_vars = variable.substitute(expression, variables) constants = { 'MACOS': 'macos', 'LINUX': 'linux', 'RELEASE': 'release', 'DEBUG': 'debug', 'RASPBIAN': 'raspbian', } dict_util.quote_strings(constants) exp_with_consts = string_util.replace(exp_with_vars, constants, word_boundary=True) return eval(exp_with_consts)
def substitute(self, text): return variable.substitute(text, self._substitutions)
def _env_substitite(clazz, env): for key in sorted(env.keys()): env[key] = variable.substitute(str(env[key]), env)
def __parse_list(clazz, s, variables): result = string_util.split_by_white_space(s, strip=True) return [variable.substitute(x, variables) for x in result]
def __parse_list_lines(clazz, s, variables): result = [x.strip() for x in s.split('\n')] return [variable.substitute(x, variables) for x in result if x]
def test_substitute_nested(self): self.assertEqual( 'X and ZY and Z', variable.substitute('${foo} and ${bar} and ${kiwi}', { 'foo': 'X', 'bar': '${kiwi}Y', 'kiwi': 'Z' }) )
def test_invalid_key(self): with self.assertRaises(TypeError) as context: variable.substitute('$foo and $bar', { 'foo': [ 'x' ], 'bar': 'Y' })
def test_substitute_word_boundary(self): self.assertEqual( 'X and Y and Z', variable.substitute('$foo and $foo_bar and $foo_bar_baz', { 'foo': 'X', 'foo_bar': 'Y', 'foo_bar_baz': 'Z' }) )
def test_substitute_mixed(self): self.assertEqual( 'X and Y and Z and A', variable.substitute('$foo and ${bar} and ${baz} and @PIP@', { 'foo': 'X', 'bar': 'Y', 'baz': 'Z', 'PIP': 'A' }) )
def replace(self, replacements): self.value = variable.substitute(self.value, replacements)
def _make_test_context(clazz, config, test_source): timer = debug_timer('am', 'error', disabled=True) timer.start('_make_test_context()') test_name = path.splitext(path.basename(test_source))[0] test_root_dir = path.join(config.script.test_dir, test_name) pm_root_dir = path.join(test_root_dir, 'requirements') pm = package_manager(pm_root_dir, config.artifact_manager) timer.start('load tarball') package = pm.load_tarball(config.package_tarball, config.script.build_target) timer.stop() pd = package.package_descriptor timer.start('resolve_deps') deps_packages = config.script.resolve_deps(['RUN', 'TEST'], False) timer.stop() all_packages = deps_packages + [pd] all_packages_names = [p.name for p in all_packages] timer.start('install packages') pm.install_packages(deps_packages, config.script.build_target, ['RUN', 'TEST']) timer.stop() timer.start('install tarball') pm.install_tarball(config.package_tarball, ['RUN', 'TEST']) timer.stop() tool_reqs = pd.requirements.filter_by( ['TOOL'], config.script.env.config.host_build_target.system) tool_reqs_names = tool_reqs.names() timer.start('resolve tools deps') resolved_tool_reqs = config.artifact_manager.resolve_deps( tool_reqs_names, config.script.env.config.host_build_target, ['TOOL'], True) timer.stop() timer.start('tools update') config.tools_manager.ensure_tools(resolved_tool_reqs) timer.stop() saved_env = os_env.clone_current_env() shell_env = os_env.make_clean_env() shell_env = config.tools_manager.transform_env(shell_env, resolved_tool_reqs) shell_env = pm.transform_env(shell_env, all_packages_names) test_source_with_replacements = path.join(test_root_dir, path.basename(test_source)) substitutions = {} substitutions.update(config.script.substitutions) substitutions.update({ 'REBUILD_TEST_NAME': test_name, 'REBUILD_SHELL_FRAMEWORK_DIR': pm.shell_framework_dir, '_BES_DEV_ROOT': pm.shell_framework_dir, }) for kv in config.extra_env: shell_env[kv.key] = variable.substitute(kv.value, substitutions) shell_env.update(substitutions) file_replace.copy_with_substitute(test_source, test_source_with_replacements, substitutions, backup=False) timer.stop() return clazz._test_context(package.package_descriptor, shell_env, saved_env, test_root_dir, test_name, test_source_with_replacements, pm)
def substitute_variables(self, d, word_boundary = True): for i, kv in enumerate(self._values): self._values[i] = key_value(kv.key, variable.substitute(kv.value, d, word_boundary = word_boundary))