def _match(clazz, filenames, patterns, match_func, match_type): ''' Match a list of files with patterns using match_func and match_type. match_func should be the form match_func(filename, patterns) ''' assert clazz.match_type_is_valid(match_type) filenames = object_util.listify(filenames) patterns = object_util.listify(patterns) # or []) result = [] if not patterns: if match_type == clazz.ANY: return [] elif match_type == clazz.NONE: return filenames elif match_type == clazz.ALL: return [] func_map = { clazz.ANY: clazz._match_any, clazz.NONE: clazz._match_none, clazz.ALL: clazz._match_all, } func = func_map[match_type] result = [] for filename in filenames: if func(match_func, path.basename(filename), patterns): result.append(filename) return sorted(algorithm.unique(result))
def make_static_lib(self, lib, objects, arflags = None): assert objects objects = object_util.listify(objects) cmd = '$AR $AR_FLAGS %s %s' % (lib, ' '.join(objects)) cmd = variable.substitute(cmd, self.variables) self._execute_cmd(cmd) return lib
def _requires_no_resolve(self, modules): modules = object_util.listify(modules) check.check_string_seq(modules) reqs = [] for name in modules: reqs.extend(self._get_pc_file(module_name).requires) return algorithm.unique(reqs)
def _make_test_amt(self, recipes, publish): amt = AMT(recipes=recipes) publish = object_util.listify(publish) check.check_string_seq(publish) for p in publish: amt.publish(p) return amt
def remove_checksums(self, packages, level): packages = object_util.listify(packages) check.check_package_descriptor_seq(packages) checksum_dirs = [self._checksum_dir(pd, level) for pd in packages] for d in checksum_dirs: self.blurb('removing checksums: %s' % (path.relpath(d))) file_util.remove(checksum_dirs)
def resolve_deps(self, names, system, hardness, include_names): 'Resolve dependencies.' self.log_i( 'resolve_deps(names=%s, system=%s, hardness=%s, include_names=%s)' % (' '.join(names), system, ' '.join(hardness), include_names)) check.check_string_seq(names) names = object_util.listify(names) hardness = self._normalize_hardness(hardness) reqs = requirement_list() for name in names: desc = self._descriptor_map.get(name, None) if not desc: raise KeyError('Not found in packages: %s' % (name)) reqs.extend(desc.requirements.filter_by(hardness, system)) self.log_i('resolve_deps() reqs=%s' % (str(reqs.names()))) dep_map = self.dependency_map(hardness, system) result = package_descriptor_list( dependency_resolver.resolve_and_order_deps(reqs.names(), self._descriptor_map, dep_map)) self.log_i('resolve_deps() 1 result=%s' % (str(result.names()))) if include_names: result += self.descriptors(names) result.remove_dups() self.log_i('resolve_deps() 2 result=%s' % (str(result.names()))) return result
def compile_c(self, sources, objects = None, cflags = None): assert sources cflags = cflags or [] sources = object_util.listify(sources) if objects: objects = object_util.listify(objects) targets = self._make_targets(sources, objects) for src, obj in targets: cmd = '$CC $CFLAGS %s -c $(SRC) -o $(OBJ)' % (' '.join(cflags)) variables = copy.deepcopy(self.variables) variables['SRC'] = src variables['OBJ'] = obj cmd = variable.substitute(cmd, variables) print(cmd) self._execute_cmd(cmd) return targets
def link_exe(self, exe, objects, ldflags = None): assert objects ldflags = ldflags or [] objects = object_util.listify(objects) cmd = '$CC %s -o %s $LDFLAGS %s' % (' '.join(objects), exe, ' '.join(ldflags)) cmd = variable.substitute(cmd, self.variables) self._execute_cmd(cmd) return exe
def glob(clazz, paths, glob_expression): 'Like glob but handles a single path or a sequence of paths' paths = object_util.listify(paths) paths = [ path.join(p, glob_expression) for p in paths ] result = [] for p in paths: result.extend(glob.glob(p)) return sorted(algorithm.unique(result))
def make_shared_lib(self, lib, objects, ldflags = None): assert objects ldflags = ldflags or [] objects = object_util.listify(objects) cmd = '$CC -shared $LDFLAGS %s %s -o %s' % (' '.join(ldflags), ' '.join(objects), lib) cmd = variable.substitute(cmd, self.variables) self._execute_cmd(cmd) return lib
def call_shell(clazz, command, script, env, shell_env = None, save_logs = None, execution_dir = None): command = execute.listify_command(command) command = [ part for part in command if part ] shell_env = shell_env or key_value_list() save_logs = save_logs or [] check.check_key_value_list(shell_env) log.log_i('rebuild', 'call_shell(command=%s)' % (command)) build_blurb.blurb_verbose('rebuild', 'call_shell(cmd=%s, shell_env=%s)' % (command, shell_env)) env = clazz.create_command_env(script) env.update(shell_env) #clazz.env_dump(env, script.descriptor.name, 'PRE ENVIRONMENT') clazz._env_substitite(env) rogue_key = clazz._env_find_roque_dollar_sign(env) if rogue_key: raise RuntimeError('Rogue dollar sign (\"$\") found in %s: %s' % (rogue_key, env[rogue_key])) command = [ variable.substitute(part, env) for part in object_util.listify(command) ] # Check that no rogue dollar signs are in the command for part in command: if variable.has_rogue_dollar_signs(part): raise RuntimeError('Rogue dollar sign (\"$\") found in: %s' % (part)) build_blurb.blurb('rebuild', '%s - %s' % (script.descriptor.name, ' '.join(command))) # file_util.mkdir(script.build_dir) retry_script = clazz._write_retry_script(command, env, script) #clazz.env_dump(env, script.descriptor.name, clazz.__name__ + ' : ' + 'POST ENVIRONMENT') for k,v in env.items(): if string_util.is_quoted(v): env[k] = string_util.unquote(v) if execution_dir: cwd = path.join(script.build_dir, execution_dir) else: cwd = script.build_dir rv = execute.execute(command, cwd = cwd, env = env, shell = True, non_blocking = build_blurb.verbose, stderr_to_stdout = build_blurb.verbose, raise_error = False) message = rv.stdout if rv.stderr: message = message + '\n' + rv.stderr result = step_result(rv.exit_code == 0, message) clazz.save_build_dir_logs(script, save_logs) return result
def match_all(self, expressions, strip_comments = False): expressions = object_util.listify(expressions) result = [] for line in self._lines: text = line.get_text(strip_comments = strip_comments) for expression in expressions: if re.match(expression, text): result.append(line) return text_line_parser(result)
def libs(clazz, modules, PKG_CONFIG_PATH=[], static=False): modules = object_util.listify(modules) args = ['--libs'] + modules if static: args.append('--static') rv = clazz._call_pkg_config(args, PKG_CONFIG_PATH=PKG_CONFIG_PATH) libs = clazz._parse_flags(rv.stdout) libs = clazz._libs_cleanup(libs) return libs
def _resolve_requires(self, modules): modules = object_util.listify(modules) check.check_string_seq(modules) immediate_reqs = self.module_requires(module_name) immediate_req_names = [req.name for req in reqs] order = dependency_resolver.resolve_deps(self.dep_map, req_names) for mod in reqs: cflags.extend(self._single_module_cflags(mod.name)) cflags.extend(self._single_module_cflags(module_name)) return cflags
def find(clazz, dirs, name, version): dirs = object_util.listify(dirs) filenames = [] for d in dirs: if path.isdir(d): filenames += file_find.find(d, max_depth=4, relative=False, file_type=file_find.FILE | file_find.LINK) return clazz.find_in_list(filenames, name, version)
def hardness_matches(self, hardness): 'Return True if hardness matches.' hardness = object_util.listify(hardness) if not requirement_hardness.is_valid_seq(hardness): raise ValueError('invalid hardness: %s - %s' % (str(hardness), type(hardness))) self_hardness = self.hardness or requirement_hardness.DEFAULT for h in hardness: if self_hardness == requirement_hardness(h): return True return False
def remove(clazz, files): files = object_util.listify(files) for f in files: try: if path.isdir(f): shutil.rmtree(f) else: os.remove(f) except Exception as ex: clazz.log_d('Caught exception %s removing %s' % (ex, f)) pass
def match_first(self, expressions, strip_comments = False, line_number = None): expressions = object_util.listify(expressions) for line in self._lines: if line_number is not None: if line.line_number < line_number: continue text = line.get_text(strip_comments = strip_comments) for expression in expressions: match = re.findall(expression, text) if match: return self._match_result(expression, match, line) return None
def match_backwards(self, line_number, expressions, strip_comments = False): expressions = object_util.listify(expressions) start_index = self.find_by_line_number(line_number) - 1 if start_index < 0: return None for i in range(start_index, -1, -1): line = self._lines[i] text = line.get_text(strip_comments = strip_comments) for expression in expressions: match = re.findall(expression, text) if match: return self._match_result(expression, match, line) return None
def patch(clazz, patches, cwd, strip=1, backup=True, posix=True, program=None): 'Apply the given patches by calling patch with strip, backup and posix.' patches = object_util.listify(patches) target_files = file_find.find(cwd, relative=True) for patch in patches: rv = clazz._call_patch(patch, cwd, strip, backup, posix, program) if rv[0] != 0: sys.stderr.write(rv[1]) sys.stderr.flush() return rv return (0, None)
def resolve_deps(clazz, dep_map, names): ''' Return a set of resolved dependencies for the given name or names. Sorted alphabetically, not in build order. ''' cyclic_deps = clazz.cyclic_deps(dep_map) if len(cyclic_deps) > 0: raise cyclic_dependency_error('Cyclic dependencies found: %s' % (' '.join(cyclic_deps)), cyclic_deps) order = clazz.build_order_flat(dep_map) names = object_util.listify(names) result = set(names) for name in names: result |= clazz._resolve_deps(dep_map, name) return sorted(list(result))
def _call_pkg_config(clazz, args, PKG_CONFIG_LIBDIR=[], PKG_CONFIG_PATH=[]): check.check_string_seq(PKG_CONFIG_PATH) cmd = [clazz._PKG_CONFIG_EXE] + object_util.listify(args) env = { 'PKG_CONFIG_DEBUG_SPEW': '1', 'PKG_CONFIG_LIBDIR': ':'.join(PKG_CONFIG_LIBDIR), 'PKG_CONFIG_PATH': ':'.join(PKG_CONFIG_PATH), # 'PATH': os_env_var('PATH').value, } for p in PKG_CONFIG_PATH: file_util.mkdir(p) #build_blurb.blurb_verbose('pkg_config', '_call_pkg_config() cmd=%s' % (str(cmd))) #print('pkg_config', '_call_pkg_config() cmd=%s; env=%s' % (str(cmd), str(env))) #print('pkg_config', '_call_pkg_config() cmd=%s' % (str(cmd))) rv = execute.execute(cmd, env=env) return rv
def search_string(clazz, content, patterns, word_boundary = False, ignore_case = False): assert string_util.is_string(content) patterns = object_util.listify(patterns) result = [] original_patterns = None if word_boundary: original_patterns = patterns[:] patterns = [ clazz._make_expresion(p, ignore_case) for p in patterns ] else: if ignore_case: original_patterns = patterns[:] patterns = [ p.lower() for p in patterns ] original_patterns = original_patterns or patterns patterns = list(zip(patterns, original_patterns)) for line_number, line in enumerate(content.split('\n'), 1): if word_boundary: result += clazz._search_line_with_re(line, patterns, '<unknown>', line_number) else: result += clazz._search_line_with_find(line, patterns, '<unknown>', line_number, ignore_case) return result
def find_python_compiled_files(clazz, dirs): dirs = object_util.listify(dirs) result = [] for d in dirs: result.extend(file_find.find_fnmatch(d, [ '*.pyc' ], relative = False)) return algorithm.unique(result)
def pkg_config_name(self): 'Return the name of the .pc file for pkg config this package uses.' return object_util.listify( self.properties.get(self.PROPERTY_PKG_CONFIG_NAME, [self.name]))
def from_files(clazz, filenames, root_dir = None, function_name = None): filenames = object_util.listify(filenames) result = clazz() for filename in filenames: result.append(file_checksum.from_file(filename, root_dir = root_dir, function_name = function_name)) return result
def make_paths_relative(clazz, paths): paths = object_util.listify(paths) return [ path.relpath(p) for p in paths ]
def make_paths_absolute(clazz, paths): paths = object_util.listify(paths) return [ clazz.ensure_abspath(p) for p in paths ]
def __init__(self, ignore_filenames): ignore_filenames = object_util.listify(ignore_filenames) self._ignorers = [ file_ignore(f) for f in ignore_filenames ]
def cflags(clazz, modules, PKG_CONFIG_PATH=[]): modules = object_util.listify(modules) args = ['--cflags'] + modules rv = clazz._call_pkg_config(args, PKG_CONFIG_PATH=PKG_CONFIG_PATH) return clazz._parse_flags(rv.stdout)
def _git_roots(clazz, files): files = object_util.listify(files) roots = [ git.root(f) for f in files ] roots = [ r for r in roots if r ] return algorithm.unique(roots)
def _normalize_hardness(clazz, hardness): return sorted(object_util.listify(hardness))
def include_path_add(self, path_or_paths): path_or_paths = object_util.listify(path_or_paths) self.env.AppendUnique(CPPPATH=path_or_paths)
def __report_new_targets(self, targets): targets = object_util.listify(targets) longest_new_target = max(len(str(target)) for target in targets) self._longest_target = max(self._longest_target, longest_new_target)
def requires(self, modules, resolved=False): modules = object_util.listify(modules) check.check_string_seq(modules)
def file_is_of_type(self, filename, types): 'Return True of file is of one of the given type or types.' types = object_util.listify(types) return self.file_type(filename) in types
def __init__(self, root_dir, criteria = None, relative = True): self.root_dir = path.normpath(root_dir) self.criteria = object_util.listify(criteria or [])
def publish(self, adescs, mutations={}): adescs = object_util.listify(adescs) for adesc in adescs: self._publish_one(adesc, mutations)
def package_flags(self, packages): flags = PackageFlags() for package in object_util.listify(packages): flags += self._packages[package] return flags
def test_listify(self): self.assertEqual( [ 6 ], object_util.listify(6) ) self.assertEqual( [ 6 ], object_util.listify([ 6 ]) ) self.assertEqual( [ 'foo' ], object_util.listify('foo') )