def _match(clazz, filenames, patterns, match_func, match_type):
'''
Match a list of files with patterns using match_func and match_type.
match_func should be the form match_func(filename, patterns)
'''
assert clazz.match_type_is_valid(match_type)
filenames = object_util.listify(filenames)
patterns = object_util.listify(patterns) # or [])
result = []
if not patterns:
if match_type == clazz.ANY:
return []
elif match_type == clazz.NONE:
return filenames
elif match_type == clazz.ALL:
return []
func_map = {
clazz.ANY: clazz._match_any,
clazz.NONE: clazz._match_none,
clazz.ALL: clazz._match_all,
}
func = func_map[match_type]
result = []
for filename in filenames:
if func(match_func, path.basename(filename), patterns):
result.append(filename)
return sorted(algorithm.unique(result))
def make_static_lib(self, lib, objects, arflags = None): assert objects objects = object_util.listify(objects) cmd = '$AR $AR_FLAGS %s %s' % (lib, ' '.join(objects)) cmd = variable.substitute(cmd, self.variables) self._execute_cmd(cmd) return lib
def _requires_no_resolve(self, modules):
modules = object_util.listify(modules)
check.check_string_seq(modules)
reqs = []
for name in modules:
reqs.extend(self._get_pc_file(module_name).requires)
return algorithm.unique(reqs)
def _make_test_amt(self, recipes, publish):
amt = AMT(recipes=recipes)
publish = object_util.listify(publish)
check.check_string_seq(publish)
for p in publish:
amt.publish(p)
return amt
def remove_checksums(self, packages, level):
packages = object_util.listify(packages)
check.check_package_descriptor_seq(packages)
checksum_dirs = [self._checksum_dir(pd, level) for pd in packages]
for d in checksum_dirs:
self.blurb('removing checksums: %s' % (path.relpath(d)))
file_util.remove(checksum_dirs)
def resolve_deps(self, names, system, hardness, include_names):
'Resolve dependencies.'
self.log_i(
'resolve_deps(names=%s, system=%s, hardness=%s, include_names=%s)'
% (' '.join(names), system, ' '.join(hardness), include_names))
check.check_string_seq(names)
names = object_util.listify(names)
hardness = self._normalize_hardness(hardness)
reqs = requirement_list()
for name in names:
desc = self._descriptor_map.get(name, None)
if not desc:
raise KeyError('Not found in packages: %s' % (name))
reqs.extend(desc.requirements.filter_by(hardness, system))
self.log_i('resolve_deps() reqs=%s' % (str(reqs.names())))
dep_map = self.dependency_map(hardness, system)
result = package_descriptor_list(
dependency_resolver.resolve_and_order_deps(reqs.names(),
self._descriptor_map,
dep_map))
self.log_i('resolve_deps() 1 result=%s' % (str(result.names())))
if include_names:
result += self.descriptors(names)
result.remove_dups()
self.log_i('resolve_deps() 2 result=%s' % (str(result.names())))
return result
def compile_c(self, sources, objects = None, cflags = None):
assert sources
cflags = cflags or []
sources = object_util.listify(sources)
if objects:
objects = object_util.listify(objects)
targets = self._make_targets(sources, objects)
for src, obj in targets:
cmd = '$CC $CFLAGS %s -c $(SRC) -o $(OBJ)' % (' '.join(cflags))
variables = copy.deepcopy(self.variables)
variables['SRC'] = src
variables['OBJ'] = obj
cmd = variable.substitute(cmd, variables)
print(cmd)
self._execute_cmd(cmd)
return targets
def link_exe(self, exe, objects, ldflags = None):
assert objects
ldflags = ldflags or []
objects = object_util.listify(objects)
cmd = '$CC %s -o %s $LDFLAGS %s' % (' '.join(objects), exe, ' '.join(ldflags))
cmd = variable.substitute(cmd, self.variables)
self._execute_cmd(cmd)
return exe
def glob(clazz, paths, glob_expression):
'Like glob but handles a single path or a sequence of paths'
paths = object_util.listify(paths)
paths = [ path.join(p, glob_expression) for p in paths ]
result = []
for p in paths:
result.extend(glob.glob(p))
return sorted(algorithm.unique(result))
def make_shared_lib(self, lib, objects, ldflags = None):
assert objects
ldflags = ldflags or []
objects = object_util.listify(objects)
cmd = '$CC -shared $LDFLAGS %s %s -o %s' % (' '.join(ldflags), ' '.join(objects), lib)
cmd = variable.substitute(cmd, self.variables)
self._execute_cmd(cmd)
return lib
def call_shell(clazz, command, script, env, shell_env = None, save_logs = None, execution_dir = None):
command = execute.listify_command(command)
command = [ part for part in command if part ]
shell_env = shell_env or key_value_list()
save_logs = save_logs or []
check.check_key_value_list(shell_env)
log.log_i('rebuild', 'call_shell(command=%s)' % (command))
build_blurb.blurb_verbose('rebuild', 'call_shell(cmd=%s, shell_env=%s)' % (command, shell_env))
env = clazz.create_command_env(script)
env.update(shell_env)
#clazz.env_dump(env, script.descriptor.name, 'PRE ENVIRONMENT')
clazz._env_substitite(env)
rogue_key = clazz._env_find_roque_dollar_sign(env)
if rogue_key:
raise RuntimeError('Rogue dollar sign (\"$\") found in %s: %s' % (rogue_key, env[rogue_key]))
command = [ variable.substitute(part, env) for part in object_util.listify(command) ]
# Check that no rogue dollar signs are in the command
for part in command:
if variable.has_rogue_dollar_signs(part):
raise RuntimeError('Rogue dollar sign (\"$\") found in: %s' % (part))
build_blurb.blurb('rebuild', '%s - %s' % (script.descriptor.name, ' '.join(command)))
# file_util.mkdir(script.build_dir)
retry_script = clazz._write_retry_script(command, env, script)
#clazz.env_dump(env, script.descriptor.name, clazz.__name__ + ' : ' + 'POST ENVIRONMENT')
for k,v in env.items():
if string_util.is_quoted(v):
env[k] = string_util.unquote(v)
if execution_dir:
cwd = path.join(script.build_dir, execution_dir)
else:
cwd = script.build_dir
rv = execute.execute(command,
cwd = cwd,
env = env,
shell = True,
non_blocking = build_blurb.verbose,
stderr_to_stdout = build_blurb.verbose,
raise_error = False)
message = rv.stdout
if rv.stderr:
message = message + '\n' + rv.stderr
result = step_result(rv.exit_code == 0, message)
clazz.save_build_dir_logs(script, save_logs)
return result
def match_all(self, expressions, strip_comments = False):
expressions = object_util.listify(expressions)
result = []
for line in self._lines:
text = line.get_text(strip_comments = strip_comments)
for expression in expressions:
if re.match(expression, text):
result.append(line)
return text_line_parser(result)
def libs(clazz, modules, PKG_CONFIG_PATH=[], static=False):
modules = object_util.listify(modules)
args = ['--libs'] + modules
if static:
args.append('--static')
rv = clazz._call_pkg_config(args, PKG_CONFIG_PATH=PKG_CONFIG_PATH)
libs = clazz._parse_flags(rv.stdout)
libs = clazz._libs_cleanup(libs)
return libs
def _resolve_requires(self, modules):
modules = object_util.listify(modules)
check.check_string_seq(modules)
immediate_reqs = self.module_requires(module_name)
immediate_req_names = [req.name for req in reqs]
order = dependency_resolver.resolve_deps(self.dep_map, req_names)
for mod in reqs:
cflags.extend(self._single_module_cflags(mod.name))
cflags.extend(self._single_module_cflags(module_name))
return cflags
def find(clazz, dirs, name, version):
dirs = object_util.listify(dirs)
filenames = []
for d in dirs:
if path.isdir(d):
filenames += file_find.find(d,
max_depth=4,
relative=False,
file_type=file_find.FILE
| file_find.LINK)
return clazz.find_in_list(filenames, name, version)
def hardness_matches(self, hardness):
'Return True if hardness matches.'
hardness = object_util.listify(hardness)
if not requirement_hardness.is_valid_seq(hardness):
raise ValueError('invalid hardness: %s - %s' %
(str(hardness), type(hardness)))
self_hardness = self.hardness or requirement_hardness.DEFAULT
for h in hardness:
if self_hardness == requirement_hardness(h):
return True
return False
def remove(clazz, files):
files = object_util.listify(files)
for f in files:
try:
if path.isdir(f):
shutil.rmtree(f)
else:
os.remove(f)
except Exception as ex:
clazz.log_d('Caught exception %s removing %s' % (ex, f))
pass
def match_first(self, expressions, strip_comments = False, line_number = None):
expressions = object_util.listify(expressions)
for line in self._lines:
if line_number is not None:
if line.line_number < line_number:
continue
text = line.get_text(strip_comments = strip_comments)
for expression in expressions:
match = re.findall(expression, text)
if match:
return self._match_result(expression, match, line)
return None
def match_backwards(self, line_number, expressions, strip_comments = False):
expressions = object_util.listify(expressions)
start_index = self.find_by_line_number(line_number) - 1
if start_index < 0:
return None
for i in range(start_index, -1, -1):
line = self._lines[i]
text = line.get_text(strip_comments = strip_comments)
for expression in expressions:
match = re.findall(expression, text)
if match:
return self._match_result(expression, match, line)
return None
def patch(clazz,
patches,
cwd,
strip=1,
backup=True,
posix=True,
program=None):
'Apply the given patches by calling patch with strip, backup and posix.'
patches = object_util.listify(patches)
target_files = file_find.find(cwd, relative=True)
for patch in patches:
rv = clazz._call_patch(patch, cwd, strip, backup, posix, program)
if rv[0] != 0:
sys.stderr.write(rv[1])
sys.stderr.flush()
return rv
return (0, None)
def resolve_deps(clazz, dep_map, names):
'''
Return a set of resolved dependencies for the given name or names.
Sorted alphabetically, not in build order.
'''
cyclic_deps = clazz.cyclic_deps(dep_map)
if len(cyclic_deps) > 0:
raise cyclic_dependency_error('Cyclic dependencies found: %s' % (' '.join(cyclic_deps)), cyclic_deps)
order = clazz.build_order_flat(dep_map)
names = object_util.listify(names)
result = set(names)
for name in names:
result |= clazz._resolve_deps(dep_map, name)
return sorted(list(result))
def _call_pkg_config(clazz,
args,
PKG_CONFIG_LIBDIR=[],
PKG_CONFIG_PATH=[]):
check.check_string_seq(PKG_CONFIG_PATH)
cmd = [clazz._PKG_CONFIG_EXE] + object_util.listify(args)
env = {
'PKG_CONFIG_DEBUG_SPEW': '1',
'PKG_CONFIG_LIBDIR': ':'.join(PKG_CONFIG_LIBDIR),
'PKG_CONFIG_PATH': ':'.join(PKG_CONFIG_PATH),
# 'PATH': os_env_var('PATH').value,
}
for p in PKG_CONFIG_PATH:
file_util.mkdir(p)
#build_blurb.blurb_verbose('pkg_config', '_call_pkg_config() cmd=%s' % (str(cmd)))
#print('pkg_config', '_call_pkg_config() cmd=%s; env=%s' % (str(cmd), str(env)))
#print('pkg_config', '_call_pkg_config() cmd=%s' % (str(cmd)))
rv = execute.execute(cmd, env=env)
return rv
def search_string(clazz, content, patterns, word_boundary = False, ignore_case = False):
assert string_util.is_string(content)
patterns = object_util.listify(patterns)
result = []
original_patterns = None
if word_boundary:
original_patterns = patterns[:]
patterns = [ clazz._make_expresion(p, ignore_case) for p in patterns ]
else:
if ignore_case:
original_patterns = patterns[:]
patterns = [ p.lower() for p in patterns ]
original_patterns = original_patterns or patterns
patterns = list(zip(patterns, original_patterns))
for line_number, line in enumerate(content.split('\n'), 1):
if word_boundary:
result += clazz._search_line_with_re(line, patterns, '<unknown>', line_number)
else:
result += clazz._search_line_with_find(line, patterns, '<unknown>', line_number, ignore_case)
return result
def find_python_compiled_files(clazz, dirs):
dirs = object_util.listify(dirs)
result = []
for d in dirs:
result.extend(file_find.find_fnmatch(d, [ '*.pyc' ], relative = False))
return algorithm.unique(result)
def pkg_config_name(self):
'Return the name of the .pc file for pkg config this package uses.'
return object_util.listify(
self.properties.get(self.PROPERTY_PKG_CONFIG_NAME, [self.name]))
def from_files(clazz, filenames, root_dir = None, function_name = None):
filenames = object_util.listify(filenames)
result = clazz()
for filename in filenames:
result.append(file_checksum.from_file(filename, root_dir = root_dir, function_name = function_name))
return result
def make_paths_relative(clazz, paths): paths = object_util.listify(paths) return [ path.relpath(p) for p in paths ]
def make_paths_absolute(clazz, paths): paths = object_util.listify(paths) return [ clazz.ensure_abspath(p) for p in paths ]
def __init__(self, ignore_filenames): ignore_filenames = object_util.listify(ignore_filenames) self._ignorers = [ file_ignore(f) for f in ignore_filenames ]
def cflags(clazz, modules, PKG_CONFIG_PATH=[]):
modules = object_util.listify(modules)
args = ['--cflags'] + modules
rv = clazz._call_pkg_config(args, PKG_CONFIG_PATH=PKG_CONFIG_PATH)
return clazz._parse_flags(rv.stdout)
def _git_roots(clazz, files): files = object_util.listify(files) roots = [ git.root(f) for f in files ] roots = [ r for r in roots if r ] return algorithm.unique(roots)
def _normalize_hardness(clazz, hardness):
return sorted(object_util.listify(hardness))
def include_path_add(self, path_or_paths):
path_or_paths = object_util.listify(path_or_paths)
self.env.AppendUnique(CPPPATH=path_or_paths)
def __report_new_targets(self, targets):
targets = object_util.listify(targets)
longest_new_target = max(len(str(target)) for target in targets)
self._longest_target = max(self._longest_target, longest_new_target)
def requires(self, modules, resolved=False):
modules = object_util.listify(modules)
check.check_string_seq(modules)
def file_is_of_type(self, filename, types): 'Return True of file is of one of the given type or types.' types = object_util.listify(types) return self.file_type(filename) in types
def __init__(self, root_dir, criteria = None, relative = True): self.root_dir = path.normpath(root_dir) self.criteria = object_util.listify(criteria or [])
def publish(self, adescs, mutations={}):
adescs = object_util.listify(adescs)
for adesc in adescs:
self._publish_one(adesc, mutations)
def package_flags(self, packages):
flags = PackageFlags()
for package in object_util.listify(packages):
flags += self._packages[package]
return flags
def test_listify(self):
self.assertEqual( [ 6 ], object_util.listify(6) )
self.assertEqual( [ 6 ], object_util.listify([ 6 ]) )
self.assertEqual( [ 'foo' ], object_util.listify('foo') )
