def require_modules(self, nodes, dependencies):
dependencies = tuple(
dep for dep in to_sequence(dependencies) if isinstance(dep, Node))
module_depends = self.build_manager.module_depends
for node in to_sequence(nodes):
module_depends(node, dependencies)
def depends(self, nodes, dependencies):
dependencies = tuple(to_sequence(dependencies))
depends = self.build_manager.depends
for node in to_sequence(nodes):
node.depends(dependencies)
depends(node, node.dep_nodes)
def require_modules(self, nodes, dependencies):
dependencies = tuple(dep for dep in to_sequence(dependencies)
if isinstance(dep, Node))
module_depends = self.build_manager.module_depends
for node in to_sequence(nodes):
module_depends(node, dependencies)
def add_values(self, values):
try:
values = tuple(values.items()) # convert dictionary to a sequence
except AttributeError:
pass
set_default_value = self.__values.setdefault
value_type = self.value_type
for value in to_sequence(values):
it = iter(to_sequence(value))
value = value_type(next(it))
value = set_default_value(value, value)
for alias in it:
alias = value_type(alias)
v = set_default_value(alias, value)
if v != value:
if alias == v:
raise ErrorOptionTypeEnumValueIsAlreadySet(
self, alias, value)
else:
raise ErrorOptionTypeEnumAliasIsAlreadySet(
self, alias, v, value)
def depends(self, nodes, dependencies):
dependencies = tuple(to_sequence(dependencies))
depends = self.build_manager.depends
for node in to_sequence(nodes):
node.depends(dependencies)
depends(node, node.dep_nodes)
def alias_nodes(self, alias, nodes, description=None):
for alias, node in itertools.product(to_sequence(alias),
to_sequence(nodes)):
self.aliases.setdefault(alias, set()).add(node)
if description:
self.alias_descriptions[alias] = description
def alias_nodes(self, alias, nodes, description=None):
for alias, node in itertools.product(to_sequence(alias),
to_sequence(nodes)):
self.aliases.setdefault(alias, set()).add(node)
if description:
self.alias_descriptions[alias] = description
def __init__(self, options, target, rename=None, basedir=None, ext=None):
if ext is None:
ext = ".zip"
self.target = self.get_target_path(target, ext=ext)
self.rename = tuple(rename for rename in to_sequence(rename))
sep = os.path.sep
self.basedir = tuple(os.path.normcase(os.path.normpath(basedir)) + sep
for basedir in to_sequence(basedir))
def exclude_files_from_dirs(files, dirs):
result = []
folders = tuple(os.path.normcase(
os.path.abspath(folder)) + os.path.sep for folder in to_sequence(dirs))
for filename in to_sequence(files):
filename = os.path.normcase(os.path.abspath(filename))
if not filename.startswith(folders):
result.append(filename)
return result
def exclude_files_from_dirs(files, dirs):
result = []
folders = tuple(
os.path.normcase(os.path.abspath(folder)) + os.path.sep
for folder in to_sequence(dirs))
for filename in to_sequence(files):
filename = os.path.normcase(os.path.abspath(filename))
if not filename.startswith(folders):
result.append(filename)
return result
def find_files(paths=".",
mask=("*", ),
exclude_mask=tuple(),
exclude_subdir_mask=('__*', '.*')):
found_files = []
paths = to_sequence(paths)
match_mask = _masks_to_match(mask)
match_exclude_mask = _masks_to_match(exclude_mask)
match_exclude_subdir_mask = _masks_to_match(exclude_subdir_mask)
for path in paths:
for root, folders, files in os.walk(os.path.abspath(path)):
for file_name in files:
file_name_nocase = os.path.normcase(file_name)
if (not match_exclude_mask(file_name_nocase)) and\
match_mask(file_name_nocase):
found_files.append(os.path.join(root, file_name))
folders[:] = (folder for folder in folders
if not match_exclude_subdir_mask(folder))
found_files.sort()
return found_files
def find_files(paths=".",
mask=("*",),
exclude_mask=('.*',),
exclude_subdir_mask=('__*', '.*'),
found_dirs=None):
found_files = []
paths = to_sequence(paths)
match_mask = _masks_to_match(mask)
match_exclude_mask = _masks_to_match(exclude_mask)
match_exclude_subdir_mask = _masks_to_match(exclude_subdir_mask)
path_join = os.path.join
for path in paths:
for root, folders, files in os.walk(os.path.abspath(path)):
for file_name in files:
file_name_nocase = os.path.normcase(file_name)
if (not match_exclude_mask(file_name_nocase)) and\
match_mask(file_name_nocase):
found_files.append(path_join(root, file_name))
folders[:] = filterfalse(match_exclude_subdir_mask, folders)
if found_dirs is not None:
found_dirs.update(path_join(root, folder)
for folder in folders)
found_files.sort()
return found_files
def _join_args(entities, brief):
args = [_get_trace_arg(arg, brief) for arg in to_sequence(entities)]
if not brief or (len(args) < 3):
return ' '.join(args)
wish_size = 128
args_str = [args.pop(0)]
last = args.pop()
size = len(args_str[0]) + len(last)
for arg in args:
size += len(arg)
if size > wish_size:
args_str.append('...')
break
args_str.append(arg)
args_str.append(last)
return ' '.join(args_str)
def make_entities(self, entities, tags=None):
make_entity = self.make_entity
for entity in to_sequence(entities):
if isinstance(entity, EntityBase):
yield entity
else:
yield make_entity(entity, tags)
def relative_join_list(base_path, paths):
base_path = _norm_local_path(base_path)
base_path_seq = _split_path(base_path)
return [
_relative_join(base_path, base_path_seq, path)
for path in to_sequence(paths)
]
def make_entities(self, entities, tags=None):
make_entity = self.make_entity
for entity in to_sequence(entities):
if isinstance(entity, EntityBase):
yield entity
else:
yield make_entity(entity, tags)
def _join_args(entities, brief):
args = [_get_trace_arg(arg, brief) for arg in to_sequence(entities)]
if not brief or (len(args) < 3):
return ' '.join(args)
wish_size = 128
args_str = [args.pop(0)]
last = args.pop()
size = len(args_str[0]) + len(last)
for arg in args:
size += len(arg)
if size > wish_size:
args_str.append('...')
break
args_str.append(arg)
args_str.append(last)
return ' '.join(args_str)
def get_entities(self):
entities = super(NodeIndexFilter, self).get_entities()
try:
return to_sequence(entities[self.index])
except IndexError:
return tuple()
def find_files(paths=".",
mask=("*", ),
exclude_mask=('.*', ),
exclude_subdir_mask=('__*', '.*'),
found_dirs=None):
found_files = []
paths = to_sequence(paths)
match_mask = _masks_to_match(mask)
match_exclude_mask = _masks_to_match(exclude_mask)
match_exclude_subdir_mask = _masks_to_match(exclude_subdir_mask)
path_join = os.path.join
for path in paths:
for root, folders, files in os.walk(os.path.abspath(path)):
for file_name in files:
file_name_nocase = os.path.normcase(file_name)
if (not match_exclude_mask(file_name_nocase)) and\
match_mask(file_name_nocase):
found_files.append(path_join(root, file_name))
folders[:] = filterfalse(match_exclude_subdir_mask, folders)
if found_dirs is not None:
found_dirs.update(
path_join(root, folder) for folder in folders)
found_files.sort()
return found_files
def remove_files(files):
for f in to_sequence(files):
try:
os.remove(f)
except OSError as ex:
if ex.errno != errno.ENOENT:
raise
def has_all(self, context, others):
value = self.get(context)
for other in to_sequence(others):
if other not in value:
return False
return True
def __init__(self, cmd, exception):
msg = ' '.join(to_sequence(cmd))
msg += '\n%s' % (exception,)
self.exception = exception
super(ExecCommandException, self).__init__(msg)
def remove_files(files):
for f in to_sequence(files):
try:
os.remove(f)
except OSError as ex:
if ex.errno != errno.ENOENT:
raise
def __init__(self, cmd, exception):
msg = ' '.join(to_sequence(cmd))
msg += '\n%s' % (exception, )
self.exception = exception
super(ExecCommandException, self).__init__(msg)
def help_range(self):
"""
Returns a description (list of strings) about range of allowed values
"""
if self.range_help:
return list(to_sequence(self.range_help))
return []
def add_tool(self, tool_class, names):
if not issubclass(tool_class, Tool):
raise ErrorToolInvalid(tool_class)
if names:
names = tuple(to_sequence(names))
self.tool_names.setdefault(tool_class, set()).update(names)
self.__add_to_map(self.tool_classes, names, tool_class)
def has_all(self, context, others):
value = self.get(context)
for other in to_sequence(others):
if other not in value:
return False
return True
def add_tool(self, tool_class, names):
if not issubclass(tool_class, Tool):
raise ErrorToolInvalid(tool_class)
if names:
names = tuple(to_sequence(names))
self.tool_names.setdefault(tool_class, set()).update(names)
self.__add_to_map(self.tool_classes, names, tool_class)
def get_shell_script_env(script, args=None, _var_re=re.compile(r'^\w+=')):
args = to_sequence(args)
script_path = os.path.abspath(
os.path.expanduser(os.path.expandvars(script)))
os_env = os.environ
cwd, script = os.path.split(script_path)
if os.name == "nt":
cmd = ['call', script]
cmd += args
cmd += ['&&', 'set']
else:
cmd = ['.', './' + script]
cmd += args
cmd += ['&&', 'printenv']
cmd = ' '.join(cmd)
try:
p = subprocess.Popen(cmd,
cwd=cwd,
shell=True,
env=os_env,
stdin=None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=False)
stdout, stderr = p.communicate()
status = p.poll()
except Exception as ex:
raise ExecCommandException(cmd, exception=ex)
stdout = _decode_data(stdout)
stderr = _decode_data(stderr)
if status != 0:
raise ExecCommandResult(cmd, status, stdout, stderr)
script_env = {}
for line in stdout.split('\n'):
match = _var_re.match(line)
if match:
name, sep, value = line.partition('=')
value = value.strip()
current = os_env.get(name, None)
if (current is None) or (value != current.strip()):
script_env[name] = value
return script_env
def __init__(self, cmd, status=None, stdout=None, stderr=None):
self.cmd = tuple(to_sequence(cmd))
self.status = status
self.stdout = stdout if stdout else ''
self.stderr = stderr if stderr else ''
super(ExecCommandResult, self).__init__()
def help_range(self):
if self.range_help:
return list(to_sequence(self.range_help))
if isinstance(self.item_type, OptionType):
return self.item_type.help_range()
return []
def __init__(self, cmd, status=None, stdout=None, stderr=None):
self.cmd = tuple(to_sequence(cmd))
self.status = status
self.stdout = stdout if stdout else ''
self.stderr = stderr if stderr else ''
super(ExecCommandResult, self).__init__()
def _split_nodes(items):
nodes = []
values = []
for item in to_sequence(items):
if isinstance(item, Node):
nodes.append(item)
else:
values.append(item)
return nodes, values
def remove_user_handler(self, user_handlers):
user_handlers = to_sequence(user_handlers)
for handlers in self.user_handlers.values():
for user_handler in user_handlers:
try:
handlers.remove(user_handler)
except ValueError:
pass
def help_range(self):
if self.range_help:
return list(to_sequence(self.range_help))
value_type = self.value_type.get_value_type()
if isinstance(value_type, OptionType):
return value_type.help_range()
return []
def remove_user_handler(self, user_handlers):
user_handlers = to_sequence(user_handlers)
for handlers in self.user_handlers.values():
for user_handler in user_handlers:
try:
handlers.remove(user_handler)
except ValueError:
pass
def _split_nodes(items):
nodes = []
values = []
for item in to_sequence(items):
if isinstance(item, Node):
nodes.append(item)
else:
values.append(item)
return nodes, values
def __init__(self, options, target=None, target_flag=None, cwd=None):
self.targets = tuple(map(self.get_target_path, to_sequence(target)))
self.target_flag = target_flag
if cwd:
cwd = self.get_target_dir(cwd)
self.cwd = cwd
def __init__(self, options, target=None, target_flag=None, cwd=None):
self.targets = tuple(map(self.get_target_path, to_sequence(target)))
self.target_flag = target_flag
if cwd:
cwd = self.get_target_dir(cwd)
self.cwd = cwd
def get_shell_script_env(script, args=None, _var_re=re.compile(r'^\w+=')):
args = to_sequence(args)
script_path = os.path.abspath(
os.path.expanduser(os.path.expandvars(script)))
os_env = os.environ
cwd, script = os.path.split(script_path)
if os.name == "nt":
cmd = ['call', script]
cmd += args
cmd += ['&&', 'set']
else:
cmd = ['.', './' + script]
cmd += args
cmd += ['&&', 'printenv']
cmd = ' '.join(cmd)
try:
p = subprocess.Popen(cmd, cwd=cwd, shell=True, env=os_env,
stdin=None, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, universal_newlines=False)
stdout, stderr = p.communicate()
status = p.poll()
except Exception as ex:
raise ExecCommandException(cmd, exception=ex)
stdout = _decode_data(stdout)
stderr = _decode_data(stderr)
if status != 0:
raise ExecCommandResult(cmd, status, stdout, stderr)
script_env = {}
for line in stdout.split('\n'):
match = _var_re.match(line)
if match:
name, sep, value = line.partition('=')
value = value.strip()
current = os_env.get(name, None)
if (current is None) or (value != current.strip()):
script_env[name] = value
return script_env
def build_split(self, vfile, explain):
if self.split_called:
return None
self.split_called = True
self.check_actual = self._split_actual
builder = self.builder
dep_entities = self.dep_entities
if builder.is_batch():
return self._split_batch(vfile, explain)
# -----------------------------------------------------------
sources = self.source_entities
groups = self.builder.split(sources)
# No source groups, just build the sources
if (not groups) or (len(groups) < 2):
node_entity = NodeEntity(builder=builder,
source_entities=sources,
dep_entities=dep_entities)
if not node_entity.check_actual(vfile, explain):
self.check_actual = self._not_actual
self.node_entities = (node_entity,)
return None
# -----------------------------------------------------------
# create split Nodes
node_entities = []
split_nodes = []
for group in groups:
group = to_sequence(group)
node_entity = NodeEntity(builder=builder,
source_entities=group,
dep_entities=dep_entities)
if not node_entity.check_actual(vfile, explain):
node = self._split(group, (node_entity,))
split_nodes.append(node)
node_entities.append(node_entity)
self.node_entities = node_entities
return split_nodes
def list_tools_options(self, tools, brief=False):
tools = set(to_sequence(tools))
result = []
for tools_options, names in self.tools._get_tools_options().items():
names_set = tools & set(names)
if names_set:
tools -= names_set
options_name = "Options of tool: %s" % (', '.join(names))
result += tools_options.help_text(options_name, brief=brief)
if result and result[-1]:
result.append("")
return result
def __get_events(self, event_filters):
events = set()
for event_filter in to_sequence(event_filters):
if event_filter not in EVENT_ALL:
events.add(event_filter)
else:
for event, pair in self.default_handlers.items():
level = pair[1]
if event_filter == level:
events.add(event)
return events
def __get_events(self, event_filters):
events = set()
for event_filter in to_sequence(event_filters):
if event_filter not in EVENT_ALL:
events.add(event_filter)
else:
for event, pair in self.default_handlers.items():
level = pair[1]
if event_filter == level:
events.add(event)
return events
def list_tools_options(self, tools, brief=False):
tools = set(to_sequence(tools))
result = []
for tools_options, names in self.tools._get_tools_options().items():
names_set = tools & set(names)
if names_set:
tools -= names_set
options_name = "Options of tool: %s" % (', '.join(names))
result += tools_options.help_text(options_name, brief=brief)
if result and result[-1]:
result.append("")
return result
def __init__(self,
description=None,
group=None,
style=None,
true_values=None,
false_values=None,
default=False,
is_tool_key=False,
is_hidden=False
):
# noinspection PyTypeChecker
super(BoolOptionType, self).__init__(bool, description, group,
default=default,
is_tool_key=is_tool_key,
is_hidden=is_hidden)
if style is None:
style = ('true', 'false')
else:
style = map(IgnoreCaseString, style)
if true_values is None:
true_values = self.__true_values
else:
true_values = to_sequence(true_values)
if false_values is None:
false_values = self.__false_values
else:
false_values = to_sequence(false_values)
self.true_value, self.false_value = style
self.true_values = set()
self.false_values = set()
self.add_values(true_values, false_values)
self.add_values(self.true_value, self.false_value)
def __init__(self, cmd, status=None, stdout=None, stderr=None):
self.cmd = tuple(to_sequence(cmd))
self.status = status
if not stdout:
stdout = str()
if stderr:
stdout += '\n' + stderr
self.output = stdout
super(ExecCommandResult, self).__init__()
def _masks_to_match(masks, _null_match=lambda name: False):
if not masks:
return _null_match
if is_string(masks):
masks = masks.split('|')
re_list = []
for mask in to_sequence(masks):
mask = os.path.normcase(mask).strip()
re_list.append("(%s)" % fnmatch.translate(mask))
re_str = '|'.join(re_list)
return re.compile(re_str).match
def __new__(cls, name, signature, tags=None):
self = super(EntityBase, cls).__new__(cls)
if name is not NotImplemented:
if not name:
raise ErrorEntityNameEmpty()
self.name = name
if signature is not NotImplemented:
self.signature = signature
self.tags = frozenset(to_sequence(tags))
return self
def build_replace(self, chdir=os.chdir):
if self.replace_called:
return None
self.replace_called = True
chdir(self.cwd)
sources = self.builder.replace(self.options, self.source_entities)
if sources is None:
return None
# source_entities will be reinitialized later
self.sources = tuple(to_sequence(sources))
return self.get_source_nodes()
def _add_program_exts(progs, exts):
progs = to_sequence(progs)
if not exts:
return tuple(progs)
result = []
for prog in progs:
prog_ext = os.path.splitext(prog)[1]
if prog_ext or (prog_ext in exts):
result.append(prog)
else:
result += (prog + ext for ext in exts)
return result
def load_tools(self, paths, reload=False):
for path in to_sequence(paths):
path = expand_file_path(path)
if path in self.loaded_paths:
if not reload:
continue
else:
self.loaded_paths.append(path)
module_files = find_files(path, mask="*.py")
if not module_files:
continue
self._load_tools_package(path, module_files)
def build_if(self, condition, nodes):
if not isinstance(condition, _NodeCondition):
condition = _NodeCondition(condition)
cond_node = condition.get_node()
if cond_node is not None:
cond_node = (cond_node, )
depends = self._nodes.depends
set_node_condition = self._node_conditions.__setitem__
for node in to_sequence(nodes):
if cond_node is not None:
depends(node, cond_node)
set_node_condition(node, condition)
def depends(self, dependencies):
dep_nodes = self.dep_nodes
dep_entities = self.dep_entities
for entity in to_sequence(dependencies):
if isinstance(entity, Node):
dep_nodes.add(entity)
elif isinstance(entity, NodeFilter):
dep_nodes.add(entity.get_node())
elif isinstance(entity, EntityBase):
dep_entities.append(entity)
else:
raise ErrorNodeDependencyInvalid(entity)
def __init__(self, builder, sources, cwd=None):
self.builder = builder
self.options = getattr(builder, 'options', None)
if cwd is None:
self.cwd = os.path.abspath(os.getcwd())
else:
self.cwd = cwd
self.initiated = False
self.depends_called = False
self.replace_called = False
self.split_called = False
self.check_actual = self._not_actual
self.sources = tuple(to_sequence(sources))
self.dep_nodes = set()
self.dep_entities = []
