def require_modules(self, nodes, dependencies): dependencies = tuple( dep for dep in to_sequence(dependencies) if isinstance(dep, Node)) module_depends = self.build_manager.module_depends for node in to_sequence(nodes): module_depends(node, dependencies)
def depends(self, nodes, dependencies): dependencies = tuple(to_sequence(dependencies)) depends = self.build_manager.depends for node in to_sequence(nodes): node.depends(dependencies) depends(node, node.dep_nodes)
def require_modules(self, nodes, dependencies): dependencies = tuple(dep for dep in to_sequence(dependencies) if isinstance(dep, Node)) module_depends = self.build_manager.module_depends for node in to_sequence(nodes): module_depends(node, dependencies)
def add_values(self, values): try: values = tuple(values.items()) # convert dictionary to a sequence except AttributeError: pass set_default_value = self.__values.setdefault value_type = self.value_type for value in to_sequence(values): it = iter(to_sequence(value)) value = value_type(next(it)) value = set_default_value(value, value) for alias in it: alias = value_type(alias) v = set_default_value(alias, value) if v != value: if alias == v: raise ErrorOptionTypeEnumValueIsAlreadySet( self, alias, value) else: raise ErrorOptionTypeEnumAliasIsAlreadySet( self, alias, v, value)
def depends(self, nodes, dependencies): dependencies = tuple(to_sequence(dependencies)) depends = self.build_manager.depends for node in to_sequence(nodes): node.depends(dependencies) depends(node, node.dep_nodes)
def alias_nodes(self, alias, nodes, description=None): for alias, node in itertools.product(to_sequence(alias), to_sequence(nodes)): self.aliases.setdefault(alias, set()).add(node) if description: self.alias_descriptions[alias] = description
def alias_nodes(self, alias, nodes, description=None): for alias, node in itertools.product(to_sequence(alias), to_sequence(nodes)): self.aliases.setdefault(alias, set()).add(node) if description: self.alias_descriptions[alias] = description
def __init__(self, options, target, rename=None, basedir=None, ext=None): if ext is None: ext = ".zip" self.target = self.get_target_path(target, ext=ext) self.rename = tuple(rename for rename in to_sequence(rename)) sep = os.path.sep self.basedir = tuple(os.path.normcase(os.path.normpath(basedir)) + sep for basedir in to_sequence(basedir))
def exclude_files_from_dirs(files, dirs): result = [] folders = tuple(os.path.normcase( os.path.abspath(folder)) + os.path.sep for folder in to_sequence(dirs)) for filename in to_sequence(files): filename = os.path.normcase(os.path.abspath(filename)) if not filename.startswith(folders): result.append(filename) return result
def exclude_files_from_dirs(files, dirs): result = [] folders = tuple( os.path.normcase(os.path.abspath(folder)) + os.path.sep for folder in to_sequence(dirs)) for filename in to_sequence(files): filename = os.path.normcase(os.path.abspath(filename)) if not filename.startswith(folders): result.append(filename) return result
def find_files(paths=".", mask=("*", ), exclude_mask=tuple(), exclude_subdir_mask=('__*', '.*')): found_files = [] paths = to_sequence(paths) match_mask = _masks_to_match(mask) match_exclude_mask = _masks_to_match(exclude_mask) match_exclude_subdir_mask = _masks_to_match(exclude_subdir_mask) for path in paths: for root, folders, files in os.walk(os.path.abspath(path)): for file_name in files: file_name_nocase = os.path.normcase(file_name) if (not match_exclude_mask(file_name_nocase)) and\ match_mask(file_name_nocase): found_files.append(os.path.join(root, file_name)) folders[:] = (folder for folder in folders if not match_exclude_subdir_mask(folder)) found_files.sort() return found_files
def find_files(paths=".", mask=("*",), exclude_mask=('.*',), exclude_subdir_mask=('__*', '.*'), found_dirs=None): found_files = [] paths = to_sequence(paths) match_mask = _masks_to_match(mask) match_exclude_mask = _masks_to_match(exclude_mask) match_exclude_subdir_mask = _masks_to_match(exclude_subdir_mask) path_join = os.path.join for path in paths: for root, folders, files in os.walk(os.path.abspath(path)): for file_name in files: file_name_nocase = os.path.normcase(file_name) if (not match_exclude_mask(file_name_nocase)) and\ match_mask(file_name_nocase): found_files.append(path_join(root, file_name)) folders[:] = filterfalse(match_exclude_subdir_mask, folders) if found_dirs is not None: found_dirs.update(path_join(root, folder) for folder in folders) found_files.sort() return found_files
def _join_args(entities, brief): args = [_get_trace_arg(arg, brief) for arg in to_sequence(entities)] if not brief or (len(args) < 3): return ' '.join(args) wish_size = 128 args_str = [args.pop(0)] last = args.pop() size = len(args_str[0]) + len(last) for arg in args: size += len(arg) if size > wish_size: args_str.append('...') break args_str.append(arg) args_str.append(last) return ' '.join(args_str)
def make_entities(self, entities, tags=None): make_entity = self.make_entity for entity in to_sequence(entities): if isinstance(entity, EntityBase): yield entity else: yield make_entity(entity, tags)
def relative_join_list(base_path, paths): base_path = _norm_local_path(base_path) base_path_seq = _split_path(base_path) return [ _relative_join(base_path, base_path_seq, path) for path in to_sequence(paths) ]
def make_entities(self, entities, tags=None): make_entity = self.make_entity for entity in to_sequence(entities): if isinstance(entity, EntityBase): yield entity else: yield make_entity(entity, tags)
def _join_args(entities, brief): args = [_get_trace_arg(arg, brief) for arg in to_sequence(entities)] if not brief or (len(args) < 3): return ' '.join(args) wish_size = 128 args_str = [args.pop(0)] last = args.pop() size = len(args_str[0]) + len(last) for arg in args: size += len(arg) if size > wish_size: args_str.append('...') break args_str.append(arg) args_str.append(last) return ' '.join(args_str)
def get_entities(self): entities = super(NodeIndexFilter, self).get_entities() try: return to_sequence(entities[self.index]) except IndexError: return tuple()
def find_files(paths=".", mask=("*", ), exclude_mask=('.*', ), exclude_subdir_mask=('__*', '.*'), found_dirs=None): found_files = [] paths = to_sequence(paths) match_mask = _masks_to_match(mask) match_exclude_mask = _masks_to_match(exclude_mask) match_exclude_subdir_mask = _masks_to_match(exclude_subdir_mask) path_join = os.path.join for path in paths: for root, folders, files in os.walk(os.path.abspath(path)): for file_name in files: file_name_nocase = os.path.normcase(file_name) if (not match_exclude_mask(file_name_nocase)) and\ match_mask(file_name_nocase): found_files.append(path_join(root, file_name)) folders[:] = filterfalse(match_exclude_subdir_mask, folders) if found_dirs is not None: found_dirs.update( path_join(root, folder) for folder in folders) found_files.sort() return found_files
def remove_files(files): for f in to_sequence(files): try: os.remove(f) except OSError as ex: if ex.errno != errno.ENOENT: raise
def has_all(self, context, others): value = self.get(context) for other in to_sequence(others): if other not in value: return False return True
def __init__(self, cmd, exception): msg = ' '.join(to_sequence(cmd)) msg += '\n%s' % (exception,) self.exception = exception super(ExecCommandException, self).__init__(msg)
def remove_files(files): for f in to_sequence(files): try: os.remove(f) except OSError as ex: if ex.errno != errno.ENOENT: raise
def __init__(self, cmd, exception): msg = ' '.join(to_sequence(cmd)) msg += '\n%s' % (exception, ) self.exception = exception super(ExecCommandException, self).__init__(msg)
def help_range(self): """ Returns a description (list of strings) about range of allowed values """ if self.range_help: return list(to_sequence(self.range_help)) return []
def add_tool(self, tool_class, names): if not issubclass(tool_class, Tool): raise ErrorToolInvalid(tool_class) if names: names = tuple(to_sequence(names)) self.tool_names.setdefault(tool_class, set()).update(names) self.__add_to_map(self.tool_classes, names, tool_class)
def has_all(self, context, others): value = self.get(context) for other in to_sequence(others): if other not in value: return False return True
def add_tool(self, tool_class, names): if not issubclass(tool_class, Tool): raise ErrorToolInvalid(tool_class) if names: names = tuple(to_sequence(names)) self.tool_names.setdefault(tool_class, set()).update(names) self.__add_to_map(self.tool_classes, names, tool_class)
def get_shell_script_env(script, args=None, _var_re=re.compile(r'^\w+=')): args = to_sequence(args) script_path = os.path.abspath( os.path.expanduser(os.path.expandvars(script))) os_env = os.environ cwd, script = os.path.split(script_path) if os.name == "nt": cmd = ['call', script] cmd += args cmd += ['&&', 'set'] else: cmd = ['.', './' + script] cmd += args cmd += ['&&', 'printenv'] cmd = ' '.join(cmd) try: p = subprocess.Popen(cmd, cwd=cwd, shell=True, env=os_env, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=False) stdout, stderr = p.communicate() status = p.poll() except Exception as ex: raise ExecCommandException(cmd, exception=ex) stdout = _decode_data(stdout) stderr = _decode_data(stderr) if status != 0: raise ExecCommandResult(cmd, status, stdout, stderr) script_env = {} for line in stdout.split('\n'): match = _var_re.match(line) if match: name, sep, value = line.partition('=') value = value.strip() current = os_env.get(name, None) if (current is None) or (value != current.strip()): script_env[name] = value return script_env
def __init__(self, cmd, status=None, stdout=None, stderr=None): self.cmd = tuple(to_sequence(cmd)) self.status = status self.stdout = stdout if stdout else '' self.stderr = stderr if stderr else '' super(ExecCommandResult, self).__init__()
def help_range(self): if self.range_help: return list(to_sequence(self.range_help)) if isinstance(self.item_type, OptionType): return self.item_type.help_range() return []
def __init__(self, cmd, status=None, stdout=None, stderr=None): self.cmd = tuple(to_sequence(cmd)) self.status = status self.stdout = stdout if stdout else '' self.stderr = stderr if stderr else '' super(ExecCommandResult, self).__init__()
def _split_nodes(items): nodes = [] values = [] for item in to_sequence(items): if isinstance(item, Node): nodes.append(item) else: values.append(item) return nodes, values
def remove_user_handler(self, user_handlers): user_handlers = to_sequence(user_handlers) for handlers in self.user_handlers.values(): for user_handler in user_handlers: try: handlers.remove(user_handler) except ValueError: pass
def help_range(self): if self.range_help: return list(to_sequence(self.range_help)) value_type = self.value_type.get_value_type() if isinstance(value_type, OptionType): return value_type.help_range() return []
def remove_user_handler(self, user_handlers): user_handlers = to_sequence(user_handlers) for handlers in self.user_handlers.values(): for user_handler in user_handlers: try: handlers.remove(user_handler) except ValueError: pass
def _split_nodes(items): nodes = [] values = [] for item in to_sequence(items): if isinstance(item, Node): nodes.append(item) else: values.append(item) return nodes, values
def __init__(self, options, target=None, target_flag=None, cwd=None): self.targets = tuple(map(self.get_target_path, to_sequence(target))) self.target_flag = target_flag if cwd: cwd = self.get_target_dir(cwd) self.cwd = cwd
def __init__(self, options, target=None, target_flag=None, cwd=None): self.targets = tuple(map(self.get_target_path, to_sequence(target))) self.target_flag = target_flag if cwd: cwd = self.get_target_dir(cwd) self.cwd = cwd
def get_shell_script_env(script, args=None, _var_re=re.compile(r'^\w+=')): args = to_sequence(args) script_path = os.path.abspath( os.path.expanduser(os.path.expandvars(script))) os_env = os.environ cwd, script = os.path.split(script_path) if os.name == "nt": cmd = ['call', script] cmd += args cmd += ['&&', 'set'] else: cmd = ['.', './' + script] cmd += args cmd += ['&&', 'printenv'] cmd = ' '.join(cmd) try: p = subprocess.Popen(cmd, cwd=cwd, shell=True, env=os_env, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=False) stdout, stderr = p.communicate() status = p.poll() except Exception as ex: raise ExecCommandException(cmd, exception=ex) stdout = _decode_data(stdout) stderr = _decode_data(stderr) if status != 0: raise ExecCommandResult(cmd, status, stdout, stderr) script_env = {} for line in stdout.split('\n'): match = _var_re.match(line) if match: name, sep, value = line.partition('=') value = value.strip() current = os_env.get(name, None) if (current is None) or (value != current.strip()): script_env[name] = value return script_env
def build_split(self, vfile, explain): if self.split_called: return None self.split_called = True self.check_actual = self._split_actual builder = self.builder dep_entities = self.dep_entities if builder.is_batch(): return self._split_batch(vfile, explain) # ----------------------------------------------------------- sources = self.source_entities groups = self.builder.split(sources) # No source groups, just build the sources if (not groups) or (len(groups) < 2): node_entity = NodeEntity(builder=builder, source_entities=sources, dep_entities=dep_entities) if not node_entity.check_actual(vfile, explain): self.check_actual = self._not_actual self.node_entities = (node_entity,) return None # ----------------------------------------------------------- # create split Nodes node_entities = [] split_nodes = [] for group in groups: group = to_sequence(group) node_entity = NodeEntity(builder=builder, source_entities=group, dep_entities=dep_entities) if not node_entity.check_actual(vfile, explain): node = self._split(group, (node_entity,)) split_nodes.append(node) node_entities.append(node_entity) self.node_entities = node_entities return split_nodes
def list_tools_options(self, tools, brief=False): tools = set(to_sequence(tools)) result = [] for tools_options, names in self.tools._get_tools_options().items(): names_set = tools & set(names) if names_set: tools -= names_set options_name = "Options of tool: %s" % (', '.join(names)) result += tools_options.help_text(options_name, brief=brief) if result and result[-1]: result.append("") return result
def __get_events(self, event_filters): events = set() for event_filter in to_sequence(event_filters): if event_filter not in EVENT_ALL: events.add(event_filter) else: for event, pair in self.default_handlers.items(): level = pair[1] if event_filter == level: events.add(event) return events
def __get_events(self, event_filters): events = set() for event_filter in to_sequence(event_filters): if event_filter not in EVENT_ALL: events.add(event_filter) else: for event, pair in self.default_handlers.items(): level = pair[1] if event_filter == level: events.add(event) return events
def list_tools_options(self, tools, brief=False): tools = set(to_sequence(tools)) result = [] for tools_options, names in self.tools._get_tools_options().items(): names_set = tools & set(names) if names_set: tools -= names_set options_name = "Options of tool: %s" % (', '.join(names)) result += tools_options.help_text(options_name, brief=brief) if result and result[-1]: result.append("") return result
def __init__(self, description=None, group=None, style=None, true_values=None, false_values=None, default=False, is_tool_key=False, is_hidden=False ): # noinspection PyTypeChecker super(BoolOptionType, self).__init__(bool, description, group, default=default, is_tool_key=is_tool_key, is_hidden=is_hidden) if style is None: style = ('true', 'false') else: style = map(IgnoreCaseString, style) if true_values is None: true_values = self.__true_values else: true_values = to_sequence(true_values) if false_values is None: false_values = self.__false_values else: false_values = to_sequence(false_values) self.true_value, self.false_value = style self.true_values = set() self.false_values = set() self.add_values(true_values, false_values) self.add_values(self.true_value, self.false_value)
def __init__(self, cmd, status=None, stdout=None, stderr=None): self.cmd = tuple(to_sequence(cmd)) self.status = status if not stdout: stdout = str() if stderr: stdout += '\n' + stderr self.output = stdout super(ExecCommandResult, self).__init__()
def _masks_to_match(masks, _null_match=lambda name: False): if not masks: return _null_match if is_string(masks): masks = masks.split('|') re_list = [] for mask in to_sequence(masks): mask = os.path.normcase(mask).strip() re_list.append("(%s)" % fnmatch.translate(mask)) re_str = '|'.join(re_list) return re.compile(re_str).match
def __new__(cls, name, signature, tags=None): self = super(EntityBase, cls).__new__(cls) if name is not NotImplemented: if not name: raise ErrorEntityNameEmpty() self.name = name if signature is not NotImplemented: self.signature = signature self.tags = frozenset(to_sequence(tags)) return self
def build_replace(self, chdir=os.chdir): if self.replace_called: return None self.replace_called = True chdir(self.cwd) sources = self.builder.replace(self.options, self.source_entities) if sources is None: return None # source_entities will be reinitialized later self.sources = tuple(to_sequence(sources)) return self.get_source_nodes()
def _add_program_exts(progs, exts): progs = to_sequence(progs) if not exts: return tuple(progs) result = [] for prog in progs: prog_ext = os.path.splitext(prog)[1] if prog_ext or (prog_ext in exts): result.append(prog) else: result += (prog + ext for ext in exts) return result
def load_tools(self, paths, reload=False): for path in to_sequence(paths): path = expand_file_path(path) if path in self.loaded_paths: if not reload: continue else: self.loaded_paths.append(path) module_files = find_files(path, mask="*.py") if not module_files: continue self._load_tools_package(path, module_files)
def build_if(self, condition, nodes): if not isinstance(condition, _NodeCondition): condition = _NodeCondition(condition) cond_node = condition.get_node() if cond_node is not None: cond_node = (cond_node, ) depends = self._nodes.depends set_node_condition = self._node_conditions.__setitem__ for node in to_sequence(nodes): if cond_node is not None: depends(node, cond_node) set_node_condition(node, condition)
def depends(self, dependencies): dep_nodes = self.dep_nodes dep_entities = self.dep_entities for entity in to_sequence(dependencies): if isinstance(entity, Node): dep_nodes.add(entity) elif isinstance(entity, NodeFilter): dep_nodes.add(entity.get_node()) elif isinstance(entity, EntityBase): dep_entities.append(entity) else: raise ErrorNodeDependencyInvalid(entity)
def __init__(self, builder, sources, cwd=None): self.builder = builder self.options = getattr(builder, 'options', None) if cwd is None: self.cwd = os.path.abspath(os.getcwd()) else: self.cwd = cwd self.initiated = False self.depends_called = False self.replace_called = False self.split_called = False self.check_actual = self._not_actual self.sources = tuple(to_sequence(sources)) self.dep_nodes = set() self.dep_entities = []