def __read_script(script, encoding = 'utf8'): file_reg = '\[(.*)\]' cmd_reg = '(\w+)\s*:\s*(.*)' tag_file = None cmd_set = {} if isinstance(script, str): content = text_file.read_file(script, 'all', encoding, True) elif isinstance(script, list): content = script elif isinstance(script, dict): return script else: raise IOError('script must be a filename or script command list') for line in content: if str_utils.is_empty(line): continue line = line.strip() if str_utils.startswith(line, '#'): continue if regex_utils.check_line(file_reg, line): tag_file = regex_utils.parse_line(file_reg, line)[0] if str_utils.is_not_empty(tag_file): tag_file = tag_file.strip() elif regex_utils.check_line(cmd_reg, line): cmd_item = regex_utils.parse_line(cmd_reg, line) if tag_file is not None and cmd_item is not None and len(cmd_item) == 2: cmd_list = cmd_set.get(tag_file) if cmd_set.has_key(tag_file) else [] cmd_list.append(cmd_item) cmd_set[tag_file] = cmd_list return cmd_set
def parse(string, rule_list): if string_utils.is_blank(string) or not rule_list: return None string = string.strip() for rule in rule_list: rule_type = rule.get('type').strip() if rule.has_key('type') else '' rule_type = 'regex' if rule_type == '' else rule_type pattern = rule.get('pattern') if rule.has_key('pattern') else '' if rule_type == 'regex': if regex_utils.check_line(pattern, string): value_list = regex_utils.parse_line(pattern, string) map_index = __build_map_index(rule) return __map_with_index(value_list, map_index) elif rule_type == 'split': match = rule.get('match') if rule.has_key('match') else '' if __is_match(match, string): value_list = string.split(pattern) map_index = __build_map_index(rule) return __map_with_index(value_list, map_index) elif rule_type == 'keyword': match = rule.get('match') if rule.has_key('match') else '' if __is_match(match, string): map_index = __build_map_index(rule) return __map_with_keyword(string, map_index.values()) return None
def __parse_line_num(cmd_arg, content): line_nums = [] if cmd_arg.startswith('/') and cmd_arg.endswith('/'): line_items = cmd_arg[1:len(cmd_arg) - 1].split(',') for line_item in line_items: line_item = line_item.strip() if '-' in line_item: line_min, line_max = line_item.split('-') if line_max > line_min: line_nums.extend([ i - 1 for i in range(int(line_min), int(line_max) + 1) ]) else: if str_utils.is_numeric(line_item): line_nums.append(int(line_item) - 1) elif cmd_arg == '[:end]': line_nums.append(len(content)) elif cmd_arg == '[:start]': line_nums.append(0) else: line_counter = 0 for line in content: try: if regex_utils.check_line(cmd_arg, line): line_nums.append(line_counter) except Exception, e: print e line_counter += 1
def __parse_line_num(cmd_arg, content): line_nums = [] if cmd_arg.startswith('/') and cmd_arg.endswith('/'): line_items = cmd_arg[1 : len(cmd_arg) - 1].split(',') for line_item in line_items: line_item = line_item.strip() if '-' in line_item: line_min, line_max = line_item.split('-') if line_max > line_min: line_nums.extend([i - 1 for i in range(int(line_min), int(line_max) + 1)]) else: if str_utils.is_numeric(line_item): line_nums.append(int(line_item) - 1) elif cmd_arg == '[:end]': line_nums.append(len(content)) elif cmd_arg == '[:start]': line_nums.append(0) else: line_counter = 0 for line in content: try: if regex_utils.check_line(cmd_arg, line): line_nums.append(line_counter) except Exception,e: print e line_counter += 1
def filter(self, filename): if string_utils.is_empty(filename): return False if hasattr(self, "file_filter"): flag = False for filter_item in self.file_filter: if regex_utils.check_line(str(filter_item), filename): flag = True break return flag else: return True
def __read_script(script, encoding='utf8'): file_reg = '\[(.*)\]' cmd_reg = '(\w+)\s*:\s*(.*)' tag_file = None cmd_set = {} if isinstance(script, str): content = text_file.read_file(script, 'all', encoding, True) elif isinstance(script, list): content = script elif isinstance(script, dict): return script else: raise IOError('script must be a filename or script command list') for line in content: if str_utils.is_empty(line): continue line = line.strip() if str_utils.startswith(line, '#'): continue if regex_utils.check_line(file_reg, line): tag_file = regex_utils.parse_line(file_reg, line)[0] if str_utils.is_not_empty(tag_file): tag_file = tag_file.strip() elif regex_utils.check_line(cmd_reg, line): cmd_item = regex_utils.parse_line(cmd_reg, line) if tag_file is not None and cmd_item is not None and len( cmd_item) == 2: cmd_list = cmd_set.get(tag_file) if cmd_set.has_key( tag_file) else [] cmd_list.append(cmd_item) cmd_set[tag_file] = cmd_list return cmd_set
def __is_match(match, line): result = False if isinstance(match, basestring): if regex_utils.check_line('(\S+)\((\S+)\)', match): fun, value = regex_utils.parse_line('(\S+)\((\S+)\)', match) if fun == 'startswith': result = line.startswith(value) elif fun == 'endswith': result = line.endswith(value) elif fun == 'in': result = value in line elif type(match) == types.FunctionType: try: result = match(line) except: result = False return result
def group(text, rule_list): if string_utils.is_blank(text) or not rule_list: return None group_value = {} for rule_item in rule_list: name = rule_item.get('name') if rule_item.has_key('name') else 'NONE' patterns = rule_item.get('pattern') if rule_item.has_key('pattern') else [] if type(patterns) == types.StringType: patterns = [patterns] for line in text: for pattern in patterns: if regex_utils.check_line(pattern, line): group_items = group_value.get(name) if group_value.has_key(name) else [] group_items.append(line) group_value[name] = group_items return group_value
def group(text, rule_list): if string_utils.is_blank(text) or not rule_list: return None group_value = {} for rule_item in rule_list: name = rule_item.get('name') if rule_item.has_key('name') else 'NONE' patterns = rule_item.get('pattern') if rule_item.has_key( 'pattern') else [] if type(patterns) == types.StringType: patterns = [patterns] for line in text: for pattern in patterns: if regex_utils.check_line(pattern, line): group_items = group_value.get(name) if group_value.has_key( name) else [] group_items.append(line) group_value[name] = group_items return group_value
def __match(line_num, line_text, model, pattern): if str_utils.is_blank(line_text): return False if str_utils.is_blank(pattern): return True patterns = [] if type(pattern) == types.ListType: patterns = pattern elif type(pattern) == types.FunctionType: patterns = [pattern] else: patterns = [str(pattern)] if str_utils.is_empty(model): model = "s" model = model.lower() for match_pattern in patterns: if model == "s": if match_pattern in line_text: return True elif model == "n": _min, _max = __split_region(match_pattern) if line_num >= _min and line_num <= _max: return True elif model == "e": if regex_utils.check_line(match_pattern, line_text): return True elif model == "a": if type(pattern) == types.FunctionType: if pattern(line_text): return True return False
def tar(filelist, tar_path = None, exclude = None): if string_utils.is_not_empty(tar_path): t = tarfile.open(name = tar_path, mode = __get_tar_mode(tar_path)) else: fileobj = tempfile.NamedTemporaryFile() t = tarfile.open(mode = 'w', fileobj=fileobj) exclude = exclude or [] if isinstance(filelist, str): filepath = os.path.abspath(filelist) filelist = [] if os.path.isdir(filepath): for root, dirs, files in os.walk(filepath): for _file in files: match = True for pattern in exclude: if regex_utils.check_line(pattern, _file): match = False break if match: filelist.append(os.path.join(root, _file)) elif os.path.isfile(filepath): filelist = [filelist] if isinstance(filelist, list): for _file in filelist: t.add(_file) t.close() if string_utils.is_not_empty(tar_path): return tar_path else: fileobj.seek(0) return fileobj
def __match(line_num, line_text, model, pattern): if str_utils.is_blank(line_text): return False if str_utils.is_blank(pattern): return True patterns = [] if type(pattern) == types.ListType: patterns = pattern elif type(pattern) == types.FunctionType: patterns = [pattern] else: patterns = [str(pattern)] if str_utils.is_empty(model): model = 's' model = model.lower() for match_pattern in patterns: if model == 's': if match_pattern in line_text: return True elif model == 'n': _min, _max = __split_region(match_pattern) if line_num >= _min and line_num <= _max: return True elif model == 'e': if regex_utils.check_line(match_pattern, line_text): return True elif model == 'a': if type(pattern) == types.FunctionType: if pattern(line_text): return True return False