def specialFilter(self): if len(self.filters) > 0: for filter in self.filters: rule = filter; rule = rule.replace('(*)', '(.+)?') if isinstance(self.content, unicode): rule = safeunicode(rule) else: rule = safestr(rule) self.content = re.compile(rule, re.I).sub("", self.content);
def getElementData(obj, rule, images=None, fetch_all=0): """ 根据rule对obj的进行解析 obj可以是pq后的对象, 也可以是html页面 images将会把解析过程的image连接插入此表中 规则可以有两种模式: 1. DOM selector 1.1 选择器类似于jquery 比如你要某个a的url >> a.attr("href") 1.2 需要一个标签内的文本内容 >> div[id="content"].text() 1.3 需要获得某个子元素中的内容 >> li.eq(1).text() #li元素组中的第2个文本内容 2. 正则模式 正则模式需要的内容使用[arg]标签,其余可以使用(*)填充 """ if not isinstance(obj, pq): obj = pq(obj); old_rule = rule rule = rule.split(".") #避免有url链接 if len(rule) > 1 and old_rule.find("[arg]") == -1: #第一个永远是dom选择 selectRule = rule.pop(0) #移除 ( ) selectRule = selectRule.replace("(", ""); selectRule = selectRule.replace(")", ""); selecteddom = obj.find(selectRule); for attr in rule: m = attrParrent.match(attr) if m: action, v = m.groups() if v: v = v.encode("utf-8") #去除引号 v = v.strip("\'").strip('\"'); if action == "attr" and hasattr(selecteddom, "attr") and v: if fetch_all == 1: values = [] dom_count = len(selecteddom) for i in range(dom_count): vv = selecteddom.eq(i).attr(v) if vv: values.append(vv) if is_image(vv): images.append(vv) return values else: value = selecteddom.attr(v) if selecteddom and selecteddom[0].tag == "img" and v == "src" and images is not None: images.append(value) return value elif action == "eq" and hasattr(selecteddom, "eq"): _rules = attr.split(" ") if len(rule) > 1: selecteddom = selecteddom.eq(int(v)) if len(_rules) > 1: ''' 假设eq后面还有子元素 eq(1) a ''' _rules.pop(0) _dom = " ".join(_rules) selecteddom = selecteddom.find(_dom) else: return selecteddom.eq(int(v)) elif action == "text" and hasattr(selecteddom, "text"): return safeunicode(selecteddom.text()).strip() elif action == "html" and hasattr(selecteddom, "html"): return safeunicode(selecteddom.html()).strip() elif len(rule) == 1: rule = rule.pop() #正则模式 if rule.find('[arg]'): content = obj.html() content_text = obj.text() rule = rule.replace('[arg]', '(.+)?') rule = rule.replace('(*)', '.+?') if isinstance(content, unicode): rule = safeunicode(rule) else: rule = safestr(rule) parrent = re.compile(rule, re.MULTILINE | re.UNICODE) try: result = parrent.search(content) if result is not None: result = safeunicode(result.group(1)).strip() return result else: result = parrent.search(content_text) if result is not None: result = safeunicode(result.group(1)).strip() return result except: return None return None