def looper_js(self, result, raw, exists, ruler, captup=None): cap = captup if captup != None: if captup.count(' ') == 2: cap = Sh.str_to_tup(captup) list = ExtraJSON.extra_any_json(raw, ruler, cap=cap) if len(list) > 0: for item in list: """日常抓取时的重复验证""" if 1>0:#if exists.count(item['link']) < 1: result.append(item) else: break
def looper_js(self, result:list, raw:str, exists:list, ruler:str, extra3_tup:str=None, list_json_path:str=None, identifier_key:str=None): # cap = ('data_callback(', ')') cap = None path = None if identifier_key is None: identifier_key = 'link' if extra3_tup is not None: cap = Sh.str_to_tup(extra3_tup) elif list_json_path is not None: path = Sh.separator(list_json_path, '->') list = ExtraJSON.extra_any_json(raw, ruler, cap=cap, list_path=path) if len(list) > 0: for item in list: """日常抓取时的重复验证""" if exists.count(item[identifier_key]) < 1: result.append(item) else: break