def parse(url, r, ts): d = pq(common.visit(url)) src = d("#player_one script").text() src = src[20:-8] context = js2py.EvalJs() jscode = open("./js/strencode2.js", 'r', encoding='utf8').read() context.execute(jscode) src = context.eval(src) if src is not None: src = pq(src) src = src("source").attr("src") html = common.visit(url) soup = BeautifulSoup(html, "lxml") con = soup.find(name="div", attrs={"class": "boxPart"}).text con = "".join(con.split()) t = con.split(":") times = int(t[1]) stand = int(ts) if times >= stand: print(threading.current_thread().name, " 满足条件插入redis: ", src) with open("./sed/urlwithtitle.txt", "a") as f: f.write(src) f.write("\n") redisUtil.add(src, common.KEY_SRC) r.lrem(common.KEY, 1, url) else: print(threading.current_thread().name, src, "时长不够,时长:", times, "分钟") else: print(threading.current_thread().name, url, " url的src解析为None, 插入 redis_error") redisUtil.add(url, common.KEY_NONE)
def check_adresbalk(context, role): for link in links[role]['niet']: visit(context, link) sleep(1) assert not cmp_url( context, link + '/'), '%s kan onterecht bij %s' % (role, context.browser.url)
def parse(url, c, ts): d = pq(common.visit(url)) # 旧版获取src #src = d("video").find("source").attr("src") # 新版获取src src = d("#player_one script").text() src = src[20:-8] context = js2py.EvalJs() js_code = ''';var encode_version = 'sojson.v5', lbbpm = '__0x33ad7', __0x33ad7=['QMOTw6XDtVE=','w5XDgsORw5LCuQ==','wojDrWTChFU=','dkdJACw=','w6zDpXDDvsKVwqA=','ZifCsh85fsKaXsOOWg==','RcOvw47DghzDuA==','w7siYTLCnw=='];(function(_0x94dee0,_0x4a3b74){var _0x588ae7=function(_0x32b32e){while(--_0x32b32e){_0x94dee0['push'](_0x94dee0['shift']());}};_0x588ae7(++_0x4a3b74);}(__0x33ad7,0x8f));var _0x5b60=function(_0x4d4456,_0x5a24e3){_0x4d4456=_0x4d4456-0x0;var _0xa82079=__0x33ad7[_0x4d4456];if(_0x5b60['initialized']===undefined){(function(){var _0xef6e0=typeof window!=='undefined'?window:typeof process==='object'&&typeof require==='function'&&typeof global==='object'?global:this;var _0x221728='ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=';_0xef6e0['atob']||(_0xef6e0['atob']=function(_0x4bb81e){var _0x1c1b59=String(_0x4bb81e)['replace'](/=+$/,'');for(var _0x5e3437=0x0,_0x2da204,_0x1f23f4,_0x3f19c1=0x0,_0x3fb8a7='';_0x1f23f4=_0x1c1b59['charAt'](_0x3f19c1++);~_0x1f23f4&&(_0x2da204=_0x5e3437%0x4?_0x2da204*0x40+_0x1f23f4:_0x1f23f4,_0x5e3437++%0x4)?_0x3fb8a7+=String['fromCharCode'](0xff&_0x2da204>>(-0x2*_0x5e3437&0x6)):0x0){_0x1f23f4=_0x221728['indexOf'](_0x1f23f4);}return _0x3fb8a7;});}());var _0x43712e=function(_0x2e9442,_0x305a3a){var _0x3702d8=[],_0x234ad1=0x0,_0xd45a92,_0x5a1bee='',_0x4a894e='';_0x2e9442=atob(_0x2e9442);for(var _0x67ab0e=0x0,_0x1753b1=_0x2e9442['length'];_0x67ab0e<_0x1753b1;_0x67ab0e++){_0x4a894e+='%'+('00'+_0x2e9442['charCodeAt'](_0x67ab0e)['toString'](0x10))['slice'](-0x2);}_0x2e9442=decodeURIComponent(_0x4a894e);for(var _0x246dd5=0x0;_0x246dd5<0x100;_0x246dd5++){_0x3702d8[_0x246dd5]=_0x246dd5;}for(_0x246dd5=0x0;_0x246dd5<0x100;_0x246dd5++){_0x234ad1=(_0x234ad1+_0x3702d8[_0x246dd5]+_0x305a3a['charCodeAt'](_0x246dd5%_0x305a3a['length']))%0x100;_0xd45a92=_0x3702d8[_0x246dd5];_0x3702d8[_0x246dd5]=_0x3702d8[_0x234ad1];_0x3702d8[_0x234ad1]=_0xd45a92;}_0x246dd5=0x0;_0x234ad1=0x0;for(var _0x39e824=0x0;_0x39e824<_0x2e9442['length'];_0x39e824++){_0x246dd5=(_0x246dd5+0x1)%0x100;_0x234ad1=(_0x234ad1+_0x3702d8[_0x246dd5])%0x100;_0xd45a92=_0x3702d8[_0x246dd5];_0x3702d8[_0x246dd5]=_0x3702d8[_0x234ad1];_0x3702d8[_0x234ad1]=_0xd45a92;_0x5a1bee+=String['fromCharCode'](_0x2e9442['charCodeAt'](_0x39e824)^_0x3702d8[(_0x3702d8[_0x246dd5]+_0x3702d8[_0x234ad1])%0x100]);}return _0x5a1bee;};_0x5b60['rc4']=_0x43712e;_0x5b60['data']={};_0x5b60['initialized']=!![];}var _0x4be5de=_0x5b60['data'][_0x4d4456];if(_0x4be5de===undefined){if(_0x5b60['once']===undefined){_0x5b60['once']=!![];}_0xa82079=_0x5b60['rc4'](_0xa82079,_0x5a24e3);_0x5b60['data'][_0x4d4456]=_0xa82079;}else{_0xa82079=_0x4be5de;}return _0xa82079;};if(typeof encode_version!=='undefined'&&encode_version==='sojson.v5'){function strencode(_0x50cb35,_0x1e821d){var _0x59f053={'MDWYS':'0|4|1|3|2','uyGXL':function _0x3726b1(_0x2b01e8,_0x53b357){return _0x2b01e8(_0x53b357);},'otDTt':function _0x4f6396(_0x33a2eb,_0x5aa7c9){return _0x33a2eb<_0x5aa7c9;},'tPPtN':function _0x3a63ea(_0x1546a9,_0x3fa992){return _0x1546a9%_0x3fa992;}};var _0xd6483c=_0x59f053[_0x5b60('0x0','cEiQ')][_0x5b60('0x1','&]Gi')]('|'),_0x1a3127=0x0;while(!![]){switch(_0xd6483c[_0x1a3127++]){case'0':_0x50cb35=_0x59f053[_0x5b60('0x2','ofbL')](atob,_0x50cb35);continue;case'1':code='';continue;case'2':return _0x59f053[_0x5b60('0x3','mLzQ')](atob,code);case'3':for(i=0x0;_0x59f053[_0x5b60('0x4','J2rX')](i,_0x50cb35[_0x5b60('0x5','Z(CX')]);i++){k=_0x59f053['tPPtN'](i,len);code+=String['fromCharCode'](_0x50cb35[_0x5b60('0x6','s4(u')](i)^_0x1e821d['charCodeAt'](k));}continue;case'4':len=_0x1e821d[_0x5b60('0x7','!Mys')];continue;}break;}}}else{alert('');};''' context.execute(js_code) src = context.eval(src) # 新版获取src if src != None: src = pq(src) src = src("source").attr("src") m = common.visit(url) soup = BeautifulSoup(m, "lxml") con = soup.find(name="div", attrs={"class": "boxPart"}).text con = "".join(con.split()) t = con.split(":") times = int(t[1]) ts = int(ts) if times >= ts: print(threading.current_thread().name, " 满足条件插入redis: ", src) redisutil.add(src, common.KEY_SRC) c.lrem(common.KEY, 1, url) else: print(threading.current_thread().name, src, "时长不够,时长:", times, "分钟") else: print(threading.current_thread().name, url, " url的src解析为None, 插入 redis_error") redisutil.add(url, common.KEY_NONE)
def parseList(url, page): # 将当前现场访问的页码写入日志 with open("f:/" + threading.current_thread().name + ".log", "w") as f: f.write(str(page)) # 程序强制退出后下次就不再从第一页开始了,直接从文件中读取 html = common.visit(url) if html == "error": redisutil.add(url, "error") print("出现错误了, ", url) return lst = re.compile(r'href="(.+?)"').findall(html) result = [] for a in lst: if "category=mr" in a: result.append(a) if result is not None and len(result) > 0: for a in set(result): if not redisutil.exists(a, common.KEY) and not redisutil.exists( a, common.VISITED): redisutil.add(a, common.KEY) print(threading.current_thread().name, " insert into redis ", a) else: print(threading.current_thread().name, " redis 已经存在,不再访问 ", a)
def parseList(url): html = common.visit(url) soup = BeautifulSoup(html, 'html.parser') urls = soup.find_all( name='a', attrs={"href": re.compile(r'^http(.*)view_video(.*)')}) for url in urls: lst = url.get('href') redisUtil.add(lst, common.KEY) print(threading.current_thread().name, " insert into redis ", lst)
def parse(url, c, ts): d = pq(common.visit(url)) src = d("video").find("source").attr("src") if src != None: m = common.visit(url) soup = BeautifulSoup(m, "lxml") con = soup.find(name="div", attrs={"class": "boxPart"}).text con = "".join(con.split()) t = con.split(":") times = int(t[1]) ts = int(ts) if times >= ts: print(threading.current_thread().name, " insert into redis ", src) redisutil.add(src, common.KEY_SRC) c.lrem(common.KEY, 1, url) else: print(threading.current_thread().name, src, "Not enough time") else: print(threading.current_thread().name, src, "解析为None, 插入 redis_error") redisutil.add(src, common.KEY_NONE)
def parseList(url): #lst = re.compile(r'http:\/\/91\.91p17\.space\/view_video\.php\?viewkey\=\w+').findall(common.visit(url)) m = common.visit(url) soup = BeautifulSoup(m, 'html.parser') urls = soup.find_all( name='a', attrs={"href": re.compile(r'^http(.*)view_video(.*)')}) for url in urls: lst = url.get('href') if not redisutil.exists(lst, common.KEY): redisutil.add(lst, common.KEY) print(threading.current_thread().name, " insert into redis ", lst) else: print(threading.current_thread().name, " redis 已经存在,不再访问 ", lst)
def parse(url, c, ts): d = pq(common.visit(url)) src = d("video").find("source").attr("src") m = d("#useraction .boxPart").html() cn = re.search(u'时长:</span>(.*?)<span', m, re.S).group(1) tc = "".join(cn.split()) t = tc.split(":") times = 0 if len(t) == 3: times = int(t[1]) + 60 else: times = int(t[0]) ts = int(ts) if times < ts: pass #print( "时长不够不予处理") elif src != None: print( threading.current_thread().name, " insert into redis ", src) redisutil.add(src, common.KEY_SRC) c.lrem(common.KEY, 1, url) else: print(threading.current_thread().name, src, "解析为None, 插入 redis_error") redisutil.add(src, common.KEY_NONE)