def parse(url, r, ts):
d = pq(common.visit(url))
src = d("#player_one script").text()
src = src[20:-8]
context = js2py.EvalJs()
jscode = open("./js/strencode2.js", 'r', encoding='utf8').read()
context.execute(jscode)
src = context.eval(src)
if src is not None:
src = pq(src)
src = src("source").attr("src")
html = common.visit(url)
soup = BeautifulSoup(html, "lxml")
con = soup.find(name="div", attrs={"class": "boxPart"}).text
con = "".join(con.split())
t = con.split(":")
times = int(t[1])
stand = int(ts)
if times >= stand:
print(threading.current_thread().name, " 满足条件插入redis： ", src)
with open("./sed/urlwithtitle.txt", "a") as f:
f.write(src)
f.write("\n")
redisUtil.add(src, common.KEY_SRC)
r.lrem(common.KEY, 1, url)
else:
print(threading.current_thread().name, src, "时长不够,时长：", times,
"分钟")
else:
print(threading.current_thread().name, url,
" url的src解析为None, 插入 redis_error")
redisUtil.add(url, common.KEY_NONE)
def check_adresbalk(context, role):
for link in links[role]['niet']:
visit(context, link)
sleep(1)
assert not cmp_url(
context, link +
'/'), '%s kan onterecht bij %s' % (role, context.browser.url)
def parse(url, c, ts):
d = pq(common.visit(url))
# 旧版获取src
#src = d("video").find("source").attr("src")
# 新版获取src
src = d("#player_one script").text()
src = src[20:-8]
context = js2py.EvalJs()
js_code = ''';var encode_version = 'sojson.v5', lbbpm = '__0x33ad7', __0x33ad7=['QMOTw6XDtVE=','w5XDgsORw5LCuQ==','wojDrWTChFU=','dkdJACw=','w6zDpXDDvsKVwqA=','ZifCsh85fsKaXsOOWg==','RcOvw47DghzDuA==','w7siYTLCnw=='];(function(_0x94dee0,_0x4a3b74){var _0x588ae7=function(_0x32b32e){while(--_0x32b32e){_0x94dee0['push'](_0x94dee0['shift']());}};_0x588ae7(++_0x4a3b74);}(__0x33ad7,0x8f));var _0x5b60=function(_0x4d4456,_0x5a24e3){_0x4d4456=_0x4d4456-0x0;var _0xa82079=__0x33ad7[_0x4d4456];if(_0x5b60['initialized']===undefined){(function(){var _0xef6e0=typeof window!=='undefined'?window:typeof process==='object'&&typeof require==='function'&&typeof global==='object'?global:this;var _0x221728='ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=';_0xef6e0['atob']||(_0xef6e0['atob']=function(_0x4bb81e){var _0x1c1b59=String(_0x4bb81e)['replace'](/=+$/,'');for(var _0x5e3437=0x0,_0x2da204,_0x1f23f4,_0x3f19c1=0x0,_0x3fb8a7='';_0x1f23f4=_0x1c1b59['charAt'](_0x3f19c1++);~_0x1f23f4&&(_0x2da204=_0x5e3437%0x4?_0x2da204*0x40+_0x1f23f4:_0x1f23f4,_0x5e3437++%0x4)?_0x3fb8a7+=String['fromCharCode'](0xff&_0x2da204>>(-0x2*_0x5e3437&0x6)):0x0){_0x1f23f4=_0x221728['indexOf'](_0x1f23f4);}return _0x3fb8a7;});}());var _0x43712e=function(_0x2e9442,_0x305a3a){var _0x3702d8=[],_0x234ad1=0x0,_0xd45a92,_0x5a1bee='',_0x4a894e='';_0x2e9442=atob(_0x2e9442);for(var _0x67ab0e=0x0,_0x1753b1=_0x2e9442['length'];_0x67ab0e<_0x1753b1;_0x67ab0e++){_0x4a894e+='%'+('00'+_0x2e9442['charCodeAt'](_0x67ab0e)['toString'](0x10))['slice'](-0x2);}_0x2e9442=decodeURIComponent(_0x4a894e);for(var _0x246dd5=0x0;_0x246dd5<0x100;_0x246dd5++){_0x3702d8[_0x246dd5]=_0x246dd5;}for(_0x246dd5=0x0;_0x246dd5<0x100;_0x246dd5++){_0x234ad1=(_0x234ad1+_0x3702d8[_0x246dd5]+_0x305a3a['charCodeAt'](_0x246dd5%_0x305a3a['length']))%0x100;_0xd45a92=_0x3702d8[_0x246dd5];_0x3702d8[_0x246dd5]=_0x3702d8[_0x234ad1];_0x3702d8[_0x234ad1]=_0xd45a92;}_0x246dd5=0x0;_0x234ad1=0x0;for(var _0x39e824=0x0;_0x39e824<_0x2e9442['length'];_0x39e824++){_0x246dd5=(_0x246dd5+0x1)%0x100;_0x234ad1=(_0x234ad1+_0x3702d8[_0x246dd5])%0x100;_0xd45a92=_0x3702d8[_0x246dd5];_0x3702d8[_0x246dd5]=_0x3702d8[_0x234ad1];_0x3702d8[_0x234ad1]=_0xd45a92;_0x5a1bee+=String['fromCharCode'](_0x2e9442['charCodeAt'](_0x39e824)^_0x3702d8[(_0x3702d8[_0x246dd5]+_0x3702d8[_0x234ad1])%0x100]);}return _0x5a1bee;};_0x5b60['rc4']=_0x43712e;_0x5b60['data']={};_0x5b60['initialized']=!![];}var _0x4be5de=_0x5b60['data'][_0x4d4456];if(_0x4be5de===undefined){if(_0x5b60['once']===undefined){_0x5b60['once']=!![];}_0xa82079=_0x5b60['rc4'](_0xa82079,_0x5a24e3);_0x5b60['data'][_0x4d4456]=_0xa82079;}else{_0xa82079=_0x4be5de;}return _0xa82079;};if(typeof encode_version!=='undefined'&&encode_version==='sojson.v5'){function strencode(_0x50cb35,_0x1e821d){var _0x59f053={'MDWYS':'0|4|1|3|2','uyGXL':function _0x3726b1(_0x2b01e8,_0x53b357){return _0x2b01e8(_0x53b357);},'otDTt':function _0x4f6396(_0x33a2eb,_0x5aa7c9){return _0x33a2eb<_0x5aa7c9;},'tPPtN':function _0x3a63ea(_0x1546a9,_0x3fa992){return _0x1546a9%_0x3fa992;}};var _0xd6483c=_0x59f053[_0x5b60('0x0','cEiQ')][_0x5b60('0x1','&]Gi')]('|'),_0x1a3127=0x0;while(!![]){switch(_0xd6483c[_0x1a3127++]){case'0':_0x50cb35=_0x59f053[_0x5b60('0x2','ofbL')](atob,_0x50cb35);continue;case'1':code='';continue;case'2':return _0x59f053[_0x5b60('0x3','mLzQ')](atob,code);case'3':for(i=0x0;_0x59f053[_0x5b60('0x4','J2rX')](i,_0x50cb35[_0x5b60('0x5','Z(CX')]);i++){k=_0x59f053['tPPtN'](i,len);code+=String['fromCharCode'](_0x50cb35[_0x5b60('0x6','s4(u')](i)^_0x1e821d['charCodeAt'](k));}continue;case'4':len=_0x1e821d[_0x5b60('0x7','!Mys')];continue;}break;}}}else{alert('');};'''
context.execute(js_code)
src = context.eval(src)
# 新版获取src
if src != None:
src = pq(src)
src = src("source").attr("src")
m = common.visit(url)
soup = BeautifulSoup(m, "lxml")
con = soup.find(name="div", attrs={"class": "boxPart"}).text
con = "".join(con.split())
t = con.split(":")
times = int(t[1])
ts = int(ts)
if times >= ts:
print(threading.current_thread().name, " 满足条件插入redis： ", src)
redisutil.add(src, common.KEY_SRC)
c.lrem(common.KEY, 1, url)
else:
print(threading.current_thread().name, src, "时长不够,时长：", times,
"分钟")
else:
print(threading.current_thread().name, url,
" url的src解析为None, 插入 redis_error")
redisutil.add(url, common.KEY_NONE)
def parseList(url, page):
# 将当前现场访问的页码写入日志
with open("f:/" + threading.current_thread().name + ".log", "w") as f:
f.write(str(page))
# 程序强制退出后下次就不再从第一页开始了，直接从文件中读取
html = common.visit(url)
if html == "error":
redisutil.add(url, "error")
print("出现错误了, ", url)
return
lst = re.compile(r'href="(.+?)"').findall(html)
result = []
for a in lst:
if "category=mr" in a:
result.append(a)
if result is not None and len(result) > 0:
for a in set(result):
if not redisutil.exists(a, common.KEY) and not redisutil.exists(
a, common.VISITED):
redisutil.add(a, common.KEY)
print(threading.current_thread().name, " insert into redis ",
a)
else:
print(threading.current_thread().name, " redis 已经存在，不再访问 ", a)
def parseList(url):
html = common.visit(url)
soup = BeautifulSoup(html, 'html.parser')
urls = soup.find_all(
name='a', attrs={"href": re.compile(r'^http(.*)view_video(.*)')})
for url in urls:
lst = url.get('href')
redisUtil.add(lst, common.KEY)
print(threading.current_thread().name, " insert into redis ", lst)
def parse(url, c, ts):
d = pq(common.visit(url))
src = d("video").find("source").attr("src")
if src != None:
m = common.visit(url)
soup = BeautifulSoup(m, "lxml")
con = soup.find(name="div", attrs={"class": "boxPart"}).text
con = "".join(con.split())
t = con.split(":")
times = int(t[1])
ts = int(ts)
if times >= ts:
print(threading.current_thread().name, " insert into redis ", src)
redisutil.add(src, common.KEY_SRC)
c.lrem(common.KEY, 1, url)
else:
print(threading.current_thread().name, src, "Not enough time")
else:
print(threading.current_thread().name, src, "解析为None, 插入 redis_error")
redisutil.add(src, common.KEY_NONE)
def parseList(url):
#lst = re.compile(r'http:\/\/91\.91p17\.space\/view_video\.php\?viewkey\=\w+').findall(common.visit(url))
m = common.visit(url)
soup = BeautifulSoup(m, 'html.parser')
urls = soup.find_all(
name='a', attrs={"href": re.compile(r'^http(.*)view_video(.*)')})
for url in urls:
lst = url.get('href')
if not redisutil.exists(lst, common.KEY):
redisutil.add(lst, common.KEY)
print(threading.current_thread().name, " insert into redis ", lst)
else:
print(threading.current_thread().name, " redis 已经存在，不再访问 ", lst)
def parse(url, c, ts):
d = pq(common.visit(url))
src = d("video").find("source").attr("src")
m = d("#useraction .boxPart").html()
cn = re.search(u'时长:</span>(.*?)<span', m, re.S).group(1)
tc = "".join(cn.split())
t = tc.split(":")
times = 0
if len(t) == 3:
times = int(t[1]) + 60
else:
times = int(t[0])
ts = int(ts)
if times < ts:
pass
#print( "时长不够不予处理")
elif src != None:
print( threading.current_thread().name, " insert into redis ", src)
redisutil.add(src, common.KEY_SRC)
c.lrem(common.KEY, 1, url)
else:
print(threading.current_thread().name, src, "解析为None, 插入 redis_error")
redisutil.add(src, common.KEY_NONE)
