def decode(html, cw=None):
'''
decode
'''
print_ = get_print(cw)
print_('decode')
soup = Soup(html)
for script in soup.findAll('script'):
script = script.text or script.string or ''
script = script.strip()
if 'videoUrl' in script:
break
else:
raise Exception('No script')
flashvars = script.split()[1]
script = 'playerObjList={};' + script
context = js2py.EvalJs()
context.execute(script)
return context.eval(flashvars).to_dict()
def getcookies():
url = 'https://www.hostloc.com/forum.php?mod=forumdisplay&fid=45&filter=author&orderby=dateline'
js = js2py.EvalJs()
headers = {
"Referer":
"http://www.baidu.com",
'user-agent':
'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.76 Mobile Safari/537.36'
}
aesjs = requests.get("https://www.hostloc.com/aes.min.js",
headers=headers,
timeout=5).text
print aesjs
js.execute(aesjs)
getcookie = requests.get(url).text
#print getcookie
getcookie_script = re.findall("<script>(.*?)</script>", getcookie)
js.execute(getcookie_script[0].split("document")[0])
data = js.toHex(js.slowAES.decrypt(js.c, 2, js.a, js.b))
cookie = "L7FW=" + data
print cookie
return cookie
def __init__(self):
#加载主页
self.session = requests.Session()
self.headers={
"Cookie":"Hm_lpvt_64ecd82404c51e03dc91cb9e8c025574=1541668433; Hm_lvt_64ecd82404c51e03dc91cb9e8c025574=1541430897,1541471052,1541641285,1541668433; from_lang_often=%5B%7B%22value%22%3A%22de%22%2C%22text%22%3A%22%u5FB7%u8BED%22%7D%2C%7B%22value%22%3A%22zh%22%2C%22text%22%3A%22%u4E2D%u6587%22%7D%2C%7B%22value%22%3A%22en%22%2C%22text%22%3A%22%u82F1%u8BED%22%7D%5D; to_lang_often=%5B%7B%22value%22%3A%22en%22%2C%22text%22%3A%22%u82F1%u8BED%22%7D%2C%7B%22value%22%3A%22zh%22%2C%22text%22%3A%22%u4E2D%u6587%22%7D%5D; FANYI_WORD_SWITCH=1; HISTORY_SWITCH=1; REALTIME_TRANS_SWITCH=1; SOUND_PREFER_SWITCH=1; SOUND_SPD_SWITCH=1; locale=zh; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; H_PS_PSSID=1430_21119_27401_26350_22158; PSINO=1; delPer=0; BDRCVFR[feWj1Vr5u3D]=I67x6TjHwwYf0; BDUSS=hsVWV6czh0a1hOQ3BaYkhTM0FrOXhNYnBCUWFsMlY0clhlYkNvTkRKdENDUGxiQUFBQUFBJCQAAAAAAAAAAAEAAAAFyMhmAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEJ70VtCe9FbY0; BIDUPSID=C53590ACE23DAC88DBE0C3D65AEBAA30; PSTM=1539535646; BAIDUID=DB00283B42FBC875B67496A00F47ABAB:FG=1",
"Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Upgrade-Insecure-Requests":"1",
"Host":"fanyi.baidu.com",
"User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0.1 Safari/605.1.15",
"Accept-Language":"zh-cn",
"Accept-Encoding": "gzip, deflate"
}
self.session.headers = self.headers
response = self.session.get("https://fanyi.baidu.com/")
#获取token
self.token = re.findall("token: ('.*'),",response.content.decode())[0]
self.token=self.token[1:-1]
print(self.token)
#获取gtk
# 获取window.gtk的值。
self.gtk = re.findall(";window.gtk = ('.*?');", response.content.decode())[0]
#初始化Javascript脚本执行上下文环境
self.context = js2py.EvalJs()
def _myfreshinfo(query, data, url):
vido_url = ''
data2 = getUrl(query)
s_hex = re.compile('unescape\(["\'](.*?)["\']\)').findall(data2)
s_hex0 = s_hex[0].replace('%', '').decode('hex')
f_name = re.compile('function (.*?)\(').findall(s_hex0)[0]
s_hex1 = s_hex[1].replace('%', '').decode('hex')
encrypted = re.compile(
'<SCRIPT LANGUAGE="JavaScript">%s\("(.*?)"\);</SCRIPT>' % f_name,
re.DOTALL).findall(data2)
if encrypted:
code = encrypted[0]
#code = code.encode('utf-8')
code = '' + code.strip('\\n') + ''
try:
fun = s_hex0.replace('document.write(tttmmm)', 'return tttmmm')
ctx = execjs.compile(fun)
decoded = ctx.call(f_name, code)
except:
context = js2py.EvalJs()
context.execute(fun)
decoded = getattr(context, f_name)(code)
return vido_url
def fixed_fun(function,url):
try:
#print(function)
js = function.replace("<script>", "").replace("</script>", "").replace("{eval(", "{var my_data_1 = (")
# print(js)
# 使用js2py的js交互功能获得刚才赋值的data1对象
context = js2py.EvalJs()
context.execute(js)
js_temp = context.my_data_1
#print(js_temp)
index1 = js_temp.find("document.")
index2 = js_temp.find("};if((")
js_temp = js_temp[index1:index2].replace("document.cookie", "my_data_2")
new_js_temp = re.sub(r'document.create.*?firstChild.href', '"{}"'.format(url), js_temp)
# print(new_js_temp)
# print(type(new_js_temp))
context.execute(new_js_temp)
data = context.my_data_2
# print(data)
__jsl_clearance = str(data).split(';')[0]
return __jsl_clearance
except:
return None
def get_page_from_url(self, url):
headers = get_request_headers()
response = requests.get(url, headers=headers)
if response.status_code == 521:
# 生成cookie信息, 再携带cookie发送请求
# 生成 `_ydclearance` cookie信息,控制台preserve log,第一个页面就是加密页面521用来做反爬跳转
# 1. 确定 _ydclearance 是从哪里来的;
# 观察发现: 这个cookie在前两个页面都没有返回，说明信息不使用通过服务器响应设置过来的; 那么他就是通过js生成.
# 2. 第一次发送请求的页面中, 有一个生成这个cookie的js; 执行这段js, 生成我们需要的cookie
# 这段js是经过加密处理后的js, 真正js在 "po" 中.
# 提取 `jp(107)` 调用函数的方法, 以及函数
result = re.findall(
'window.onload=setTimeout\("(.+?)", 200\);\s*(.+?)\s*</script> ',
response.content.decode('GBK'))
# print(result)
# 我希望执行js时候, 返回真正要执行的js
# 把 `eval("qo=eval;qo(po);")` 替换为 return po
func_str = result[0][1]
func_str = func_str.replace('eval("qo=eval;qo(po);")', 'return po')
# print(func_str)
# 获取执行js的环境
context = js2py.EvalJs()
# 加载(执行) func_str
context.execute(func_str)
# 执行这个方法, 生成我们需要的js
# code = gv(50)
context.execute('code = {};'.format(result[0][0]))
# 打印最终生成的代码
# print(context.code)
cookie_str = re.findall("document.cookie='(.+?); ",
context.code)[0]
# print(cookie_str)
headers['Cookie'] = cookie_str
response = requests.get(url, headers=headers)
return response.content.decode('GBK')
else:
return response.content.decode('GBK')
def send_request(self):
# get百度翻译主页，是为了得到token
# req_get = requests.get(self.url_index, headers=self.headers)
# token = re.search(r"token: '(.*?)',", req_get.text, re.S).group(1)
# 使用js2py在python中运行js代码并得到sign
# 实例化js解释对象
run_js = js2py.EvalJs()
# 调用js对象编码
run_js.execute(self.get_js())
sign = run_js.e(self.keyword)
data = {
"query": self.keyword,
"from": "zh",
"to": "en",
"token": "519477ac7aa72b946a3b7d9e3190461e",
"sign": sign,
}
response = requests.post(self.url_api, headers=self.headers, data=data)
# result = response.json()
result = json.loads(response.content.decode())
# print(result)
return result
def get_tt_fund_manager_detail(funCode):
timestamp = time.time() * 1000
apiUrl = "http://fund.eastmoney.com/pingzhongdata/" + funCode + ".js?v=20210226084939"
header = {
"Accept":
"* / *",
"Accept-Encoding":
"gzip, deflate",
"Accept-Language":
"zh-CN,zh-TW;q=0.9,zh;q=0.8",
"Cache-Control":
"no-cache",
"Connection":
"keep-alive",
"Cookie":
"st_si=75540904388619; st_asi=delete; ASP.NET_SessionId=nvv1yet2ffaes4z5spbaoziz; EMFUND1=null; EMFUND2=null; EMFUND3=null; EMFUND4=null; EMFUND5=null; EMFUND6=null; EMFUND7=null; EMFUND0=null; qgqp_b_id=42a63a14f8cfdba4d003f387cd2f086f; EMFUND9=02-26%2008%3A53%3A59@%23%24%u56FD%u5BCC%u5065%u5EB7%u4F18%u8D28%u751F%u6D3B%u80A1%u7968@%23%24000761; EMFUND8=02-26 08:55:54@#$%u519C%u94F6%u65B0%u80FD%u6E90%u4E3B%u9898@%23%24002190; st_pvi=97911961255996; st_sp=2021-02-24%2014%3A23%3A57; st_inirUrl=https%3A%2F%2Fwww.baidu.com%2Flink; st_sn=157; st_psi=20210226085557905-0-9129869508",
"Host":
"fund.eastmoney.com",
"Pragma":
"no-cache",
"Referer":
"http://fund.eastmoney.com/" + funCode + ".html",
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4168.3 Safari/537.36"
}
proxies = {"http": None, "https": None}
response = requests.get(apiUrl, {}, headers=header, proxies=proxies)
responseText = response.text
# - 使用js2py生成js的执行环境:context
context = js2py.EvalJs()
context.execute(responseText)
return context.Data_currentFundManager[0].power.avr
def BaiDu(self, file):
run_js = js2py.EvalJs({})
run_js.execute(self.Get_Js())
sign = run_js.e(file)
url_api = 'https://fanyi.baidu.com/v2transapi'
is_it = self.Translate(file)
if is_it:
iia = 'zh'
iib = 'en'
else:
iia = 'en'
iib = 'zh'
data = {
'from': iia,
'to': iib,
'query': file,
'transtype': 'realtime',
'simple_means_flag': '3',
'sign': sign,
'token': self.token
}
html = self.session.post(url=url_api, headers=self.headers,
data=data).json()
return (html['trans_result']['data'][0]['dst'])
def get_sojson_encoder(script):
tree = pyjsparser.parse(script)
encsname, encseles, encsnumb = None, None, None
for idx, node in enumerate(tree['body']):
if node['type'] == 'VariableDeclaration' and \
node.get('declarations'):
for encs in node.get('declarations'):
if encs.get('type') == 'VariableDeclarator' and encs.get(
'init').get('type') == 'ArrayExpression':
encsname = encs.get('id').get('name')
encseles = [
i.get('value')
for i in encs.get('init').get('elements')
]
if node['type'] == 'ExpressionStatement' and node['expression'].get(
'type') == 'CallExpression':
encsnumb = node.get('expression').get('arguments')[1].get('value')
if node['type'] == 'VariableDeclaration' and \
node.get('declarations') and \
node.get('declarations')[0].get('init') and \
node.get('declarations')[0].get('init').get('type') == 'FunctionExpression':
break
oldtree = tree['body'].copy()
tree['body'] = oldtree[:idx + 1]
funcname = node.get('declarations')[0].get('id').get('name')
use_my_rc4 = True
if use_my_rc4:
s = SojsonRc4(encseles, encsnumb)
decoder = {}
decoder[funcname] = s.sojsonrc4
else:
decoder = js2py.EvalJs()
convenience_tree_null(tree)
decoder.execute(get_script_from_tree(tree))
tree['body'] = oldtree[idx + 1:]
return funcname, decoder, tree
# 准备获取rkey
headers = {
'User-Agent':
'Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1',
}
rkey = 'http://activity.renren.com/livecell/rKey'
response = requests.get(url=rkey, headers=headers)
# {'data': {'msg': 'ok', 'maxdigits': '19', 'code': 0, 'isEncrypt': True, 'n': '91eb08c0d76c37fc1d7521fbaabbeef5d1d88063e7bb2ef24b9a08f0347ec617', 'rkey': 'fa647ac3744264c8eece684183c36fe5', 'e': '10001'}}
# print(response.json()) # 目的是去到data的头
# {'msg': 'ok', 'maxdigits': '19', 'code': 0, 'isEncrypt': True, 'n': '91eb08c0d76c37fc1d7521fbaabbeef5d1d88063e7bb2ef24b9a08f0347ec617', 'rkey': 'fa647ac3744264c8eece684183c36fe5', 'e': '10001'}
n = response.json()['data']
# print(n)
# 创建js环境
context = js2py.EvalJs()
# 准备参数
context.t = {'password': '******'}
context.n = n
# 准备js执行环境
def get_js_from_url(url):
response = requests.get(url=url, headers=headers)
return response.content.decode()
# 把需要加载的js全部加载进来,如果这里不知道要加载哪几个个js,那就把所有js都加载进来
context.execute(
get_js_from_url(
'http://s.xnimg.cn/a85738/wap/mobile/wechatLive/js/BigInt.js'))
# Simple PyWrapper test
class Foo:
def __init__(self, bar):
self.bar = bar
self.bar_history = []
def set_bar(self, x):
import copy
self.bar_history.append(copy.deepcopy(self))
self.bar = x
def get_bar(self):
return self.bar
pyobj = Foo(5)
context = js2py.EvalJs({'foo': pyobj})
assert context.foo.bar == 5
context.execute('foo.bar = 7')
assert context.foo.bar == 7
context.execute('foo.bar += 15.22')
assert context.foo.bar == 22.22
context.execute('foo.bar /= 2')
assert context.foo.bar == 11.11
assert pyobj.bar == 11.11
context.execute('foo.set_bar(foo.get_bar())')
assert context.foo.get_bar() == 11.11
context.execute('foo.set_bar(33)')
assert context.foo.get_bar() == 33
assert context.eval('foo.bar_history.push(foo.bar_history[1].get_bar());foo.bar_history[foo.bar_history.length-1].get_bar()') == 11.11
print("Passed ECMA 5 simple tests!\n"+30*'-')
'Cookie':
'BIDUPSID=A360C8CE43082B9E2E23B5B111FC4363; PSTM=1485329732; to_lang_often=%5B%7B%22value%22%3A%22en%22%2C%22text%22%3A%22%u82F1%u8BED%22%7D%2C%7B%22value%22%3A%22zh%22%2C%22text%22%3A%22%u4E2D%u6587%22%7D%5D; REALTIME_TRANS_SWITCH=1; FANYI_WORD_SWITCH=1; HISTORY_SWITCH=1; SOUND_SPD_SWITCH=1; SOUND_PREFER_SWITCH=1; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; H_PS_PSSID=26524_1466_21124_18560_26350_22158; locale=zh; Hm_lvt_64ecd82404c51e03dc91cb9e8c025574=1531663222,1531743493; Hm_lpvt_64ecd82404c51e03dc91cb9e8c025574=1531748592; from_lang_often=%5B%7B%22value%22%3A%22zh%22%2C%22text%22%3A%22%u4E2D%u6587%22%7D%2C%7B%22value%22%3A%22en%22%2C%22text%22%3A%22%u82F1%u8BED%22%7D%5D; BAIDUID=96A2302A5D66AFBE539FFC68E881260F:FG=1',
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.104 Safari/537.36 Core/1.53.4882.400 QQBrowser/9.7.13076.400',
}
#get百度翻译主页，是为了得到token
req_get = requests.get(url=url_fanyi, headers=headers)
token = re.search(r"token: '(.*?)',", req_get.text, re.S).group(1)
#需要翻译的内容，中英文都可以
translation_content = '我是一个中国人，我不为五斗米折腰'
#读入js代码
def get_js():
with open('sign.js', 'r', encoding='utf-8') as f:
return f.read()
#使用js2py在python中运行js代码并得到sign
run_js = js2py.EvalJs({})
run_js.execute(get_js())
sign = run_js.e(translation_content)
data = {'query': translation_content, 'sign': sign, 'token': token}
req_post = requests.post(url=url_api, data=data, headers=headers)
result = json.loads(req_post.text)
print(result['trans_result']['data'][0]['dst'])
def __init__(self, username, password):
self.context = js2py.EvalJs() # python中使用js
self.username = username
self.password = password
def getTk(self, word):
evaljs = js2py.EvalJs()
js_code = js.gg_js_code
evaljs.execute(js_code)
tk = evaljs.TL(word)
return tk
winreg.CloseKey(key)
string = input('输入字符串：')
headers = {
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
'Cookie':
'BAIDUID=4650B0B34048BBAA1E0B909B42F5A564:FG=1;BIDUPSID=4650B0B34048BBAA1E0B909B42F5A564;PSTM=1537177909;BDUSS=w0VmEzUFFWTTh0bld5VWVhNVo5MEEyV2ZKdTk3U2stMGZmWVQ1TTRuSnVkOHBiQVFBQUFBJCQAAAAAAAAAAAEAAAD0GzcNaG9uZ3F1YW4xOTkxAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAG7qoltu6qJbTk;pgv_pvi=6774493184;uc_login_unique=19e6fd48035206a8abe89f98c3fc542a;uc_recom_mark=cmVjb21tYXJrXzYyNDU4NjM%3D;MCITY=-218%3A;cflag=15%3A3;SIGNIN_UC=70a2711cf1d3d9b1a82d2f87d633bd8a02893452711;locale=zh;Hm_lvt_64ecd82404c51e03dc91cb9e8c025574=1539333192;from_lang_often=%5B%7B%22value%22%3A%22en%22%2C%22text%22%3A%22%u82F1%u8BED%22%7D%2C%7B%22value%22%3A%22zh%22%2C%22text%22%3A%22%u4E2D%u6587%22%7D%5D;REALTIME_TRANS_SWITCH=1;FANYI_WORD_SWITCH=1;HISTORY_SWITCH=1;SOUND_SPD_SWITCH=1;SOUND_PREFER_SWITCH=1;to_lang_often=%5B%7B%22value%22%3A%22zh%22%2C%22text%22%3A%22%u4E2D%u6587%22%7D%2C%7B%22value%22%3A%22en%22%2C%22text%22%3A%22%u82F1%u8BED%22%7D%5D;Hm_lpvt_64ecd82404c51e03dc91cb9e8c025574=1539333307'
}
html = requests.get('https://fanyi.baidu.com',
headers=headers,
proxies={
'http': proxyserver,
'https': proxyserver
})
gtk = re.findall("window.gtk = '(.*?)';", html.text)[0]
js = js2py.EvalJs()
js.execute(
'function a(r,o){for(var t=0;t<o.length-2;t+=3){var a=o.charAt(t+2);a=a>="a"?a.charCodeAt(0)-87:Number(a),a="+"===o.charAt(t+1)?r>>>a:r<<a,r="+"===o.charAt(t)?r+a&4294967295:r^a}return r}var C=null;var hash=function(r,_gtk){var o=r.length;o>30&&(r=""+r.substr(0,10)+r.substr(Math.floor(o/2)-5,10)+r.substr(-10,10));var t=void 0,t=null!==C?C:(C=_gtk||"")||"";for(var e=t.split("."),h=Number(e[0])||0,i=Number(e[1])||0,d=[],f=0,g=0;g<r.length;g++){var m=r.charCodeAt(g);128>m?d[f++]=m:(2048>m?d[f++]=m>>6|192:(55296===(64512&m)&&g+1<r.length&&56320===(64512&r.charCodeAt(g+1))?(m=65536+((1023&m)<<10)+(1023&r.charCodeAt(++g)),d[f++]=m>>18|240,d[f++]=m>>12&63|128):d[f++]=m>>12|224,d[f++]=m>>6&63|128),d[f++]=63&m|128)}for(var S=h,u="+-a^+6",l="+-3^+b+-f",s=0;s<d.length;s++)S+=d[s],S=a(S,u);return S=a(S,l),S^=i,0>S&&(S=(2147483647&S)+2147483648),S%=1e6,S.toString()+"."+(S^h)}'
)
sign = js.hash(string, gtk)
token = re.findall("token: '(.+)',", html.text)[0]
landata = {'query': string}
lanhtml = requests.post('https://fanyi.baidu.com/langdetect',
data=landata,
proxies={
'http': proxyserver,
'https': proxyserver
})
lan = lanhtml.json()['lan']
if lan == 'zh':
to = 'en'
def get_sign(self, word): # 计算sign值
session = requests.Session()
session.headers = self.headers
response = session.get("http://fanyi.baidu.com/")
gtk = re.findall(";window.gtk = ('.*?');",
response.content.decode())[0]
context = js2py.EvalJs()
js = r'''
function a(r) {
if (Array.isArray(r)) {
for (var o = 0, t = Array(r.length); o < r.length; o++)
t[o] = r[o];
return t
}
return Array.from(r)
}
function n(r, o) {
for (var t = 0; t < o.length - 2; t += 3) {
var a = o.charAt(t + 2);
a = a >= "a" ? a.charCodeAt(0) - 87 : Number(a),
a = "+" === o.charAt(t + 1) ? r >>> a : r << a,
r = "+" === o.charAt(t) ? r + a & 4294967295 : r ^ a
}
return r
}
function e(r) {
var o = r.match(/[\uD800-\uDBFF][\uDC00-\uDFFF]/g);
if (null === o) {
var t = r.length;
t > 30 && (r = "" + r.substr(0, 10) + r.substr(Math.floor(t / 2) - 5, 10) + r.substr(-10, 10))
} else {
for (var e = r.split(/[\uD800-\uDBFF][\uDC00-\uDFFF]/), C = 0, h = e.length, f = []; h > C; C++)
"" !== e[C] && f.push.apply(f, a(e[C].split(""))),
C !== h - 1 && f.push(o[C]);
var g = f.length;
g > 30 && (r = f.slice(0, 10).join("") + f.slice(Math.floor(g / 2) - 5, Math.floor(g / 2) + 5).join("") + f.slice(-10).join(""))
}
var u = void 0
, l = "" + String.fromCharCode(103) + String.fromCharCode(116) + String.fromCharCode(107);
u = 'null !== i ? i : (i = window[l] || "") || ""';
for (var d = u.split("."), m = Number(d[0]) || 0, s = Number(d[1]) || 0, S = [], c = 0, v = 0; v < r.length; v++) {
var A = r.charCodeAt(v);
128 > A ? S[c++] = A : (2048 > A ? S[c++] = A >> 6 | 192 : (55296 === (64512 & A) && v + 1 < r.length && 56320 === (64512 & r.charCodeAt(v + 1)) ? (A = 65536 + ((1023 & A) << 10) + (1023 & r.charCodeAt(++v)),
S[c++] = A >> 18 | 240,
S[c++] = A >> 12 & 63 | 128) : S[c++] = A >> 12 | 224,
S[c++] = A >> 6 & 63 | 128),
S[c++] = 63 & A | 128)
}
for (var p = m, F = "" + String.fromCharCode(43) + String.fromCharCode(45) + String.fromCharCode(97) + ("" + String.fromCharCode(94) + String.fromCharCode(43) + String.fromCharCode(54)), D = "" + String.fromCharCode(43) + String.fromCharCode(45) + String.fromCharCode(51) + ("" + String.fromCharCode(94) + String.fromCharCode(43) + String.fromCharCode(98)) + ("" + String.fromCharCode(43) + String.fromCharCode(45) + String.fromCharCode(102)), b = 0; b < S.length; b++)
p += S[b],
p = n(p, F);
return p = n(p, D),
p ^= s,
0 > p && (p = (2147483647 & p) + 2147483648),
p %= 1e6,
p.toString() + "." + (p ^ m)
}
'''
# js中添加一行gtk
js = js.replace('\'null !== i ? i : (i = window[l] || "") || ""\'',
gtk)
# 执行js
context.execute(js)
sign = context.e(word)
return sign
def get_js_ctx():
js = open(r'./js/securityencode.js', encoding='utf-8').read()
context = js2py.EvalJs()
context.execute(js)
return context
def getTimeInfo(courseId):
text = """POST /dwr/call/plaincall/PlanNewBean.getPlanCourseDetail.dwr HTTP/1.1
Host: study.163.com
Connection: close
Content-Length: 253
Pragma: no-cache
Cache-Control: no-cache
Origin: https://study.163.com
providerId: 1021000286
User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36
Content-Type: text/plain
Accept: */*
Referer: https://study.163.com/course/introduction.htm?courseId=1004016002
Accept-Language: zh-CN,zh;q=0.9
Cookie:
callCount=1
scriptSessionId=${scriptSessionId}191
httpSessionId=0bf2a155644541f8a026097260f243f7
c0-scriptName=PlanNewBean
c0-methodName=getPlanCourseDetail
c0-id=0
c0-param0=string:1005355015
c0-param1=number:0
c0-param2=null:null
batchId=1559655289189
"""
text = text.replace("1005355015", str(courseId))
# print(text)
aa = parse_raw(text)
response = requests.post(
aa["url"],
data=aa["body"],
headers=aa["headers"],
).text
# r = re.compile()
response = response.replace("//#DWR-INSERT", "")
response = response.replace("//#DWR-REPLY", "")
response = re.sub("dwr.engine._remoteHandleCallback\(.*\);", "", response)
# print(response)
context = js2py.EvalJs()
context.execute(response.replace("\n", ""))
totalTime = 0
content = ""
pcontent = ""
for i in range(200):
try:
s = eval("context.s" + str(i))
if s["allCount"] and s["allCount"] > 0:
chapterTime = 0.0
chapterName = s["name"]
chapterList = list()
for j in range(len(s["lessonDtos"])):
lessonDto = s["lessonDtos"][j]
time = lessonDto.videoTime
chapterTime += time
chapterList.insert(0,
(lessonDto.lessonName, getTime(time)))
totalTime += chapterTime
chapterTime = getTime(chapterTime)
content += "{}({})\n".format(chapterName, chapterTime)
pcontent += "{} {}\n".format(
my_format(chapterName + " ", 19, 'l', '+'), chapterTime)
for t in range(len(chapterList)):
sn, st = chapterList[t]
zh = len_zh(sn)
content += "{},{}\n".format(sn, st)
pcontent += "{}{}\n".format(my_format(sn, 20, 'l'), st)
content += "\n"
pcontent += "\n"
except Exception as e:
if str(
e
) in ("'>' not supported between instances of 'NoneType' and 'int'",
"'int' object is not iterable"):
continue
if str(e).startswith("ReferenceError:"): continue
print(e)
traceback.print_exc()
continue
content += "总计,{}".format(getTime(totalTime))
pcontent += "{} {}\n".format(my_format("总计 ", 19, 'l', '+'),
getTime(totalTime))
return (content, pcontent)
def build_cache():
"""
Get current data from the website http://www.lfd.uci.edu/~gohlke/pythonlibs/
Returns
-------
Dictionary containing package details
"""
data = {}
response = requests.request("GET", MAIN_URL, headers=HEADER)
soup = BeautifulSoup(response.text, features="html.parser")
# We mock out a little javascript environment within which to run Gohlke's obfuscation code
context = js2py.EvalJs()
context.execute(
"""
top = {location: {href: ''}};
location = {href: ''};
function setTimeout(f, t) {
f();
};
"""
)
# We grab Gohlke's code and evaluate it within py2js
dl_function = re.search(r"function dl.*\}", soup.find("script").text).group(0)
context.execute(dl_function)
links = soup.find(class_="pylibs").find_all("a")
for link in links:
if link.get("onclick") is not None:
# Evaluate the obfuscation javascript, store the result (squirreled away within location.href) into url
regex_result = re.search(r"dl\(.*\)", link.get("onclick"))
if regex_result is None:
logger.info(u"Skip %s (wrong link format)" % unicode(link.string))
continue
context.execute(regex_result.group(0))
url = context.location.href
# Details = [package, version, pyversion, --, arch]
details = url.split("/")[-1].split("-")
pkg = details[0].lower().replace("_", "-")
# Not using EXEs and ZIPs
if len(details) != 5:
logger.info(u"Skip %s (wrong name format)" % unicode(link.string))
continue
else:
logger.debug(u"Add %s" % unicode(link.string))
# arch = win32 / win_amd64 / any
arch = details[4]
arch = arch.split(".")[0]
# ver = cpXX / pyX / pyXXx
pkg_ver = details[1]
py_ver = details[2]
py_ver_key = py_ver + "-" + arch
# print({py_ver_key: {pkg_ver: url}})
if pkg in data.keys():
if py_ver_key in data[pkg].keys():
data[pkg][py_ver_key].update({pkg_ver: url})
else:
data[pkg][py_ver_key] = {pkg_ver: url}
else:
data[pkg] = {py_ver_key: {pkg_ver: url}}
else:
if link.string:
logger.debug(u"Skip %s (missing link)" % unicode(link.string))
return data
def solve_cf_challenge(self, resp, **original_kwargs):
body = resp.text
parsed_url = urlparse(resp.url)
domain = parsed_url.netloc
if '/cdn-cgi/l/chk_captcha' in body or 'cf_chl_captcha' in body:
raise CloudflareError(
'Cloudflare captcha presented for %s, please notify SickGear for an update, ua: %s'
% (domain, self.cf_ua),
response=resp)
try:
action, method = re.findall(
r'(?sim)<form.*?id="challenge.*?action="/?([^?"]+).*?method="([^"]+)',
body)[0]
except (Exception, BaseException):
action, method = 'cdn-cgi/l/chk_jschl', resp.request.method
submit_url = '%s://%s/%s' % (parsed_url.scheme, domain, action)
cloudflare_kwargs = {
k: v
for k, v in original_kwargs.items() if k not in ['hooks']
}
params = cloudflare_kwargs.setdefault(
('data', 'params')['GET' == method.upper()], {})
headers = cloudflare_kwargs.setdefault('headers', {})
headers['Referer'] = resp.url
try:
token = re.findall(r'(?sim)__cf_chl_jschl_tk__=([^"]+)', body)[0]
cloudflare_kwargs['params'] = dict(__cf_chl_jschl_tk__=token)
except (Exception, BaseException):
pass
if self.delay == self.default_delay:
try:
# no instantiated delay, therefore check js for hard coded CF delay
self.delay = float(
re.search(r'submit\(\);[^0-9]*?([0-9]+)',
body).group(1)) / float(1000)
except (BaseException, Exception):
pass
for i in re.findall(r'(<input[^>]+?hidden[^>]+?>)',
re.sub(r'(?sim)<!--\s+<input.*?(?=<)', '', body)):
value = re.findall(r'value="([^"\']+?)["\']', i)
name = re.findall(r'name="([^"\']+?)["\']', i)
if all([name, value]):
params[name[0]] = value[0]
js = self.extract_js(body, domain)
atob = (lambda s: b64decodestring('%s' % s))
try:
# Eval the challenge algorithm
params['jschl_answer'] = str(js2py.EvalJs({'atob': atob}).eval(js))
except (BaseException, Exception):
try:
params['jschl_answer'] = str(
js2py.EvalJs({
'atob': atob
}).eval(js))
except (BaseException, Exception) as e:
# Something is wrong with the page. This may indicate Cloudflare has changed their anti-bot technique.
raise ValueError(
'Unable to parse Cloudflare anti-bot IUAM page: %r' % e)
# Requests transforms any request into a GET after a redirect,
# so the redirect has to be handled manually here to allow for
# performing other types of requests even as the first request.
cloudflare_kwargs['allow_redirects'] = False
self.wait()
response = self.request(method, submit_url, **cloudflare_kwargs)
if response:
if 200 == getattr(response, 'status_code'):
return response
# legacy redirection handler (pre 2019.11.xx)
location = response.headers.get('Location')
try:
r = urlparse(location)
except (Exception, BaseException):
# Something is wrong with the page, perhaps CF changed their anti-bot technique
raise ValueError(
'Unable to find a new location from Cloudflare anti-bot IUAM page'
)
if not r.netloc or location.startswith('/'):
location = urlunparse((parsed_url.scheme, domain, r.path,
r.params, r.query, r.fragment))
return self.request(resp.request.method, location,
**original_kwargs)
def solve_challenge(self, body, domain):
try:
js = re.search(
r"setTimeout\(function\(\){\s+(var s,t,o,p,b,r,e,a,k,i,n,g,f.+?\r?\n[\s\S]+?a\.value =.+?)\r?\n",
body).group(1)
except Exception:
raise ValueError(
"Unable to identify Cloudflare IUAM Javascript on website. {}".
format(BUG_REPORT))
js = re.sub(r"a\.value = ((.+).toFixed\(10\))?", r"\1", js)
js = re.sub(r'(e\s=\sfunction\(s\)\s{.*?};)',
'',
js,
flags=re.DOTALL | re.MULTILINE)
js = re.sub(r"\s{3,}[a-z](?: = |\.).+", "",
js).replace("t.length", str(len(domain)))
js = js.replace('; 121', '')
# Strip characters that could be used to exit the string context
# These characters are not currently used in Cloudflare's arithmetic snippet
js = re.sub(r"[\n\\']", "", js)
if 'toFixed' not in js:
raise ValueError(
"Error parsing Cloudflare IUAM Javascript challenge. {}".
format(BUG_REPORT))
try:
jsEnv = """
var t = "{domain}";
var g = String.fromCharCode;
o = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=";
e = function(s) {{
s += "==".slice(2 - (s.length & 3));
var bm, r = "", r1, r2, i = 0;
for (; i < s.length;) {{
bm = o.indexOf(s.charAt(i++)) << 18 | o.indexOf(s.charAt(i++)) << 12 | (r1 = o.indexOf(s.charAt(i++))) << 6 | (r2 = o.indexOf(s.charAt(i++)));
r += r1 === 64 ? g(bm >> 16 & 255) : r2 === 64 ? g(bm >> 16 & 255, bm >> 8 & 255) : g(bm >> 16 & 255, bm >> 8 & 255, bm & 255);
}}
return r;
}};
function italics (str) {{ return '<i>' + this + '</i>'; }};
var document = {{
getElementById: function () {{
return {{'innerHTML': '{innerHTML}'}};
}}
}};
{js}
"""
innerHTML = re.search(
'<div(?: [^<>]*)? id="([^<>]*?)">([^<>]*?)<\/div>', body,
re.MULTILINE | re.DOTALL)
innerHTML = innerHTML.group(2).replace("'",
r"\'") if innerHTML else ""
js = jsunfuck(
jsEnv.format(domain=domain, innerHTML=innerHTML, js=js))
def atob(s):
return base64.b64decode('{}'.format(s)).decode('utf-8')
js2py.disable_pyimport()
context = js2py.EvalJs({'atob': atob})
result = context.eval(js)
except Exception:
logging.error(
"Error executing Cloudflare IUAM Javascript. {}".format(
BUG_REPORT))
raise
try:
float(result)
except Exception:
raise ValueError(
"Cloudflare IUAM challenge returned unexpected answer. {}".
format(BUG_REPORT))
return result
def findvideos(item):
logger.info()
itemlist = []
data = httptools.downloadpage(item.url).data
_sa = scrapertools.find_single_match(data, 'var _sa = (true|false);')
_sl = scrapertools.find_single_match(data, 'var _sl = ([^;]+);')
sl = eval(_sl)
buttons = scrapertools.find_multiple_matches(
data, '<button.*?class="selop" sl="([^"]+)">')
if not buttons:
buttons = [0, 1, 2]
for id in buttons:
title = '%s'
new_url = golink(int(id), _sa, sl)
data_new = httptools.downloadpage(new_url).data
matches = scrapertools.find_multiple_matches(
data_new, 'javascript">(.*?)</script>')
js = ""
for part in matches:
js += part
#logger.info("test before:" + js)
try:
matches = scrapertools.find_multiple_matches(
data_new, '" id="(.*?)" val="(.*?)"')
for zanga, val in matches:
js = js.replace(
'var %s = document.getElementById("%s");' % (zanga, zanga),
"")
js = js.replace('%s.getAttribute("val")' % zanga, '"%s"' % val)
#logger.info("test1 after:" +js)
except:
pass
#v1
js = re.sub('(document\[.*?)=', 'prem=', js)
#Parcheando a lo bruto v2
video = scrapertools.find_single_match(js,
"sources: \[\{src:(.*?), type")
js = re.sub(' videojs\((.*?)\);', video + ";", js)
import js2py
js2py.disable_pyimport()
context = js2py.EvalJs({'atob': atob})
try:
result = context.eval(js)
except:
logger.error("Js2Py no puede desofuscar el codigo, ¿cambió?")
continue
url = scrapertools.find_single_match(result, 'src="(.*?)"')
#v2
if not url:
url = result.strip()
itemlist.append(
Item(channel=item.channel,
title=title,
url=url,
action='play',
language='latino',
infoLabels=item.infoLabels))
itemlist = servertools.get_servers_itemlist(
itemlist, lambda i: i.title % i.server.capitalize())
# Requerido para FilterTools
itemlist = filtertools.get_links(itemlist, item, list_language)
# Requerido para AutoPlay
autoplay.start(itemlist, item)
return itemlist
def parse(url, c, ts):
try:
d = pq(common.visit(url))
# 旧版获取src
#src = d("video").find("source").attr("src")
# 新版获取src
src = d("#player_one script").text()
src = src[20:-8]
context = js2py.EvalJs()
js_code = ''';var encode_version = 'sojson.v5', lbbpm = '__0x33ad7', __0x33ad7=['QMOTw6XDtVE=','w5XDgsORw5LCuQ==','wojDrWTChFU=','dkdJACw=','w6zDpXDDvsKVwqA=','ZifCsh85fsKaXsOOWg==','RcOvw47DghzDuA==','w7siYTLCnw=='];(function(_0x94dee0,_0x4a3b74){var _0x588ae7=function(_0x32b32e){while(--_0x32b32e){_0x94dee0['push'](_0x94dee0['shift']());}};_0x588ae7(++_0x4a3b74);}(__0x33ad7,0x8f));var _0x5b60=function(_0x4d4456,_0x5a24e3){_0x4d4456=_0x4d4456-0x0;var _0xa82079=__0x33ad7[_0x4d4456];if(_0x5b60['initialized']===undefined){(function(){var _0xef6e0=typeof window!=='undefined'?window:typeof process==='object'&&typeof require==='function'&&typeof global==='object'?global:this;var _0x221728='ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=';_0xef6e0['atob']||(_0xef6e0['atob']=function(_0x4bb81e){var _0x1c1b59=String(_0x4bb81e)['replace'](/=+$/,'');for(var _0x5e3437=0x0,_0x2da204,_0x1f23f4,_0x3f19c1=0x0,_0x3fb8a7='';_0x1f23f4=_0x1c1b59['charAt'](_0x3f19c1++);~_0x1f23f4&&(_0x2da204=_0x5e3437%0x4?_0x2da204*0x40+_0x1f23f4:_0x1f23f4,_0x5e3437++%0x4)?_0x3fb8a7+=String['fromCharCode'](0xff&_0x2da204>>(-0x2*_0x5e3437&0x6)):0x0){_0x1f23f4=_0x221728['indexOf'](_0x1f23f4);}return _0x3fb8a7;});}());var _0x43712e=function(_0x2e9442,_0x305a3a){var _0x3702d8=[],_0x234ad1=0x0,_0xd45a92,_0x5a1bee='',_0x4a894e='';_0x2e9442=atob(_0x2e9442);for(var _0x67ab0e=0x0,_0x1753b1=_0x2e9442['length'];_0x67ab0e<_0x1753b1;_0x67ab0e++){_0x4a894e+='%'+('00'+_0x2e9442['charCodeAt'](_0x67ab0e)['toString'](0x10))['slice'](-0x2);}_0x2e9442=decodeURIComponent(_0x4a894e);for(var _0x246dd5=0x0;_0x246dd5<0x100;_0x246dd5++){_0x3702d8[_0x246dd5]=_0x246dd5;}for(_0x246dd5=0x0;_0x246dd5<0x100;_0x246dd5++){_0x234ad1=(_0x234ad1+_0x3702d8[_0x246dd5]+_0x305a3a['charCodeAt'](_0x246dd5%_0x305a3a['length']))%0x100;_0xd45a92=_0x3702d8[_0x246dd5];_0x3702d8[_0x246dd5]=_0x3702d8[_0x234ad1];_0x3702d8[_0x234ad1]=_0xd45a92;}_0x246dd5=0x0;_0x234ad1=0x0;for(var _0x39e824=0x0;_0x39e824<_0x2e9442['length'];_0x39e824++){_0x246dd5=(_0x246dd5+0x1)%0x100;_0x234ad1=(_0x234ad1+_0x3702d8[_0x246dd5])%0x100;_0xd45a92=_0x3702d8[_0x246dd5];_0x3702d8[_0x246dd5]=_0x3702d8[_0x234ad1];_0x3702d8[_0x234ad1]=_0xd45a92;_0x5a1bee+=String['fromCharCode'](_0x2e9442['charCodeAt'](_0x39e824)^_0x3702d8[(_0x3702d8[_0x246dd5]+_0x3702d8[_0x234ad1])%0x100]);}return _0x5a1bee;};_0x5b60['rc4']=_0x43712e;_0x5b60['data']={};_0x5b60['initialized']=!![];}var _0x4be5de=_0x5b60['data'][_0x4d4456];if(_0x4be5de===undefined){if(_0x5b60['once']===undefined){_0x5b60['once']=!![];}_0xa82079=_0x5b60['rc4'](_0xa82079,_0x5a24e3);_0x5b60['data'][_0x4d4456]=_0xa82079;}else{_0xa82079=_0x4be5de;}return _0xa82079;};if(typeof encode_version!=='undefined'&&encode_version==='sojson.v5'){function strencode(_0x50cb35,_0x1e821d){var _0x59f053={'MDWYS':'0|4|1|3|2','uyGXL':function _0x3726b1(_0x2b01e8,_0x53b357){return _0x2b01e8(_0x53b357);},'otDTt':function _0x4f6396(_0x33a2eb,_0x5aa7c9){return _0x33a2eb<_0x5aa7c9;},'tPPtN':function _0x3a63ea(_0x1546a9,_0x3fa992){return _0x1546a9%_0x3fa992;}};var _0xd6483c=_0x59f053[_0x5b60('0x0','cEiQ')][_0x5b60('0x1','&]Gi')]('|'),_0x1a3127=0x0;while(!![]){switch(_0xd6483c[_0x1a3127++]){case'0':_0x50cb35=_0x59f053[_0x5b60('0x2','ofbL')](atob,_0x50cb35);continue;case'1':code='';continue;case'2':return _0x59f053[_0x5b60('0x3','mLzQ')](atob,code);case'3':for(i=0x0;_0x59f053[_0x5b60('0x4','J2rX')](i,_0x50cb35[_0x5b60('0x5','Z(CX')]);i++){k=_0x59f053['tPPtN'](i,len);code+=String['fromCharCode'](_0x50cb35[_0x5b60('0x6','s4(u')](i)^_0x1e821d['charCodeAt'](k));}continue;case'4':len=_0x1e821d[_0x5b60('0x7','!Mys')];continue;}break;}}}else{alert('');};'''
context.execute(js_code)
src = context.eval(src)
src = pq(src)
src = src("source").attr("src")
# 新版获取src
if src != None:
m = common.visit(url)
soup = BeautifulSoup(m, "lxml")
title = str(soup.title.string)
title = title.replace(' ', "")
title = title.replace('/', "")
title = title.replace("\\", "")
title = title.replace(",", "")
title = title.replace("，", "")
title = title.replace("。", "")
title = title.replace(".", "")
title = title.replace("\t", "")
title = title.replace("\n", "")
title = title.replace("\r", "")
title = title.replace("'", "")
title = title.replace("\"", "")
title = title.replace("‘", "")
title = title.replace("", "")
title = title.replace(" ", "")
title = title.replace("?", "")
title = title.replace("？", "")
if len(title) > 15:
title = title[0:15]
con = soup.find(name="div", attrs={"class": "boxPart"}).text
con = "".join(con.split())
t = con.split(":")
times = int(t[1])
ts = int(ts)
if times >= ts:
if '//mp' in src:
src = src.replace('//mp', '/mp')
print(threading.current_thread().name, " insert into redis ",
src)
src_title_map = {'src': src, 'title': title}
if not redisutil.exists(str(src_title_map),
common.KEY_ALREADY_DOWNLOAD):
redisutil.add(str(src_title_map), common.KEY_SRC)
else:
print('该url已下载' + src)
else:
print(threading.current_thread().name, src, "Not enough time")
else:
print(threading.current_thread().name, src,
"解析为None, 插入 redis_error")
# redisutil.add(src, common.KEY_NONE)
except Exception as e:
print(
e
) #'http://23.225.233.3//mp43/366445.mp4?st=ZADNhKoTVFRQwxKs2BW5XA&e=1587895189'
print('src----visit--error')
def push_cookie_to_redis(self):
while True:
try:
"""
2. 实现一个方法, 用于把一套代理IP, User-Agent, Cookie绑定在一起的信息放到Redis的list中
"""
# 1. 随机获取一个User-Agent
user_agent = random.choice(USER_AGENTS)
# 2. 随机获取一个代理IP
response = requests.get(
'http://localhost:16888/random?protocol=http')
proxy = response.content.decode()
# 3. 获取requests的session对象
session = requests.session()
# 4. 把User-Agent, 通过请求头, 设置给session对象
session.headers = {'User-Agent': user_agent}
# 5. 把代理IP, 通过proxies, 设置给session对象
session.proxies = {'http': proxy}
# 6. 使用session对象, 发送请求, 获取需要的cookie信息
index_url = 'http://www.gsxt.gov.cn/corp-query-entprise-info-xxgg-100000.html'
# 使用session发送请求
response = session.get(index_url)
print(response.status_code)
# print(response.content.decode())
# 1. 提取script标签中的js
js = re.findall('<script>(.+?)</script>',
response.content.decode())[0]
# print(js)
# 2. 由于这种加密js, 最终指向的js代码, 都是在eval函数中的, 所以 `{eval(` 替换为 {code=(, 然后就可以通过code, 获取真正要执行的js了
js = js.replace('{eval(', '{code=(')
# print(js)
# 3. 获取要执行的js
# 3.1 获取执行js的环境
context = js2py.EvalJs()
context.execute(js)
# 打印code的值
# print(context.code)
# 获取生成Cookie的js
cookie_code = re.findall("document.(cookie=.+)\+';Expires",
context.code)[0]
# print(cookie_code)
# 在js2py中, 是不能使用 `document`, 'window' 这些浏览器中对象
# var _1k=document.createElement('div');_1k.innerHTML='<a href=\'/\'>_D</a>';_1k=_1k.firstChild.href;
# js2py是无法处理.
# `var _31=document.createElement('div');_31.innerHTML='<a href=\'/\'>_8</a>';_31=_31.firstChild.href`
# 替换为
# _31='http://www.gsxt.gov.cn'
# cookie_code = re.sub(r"var\s+(\w+)=document.createElement\('\w+'\);\w+.innerHTML='<a href=\\'/\\'>\w+</a>';\w+=\w+.firstChild.href", r"var \1='http://www.gsxt.gov.cn'", cookie_code)
cookie_code = re.sub(r"var\s+(\w+)=document.+?firstChild.href",
r"var \1='http://www.gsxt.gov.cn'",
cookie_code)
# print(cookie_code)
# 执行js, 生成我们需要cookie信息
context.execute(cookie_code)
# 打印cookie信息
# print(context.cookie)
cookies = context.cookie.split('=')
session.cookies.set(cookies[0], cookies[1])
session.get(index_url)
# print(session.cookies)
# 获取cookie字典
cookies = requests.utils.dict_from_cookiejar(session.cookies)
# print(cookies)
# 7. 把代理IP, User-Agent, Cookie放到字典中, 序列化后, 存储到Redis的list中
cookies_dict = {
COOKIES_KEY: cookies,
COOKIES_USER_AGENT_KEY: user_agent,
COOKIES_PROXY_KEY: proxy
}
# 序列化后, 存储到Redis的list中
self.redis.lpush(REDIS_COOKIES_KEY, pickle.dumps(cookies_dict))
print(cookies_dict)
break
except Exception as ex:
print(ex)
def update_property_with_document_and_action(flow_case, document,
subs_action):
from antares.apps.document.models.document_header import DocumentHeader
from antares.apps.document.types.document import Document
if isinstance(document, DocumentHeader):
document_object = Document(document_id=document.id)
else:
document_object = document
# TODO: adapt code to work with python too, based on XPDL, for
# now, only JS will work
context = js2py.EvalJs({
'document': document_object,
'flow_case': flow_case
})
for param_def in subs_action.parameter_set.select_related().all():
if param_def.content_text is not None:
if (param_def.parameter_name and
param_def.parameter_name.lower() == '_flow_client_id'):
context.execute('result_value = ' + param_def.content_text)
if (hasattr(context, 'result_value')
and context.result_value is not None):
client = Client.find_one(
uuid.UUID(context.result_value))
if client is not None:
flow_case.client = client
elif (param_def.parameter_name and
param_def.parameter_name.lower() == '_flow_priority'):
context.execute('result_value = ' + param_def.content_text)
if hasattr(context, 'result_value'):
flow_case.priority = context.result_value
elif (param_def.parameter_name and
param_def.parameter_name.lower() == '_flow_case_name'):
context.execute('result_value = ' + param_def.content_text)
if hasattr(context, 'result_value'):
flow_case.case_name = context.result_value
else:
try:
prop_def = flow_case.property_set.select_related().get(
property_id=param_def.parameter_name)
except FlowProperty.DoesNotExist:
prop_def = None
if prop_def is not None:
context.execute('result_value =' +
param_def.content_text)
if (hasattr(context, 'result_value')
and prop_def.data_type == FlowDataType.BASIC
and context.result_value is not None):
if (prop_def.sub_data_type ==
FlowBasicDataSubtype.STRING):
prop_def.string_value = context.result_value
elif (prop_def.sub_data_type ==
FlowBasicDataSubtype.TEXT):
prop_def.text_value = context.result_value
elif (prop_def.sub_data_type ==
FlowBasicDataSubtype.DATE):
prop_def.date_value = context.result_value
elif (prop_def.sub_data_type ==
FlowBasicDataSubtype.INTEGER):
prop_def.integer_value = int(
float(context.result_value))
elif (prop_def.sub_data_type ==
FlowBasicDataSubtype.FLOAT):
prop_def.decimal_value = float(
context.result_value)
else:
raise NotImplementedError(
_(__name__ + ".type_not_implemented_yet"))
prop_def.save()
def main(mytimer: func.TimerRequest) -> None:
utc_timestamp = datetime.datetime.utcnow().replace(
tzinfo=datetime.timezone.utc).isoformat()
if mytimer.past_due:
logging.info('The timer is past due!')
logging.info('Python timer trigger function ran at %s', utc_timestamp)
# const
username = os.environ['NJUA_USER']
password = os.environ['NJUA_PASS']
url_encrypt = r'https://authserver.nju.edu.cn/authserver/custom/js/encrypt.js'
url_login = r'https://authserver.nju.edu.cn/authserver/login'
url_list = r'http://ehallapp.nju.edu.cn/xgfw/sys/yqfxmrjkdkappnju/apply/getApplyInfoList.do'
url_apply = r'http://ehallapp.nju.edu.cn/xgfw/sys/yqfxmrjkdkappnju/apply/saveApplyInfos.do'
session = requests.Session()
# encryption
response = session.get(url_encrypt)
context = js2py.EvalJs()
context.execute(response.text)
if response.status_code == 200:
logging.info('Load encryption: %d, %s' %
(response.status_code, response.reason))
else:
logging.error('Load encryption: %d, %s' %
(response.status_code, response.reason))
return
# login
response = session.get(url_login)
if response.status_code == 200:
logging.info('Open login page: %d, %s' %
(response.status_code, response.reason))
else:
logging.error('Open login page: %d, %s' %
(response.status_code, response.reason))
return
soup = BeautifulSoup(response.text, 'html.parser')
data_login = {
'username':
username,
'password':
context.encryptAES(
password,
soup.select_one(
"#casLoginForm #pwdDefaultEncryptSalt").attrs['value']),
'lt':
soup.select_one('#casLoginForm [name="lt"]').attrs['value'],
'dllt':
soup.select_one('#casLoginForm [name="dllt"]').attrs['value'],
'execution':
soup.select_one('#casLoginForm [name="execution"]').attrs['value'],
'_eventId':
soup.select_one('#casLoginForm [name="_eventId"]').attrs['value'],
'rmShown':
soup.select_one('#casLoginForm [name="rmShown"]').attrs['value'],
}
response = session.post(url_login, data_login)
if response.status_code == 200:
logging.info('Login for %s: %d, %s' %
(username, response.status_code, response.reason))
else:
logging.error('Login for %s: %d, %s' %
(username, response.status_code, response.reason))
return
# list
response = session.get(url_list)
try:
content = response.json()
except ValueError:
content = {}
if response.status_code == 200 and content.get('code') == '0':
logging.info('List: %d, %s, %s' %
(response.status_code, response.reason, content.get('msg')
or 'No messgage available'))
else:
logging.error('List: %d, %s, %s' %
(response.status_code, response.reason,
content.get('msg') or 'No messgage available'))
return
# apply
data = next(x for x in content['data'] if x.get('TJSJ') != '')
data_apply = {
'WID': content['data'][0]['WID'],
'CURR_LOCATION': data['CURR_LOCATION'],
'IS_TWZC': data['IS_TWZC'],
'IS_HAS_JKQK': data['IS_HAS_JKQK'],
'JRSKMYS': data['JRSKMYS'],
'JZRJRSKMYS': data['JZRJRSKMYS']
}
response = session.get(url_apply, params=data_apply)
try:
content = response.json()
except ValueError:
content = {}
if response.status_code == 200 and content.get('code') == '0':
logging.info('Apply: %d, %s, %s' %
(response.status_code, response.reason, content.get('msg')
or 'No messgage available'))
else:
logging.error('Apply: %d, %s, %s' %
(response.status_code, response.reason,
content.get('msg') or 'No messgage available'))
return
pass
def run(self):
resp_by_id = {}
compare_jobs = []
for req in self._requests:
response_time = -1 # time between sending the request and finishing parsing the entire response
latency = -1 # time between sending the request and finishing parsing the response headers
try:
resp, response_time = self._execute_request(req)
latency = resp.elapsed.total_seconds() * 100
except RequestException as e:
self.callback.failure(
TestFailure(
request=req,
response_time=-1,
error=f'Error issuing the request, root cause: {str(e)}'
))
continue
if req.id:
resp_by_id[req.id] = resp
if req.tail.response_handler:
h = req.tail.response_handler
if h.expected_status_code and resp.status_code != h.expected_status_code:
self.callback.failure(
TestFailure(
request=req,
response_time=response_time,
error=
f'Expected status code {h.expected_status_code},'
f' but got {resp.status_code}'))
break
# Supporting loading scripts from paths, e.g. `> scripts/my-my-script.js`
response_handler_script = None
if h.path:
if h.path not in self.response_handler_scripts:
with open(self.base_dir + '/' + h.path,
'r',
encoding='utf-8') as file:
self.response_handler_scripts[h.path] = file.read()
response_handler_script = self.response_handler_scripts[
h.path]
elif h.script:
response_handler_script = h.script
if response_handler_script:
# Response handler script should be written in JavaScript ECMAScript 5.1 specification.
# See examples in
# https://www.jetbrains.com/help/webstorm/http-response-handling-examples.html#script-var-example
# TODO support a python variant
context = js2py.EvalJs({
'client': HttpClient(),
'response': HttpResponse(resp),
})
try:
context.execute(response_handler_script)
except js2py.internals.simplex.JsException as e:
self.callback.failure(
TestFailure(
request=req,
response_time=response_time,
error=str(e).replace(
'Error: your Python function failed! ',
'')))
if req.tail.response_ref:
compare_jobs.append(
CompareJob(req=req,
resp=resp,
request_id=req.tail.response_ref.path))
self.callback.success(
TestSuccess(request=req,
latency=latency,
response_time=response_time))
for job in compare_jobs:
cmp_resp = resp_by_id.get(job.request_id)
if cmp_resp is None:
req_ids = list(resp_by_id.keys())
error = 'Request with id "{}" not found. Defined requests: {}'.format(
job.request_id, req_ids)
self.callback.failure(
TestFailure(
request=job.req,
response_time=job.resp.elapsed.total_seconds() * 100,
error=error))
continue
diff = self._diff_responses(job.resp, cmp_resp)
if diff:
self.callback.failure(
TestFailure(
request=job.req,
response_time=job.resp.elapsed.total_seconds() * 100,
error=diff))
return self.callback
def __init__(self, resp_text):
self.resp_text = resp_text
self.context = js2py.EvalJs()
self.log = logger()
def GetSign(content):
run_js = js2py.EvalJs({})
run_js.execute(GetJs())
return run_js.e(content)
