def solve_cf_challenge(self, resp, headers, **kwargs):
headers = headers.copy()
url = resp.url
parsed = urlparse(url)
domain = parsed.netloc
page = resp.content
kwargs.pop("params", None) # Don't pass on params
try:
# Extract the arithmetic operation
challenge = re.search(r'name="jschl_vc" value="(\w+)"', page).group(1)
builder = re.search(r"setTimeout.+?\r?\n([\s\S]+?a\.value =.+?)\r?\n", page).group(1)
builder = re.sub(r"a\.value =(.+?) \+ .+?;", r"\1", builder)
builder = re.sub(r"\s{3,}[a-z](?: = |\.).+", "", builder)
except AttributeError:
# Something is wrong with the page. This may indicate Cloudflare has changed their
# anti-bot technique. If you see this and are running the latest version,
# please open a GitHub issue so I can update the code accordingly.
raise IOError("Unable to parse Cloudflare anti-bots page. Try upgrading cfscrape, or "
"submit a bug report if you are running the latest version.")
# Lock must be added explicitly, because PyV8 bypasses the GIL
with PyV8.JSLocker():
with PyV8.JSContext() as ctxt:
# Safely evaluate the Javascript expression
answer = str(int(ctxt.eval(builder)) + len(domain))
params = {"jschl_vc": challenge, "jschl_answer": answer}
submit_url = "%s://%s/cdn-cgi/l/chk_jschl" % (parsed.scheme, domain)
headers["Referer"] = url
return requests.get(submit_url, params=params, headers=headers, **kwargs)
def get_javascript_tests_result(self, tests_lines=None, json_obj=None):
"""
Summary:
对js编写的用例进行断言
Args:
tests_lines: 测试语句,用分号分隔
:return:
"""
json_obj = json_obj or self.get_json_response_obj()
if json_obj is None:
return None
tests_lines = tests_lines or self.get_case_tests()
tests_lines = tests_lines.encode('utf8') if isinstance(
tests_lines, unicode) else tests_lines
js_str = """
(function(jsbody){
var responseBody = jsbody;
var tests = new Object();""" + tests_lines + """
return tests
})
"""
test_dic = dict()
with PyV8.JSLocker():
js_context = PyV8.JSContext()
js_context.enter()
test_func = js_context.eval(js_str.decode('utf8'))
test_jsobj = test_func(json_obj)
for key in test_jsobj.keys():
test_dic[key] = test_jsobj[key]
js_context.leave()
return test_dic
def __init__(self):
if HAS_PYV8:
with PyV8.JSLocker():
self.ctx = PyV8.JSContext(GlobalContext())
self.ctx.enter()
self.init()
self.ctx.leave()
def __run(self, window):
if log.Trace:
sys.settrace(log.Trace)
with PyV8.JSLocker():
with Watchdog(log.ThugOpts.timeout, callback = self.watchdog_cb):
dft = DFT(window)
dft.run()
def execute(my, js, kwargs={}):
with PyV8.JSLocker():
my.ctx.enter()
for name, value in kwargs.items():
my.ctx.locals[name] = value
data = my.ctx.eval(js)
my.ctx.leave()
return data
def energy(self, msg):
with PyV8.JSLocker():
with PyV8.JSContext() as ctex:
with open("deal.js") as jsfile:
ctex.eval(jsfile.read())
encrypt_pwd = ctex.eval('encryptString("%s","%s","%s")' %
(msg, self.e, self.m))
print encrypt_pwd
return encrypt_pwd
def tes():
with PyV8.JSLocker():
ctxt.enter()
func = ctxt.eval(
"""(function(){function hello(){return "Hello world.";}return hello();})"""
)
print(func())
ctxt.leave()
PyV8.JSUnlocker()
PyV8.JSEngine.collect()
def on_actionStart_triggered(self):
log.debug('start')
dlg = QueryDialog(self)
rc = dlg.exec_()
if not rc:
return
links = dlg.get_links()
model = self.table_tv.model()
workers = []
thc, rc = startup.CFG.value("prefs/threads", 5).toInt()
counter = {'current': 0, 'total': 0}
queue = Queue.Queue()
for l in links:
queue.put(Query(l))
counter['total'] += 1
pdlg = MyProgressDialog(u'Обработка', u'Подключение', u'Отмена', parent=self)
def closed():
log.debug('closed')
for w in workers:
#if w.isRunning():
w.done.disconnect()
w.add.disconnect()
w.active = False
transaction.commit()
from_ = model.index(0, 0)
to_ = model.index(len(model._data)-1, len(model.labels)-1)
model.dataChanged.emit(from_, to_)
model.layoutChanged.emit()
log.debug('counter %s', counter)
def add(t):
pdlg.set_text(t)
counter['total'] += 1
log.debug('counter %s', counter)
def done(t):
pdlg.set_text(t)
counter['current'] += 1
if counter['total'] <= counter['current']:
pdlg.close()
log.debug('counter %s', counter)
pdlg.closed.connect(closed)
with PyV8.JSLocker():
log.debug('total threads %i', thc)
for i in range(thc):
w = Worker(queue, self)
w.add.connect(add)
w.done.connect(done)
workers.append(w)
w.start()
pdlg.show()
def jw(code):
if not _jw:
print "No active window object, use jsrun first"
return
k = PyV8.JSLocker()
k.enter()
_jw.context.enter()
retval = eval("_jw.{}".format(code))
_jw.context.leave()
k.leave()
return retval
def dxPwdEncrypt(pwd):
with PyV8.JSLocker():
with PyV8.JSContext() as ctxt:
with open(basePath + "dx_encrypt.js", 'r') as f:
js = f.read()
js += '\n'
ctxt.eval(js)
#
encryptPwd = ctxt.eval("valAesEncryptSet('%s')" % pwd)
#
return encryptPwd
def execute(self, js, kwargs={}):
if HAS_PYV8:
with PyV8.JSLocker():
self.ctx.enter()
try:
for name, value in kwargs.items():
self.ctx.locals[name] = value
data = self.ctx.eval(js)
finally:
self.ctx.leave()
return data
def executeJS(self, js_func_string, arg):
'''
self.ctxt.enter()
func = self.ctxt.eval("({js})".format(js=js_func_string))
return func(arg)
'''
ctxt = PyV8.JSContext()
with PyV8.JSLocker():
ctxt.enter()
vl5x = ctxt.eval("({js})".format(js=js_func_string))
sign = vl5x(arg)
ctxt.leave()
return sign
def __parse(self, tvid, vid, uid, bid):
js_context = ''
with open("ArrayBuffer.js", 'r') as f:
js_context += f.read()
with open("pcweb.js", 'r') as f:
js_context += f.read()
time_str = str(int(time.time() * 1000))
ctx = PyV8.JSContext()
with PyV8.JSLocker():
ctx.enter()
ctx.eval(js_context)
authkey = ctx.locals.authkey(
ctx.locals.authkey('') + time_str + tvid)
callback = ctx.locals.callback()
params = {
'tvid': tvid,
'vid': vid,
'bid': str(bid),
'tm': time_str,
'k_uid': uid,
'callback': callback,
'authKey': authkey,
}
global_params.update(params)
params_encode = urllib.urlencode(global_params)
path_get = '/jp/dash?' + params_encode
vf = ctx.locals.vf(path_get)
path_get += '&vf=%s' % vf
ctx.leave()
req = urllib2.Request('http://cache.video.iqiyi.com/' +
path_get.lstrip('/'))
res = self.opener.open(req)
raw = res.read()
text = raw_decompress(raw, res.info())
json_str = re.search('try{\w{0,}\((.+})(\s)?\);}catch', text).group(1)
if json_str:
ret = json.loads(json_str)
else:
ret = None
res.close()
return ret
def jseval(code):
global _jw
if not _jw:
ctxt = PyV8.JSContext()
ctxt.enter()
retval = ctxt.eval(code)
ctxt.leave()
else:
k = PyV8.JSLocker()
k.enter()
_jw.context.enter()
retval = _jw.context.eval(code)
_jw.context.leave()
k.leave()
return retval
def killCallback():
src = frame.entrySrc.get().strip() # the "auto detect" string
dst = frame.entryDst.get().strip() # the domain
authInterval = float(frame.entryAuthInterval.get().strip())
numConn = int(frame.spinboxNumConn.get().strip())
numRqst = int(frame.spinboxNumRqst.get().strip())
connInterval = float(frame.entryConnInterval.get().strip())
rqstInterval = float(frame.entryRqstInterval.get().strip())
holdBefore = float(frame.entryHoldBefore.get().strip())
holdAfter = float(frame.entryHoldAfter.get().strip())
customHeader = frame.entryCustomHeader.get().strip()
cookieField = frame.entryCookieField.get().strip()
jsLock = threading.Lock()
if hasattr(PyV8, 'JSLocker'):
jsLock = PyV8.JSLocker()
jsLock.acquire = jsLock.enter
jsLock.release = jsLock.leave
_threadResults.update({
"Fill": "grey",
"Host": dst,
"Path": "/",
"Referer": "",
"Cookie": "",
})
THREAD_EVENT.clear()
frame.labelStatus.configure(text=" ")
frame.buttonKill.configure(text="Stop!", command=stopCallback)
authEvent = threading.Event()
threading.Thread(name="Thread-0",
target=authThread,
args=(0, THREAD_EVENT, authEvent, authInterval,
tkRedirect.get() == 1, tkCookie.get() == 1,
tkJavascript.get() == 1, tkCaptcha.get() == 1, dst,
cookieField, jsLock)).start()
for i in range(1, numConn + 1):
threading.Thread(target=killThread,
args=(i, THREAD_EVENT, authEvent, src, dst, numRqst,
connInterval, rqstInterval, holdBefore,
holdAfter, customHeader, cookieField)).start()
def analyseJS(code, context=None, manualAnalysis=False):
'''
Hooks the eval function and search for obfuscated elements in the Javascript code
@param code: The Javascript code (string)
@return: List with analysis information of the Javascript code: [JSCode,unescapedBytes,urlsFound,errors,context], where
JSCode is a list with the several stages Javascript code,
unescapedBytes is a list with the parameters of unescape functions,
urlsFound is a list with the URLs found in the unescaped bytes,
errors is a list of errors,
context is the context of execution of the Javascript code.
'''
errors = []
JSCode = []
unescapedBytes = []
urlsFound = []
try:
code = unescapeHTMLEntities(code)
scriptElements = re.findall(reJSscript, code,
re.DOTALL | re.IGNORECASE)
if scriptElements != []:
code = ''
for scriptElement in scriptElements:
code += scriptElement + '\n\n'
code = jsbeautifier.beautify(code)
JSCode.append(code)
if code != None and JS_MODULE and not manualAnalysis:
if context == None:
with PyV8.JSLocker():
context = PyV8.JSContext(Global())
with PyV8.JSLocker():
context.enter()
# Hooking the eval function
context.eval('eval=evalOverride')
#context.eval(preDefinedCode)
while True:
originalCode = code
try:
context.eval(code)
evalCode = context.eval('evalCode')
evalCode = jsbeautifier.beautify(evalCode)
if evalCode != '' and evalCode != code:
code = evalCode
JSCode.append(code)
else:
break
except:
error = str(sys.exc_info()[1])
open('jserror.log', 'ab').write(error + newLine)
errors.append(error)
break
context.leave()
if code != '':
escapedVars = re.findall('(\w*?)\s*?=\s*?(unescape\((.*?)\))',
code, re.DOTALL)
for var in escapedVars:
bytes = var[2]
if bytes.find('+') != -1 or bytes.find('%') == -1:
varContent = getVarContent(code, bytes)
if len(varContent) > 150:
ret = unescape(varContent)
if ret[0] != -1:
bytes = ret[1]
urls = re.findall('https?://.*$', bytes,
re.DOTALL)
if bytes not in unescapedBytes:
unescapedBytes.append(bytes)
for url in urls:
if url not in urlsFound:
urlsFound.append(url)
else:
bytes = bytes[1:-1]
if len(bytes) > 150:
ret = unescape(bytes)
if ret[0] != -1:
bytes = ret[1]
urls = re.findall('https?://.*$', bytes,
re.DOTALL)
if bytes not in unescapedBytes:
unescapedBytes.append(bytes)
for url in urls:
if url not in urlsFound:
urlsFound.append(url)
except:
traceback.print_exc(file=open(errorsFile, 'a'))
errors.append('Unexpected error in the JSAnalysis module!!')
finally:
for js in JSCode:
if js == None or js == '':
JSCode.remove(js)
return [JSCode, unescapedBytes, urlsFound, errors, context]
def run(self, window):
with PyV8.JSLocker():
dft = DFT.DFT(window)
dft.run()
def eval(self):
with PyV8.JSLocker():
self.start()
self.join(10)
return self.result
#!/usr/bin/env python2
# -*- coding: utf-8 -*-
import PyV8
with PyV8.JSLocker():
ctxt = PyV8.JSContext()
def tes():
with PyV8.JSLocker():
ctxt.enter()
func = ctxt.eval(
"""(function(){function hello(){return "Hello world.";}return hello();})"""
)
print(func())
ctxt.leave()
PyV8.JSUnlocker()
PyV8.JSEngine.collect()
if __name__ == '__main__':
tes()
def run(self,
func=None,
fargs=[],
precompil_only=False,
compil_only=False):
"""
Run JS code with libraries and return last operand result or
`func` result if it present
:param str func: (optional) JS function name
:param list fargs: (optional) list of JS function args
:param bool precompil_only: (optional) if true, then it will
return precommiled data all JS code
:param bool compil_only: (optional) if true, then it
will compile the JS code and throw errors if they were
:raise SyntaxError: JS syntax error
:raise PyV8.JSError: JS runtime error (contain fields:
name, message, scriptName, lineNum, startPos, endPos,
startCol, endCol, sourceLine, stackTrace)
"""
precompiled = OrderedDict()
compiled = OrderedDict()
with PyV8.JSLocker():
with PyV8.JSContext() as js_context:
self.logger.debug('Set JS global context class attributes')
for k, v in self.js_global_vars.items():
self.logger.debug('Set attribute name=%s, value=%s' %
(k, v))
# Convert to JS objects
setattr(js_context.locals, k,
self._get_js_obj(js_context, v))
with PyV8.JSEngine() as engine:
precompil_error = False
try:
for js_lib, js_code in self.js_libs_code.items():
self.logger.debug('Precompile JS lib: %s' % js_lib)
precompiled[js_lib] = engine.precompile(js_code)
except SyntaxError:
precompil_error = True
if not precompil_error and precompil_only:
return precompiled
for js_lib, js_code in self.js_libs_code.items():
self.logger.debug('Compile JS lib: %s' % js_lib)
cparams = dict(source=self.js_libs_code[js_lib],
name=js_lib)
if js_lib in precompiled:
cparams['precompiled'] = precompiled[js_lib]
compiled[js_lib] = engine.compile(**cparams)
if compil_only:
return True
result = None
for js_lib, js_script in compiled.items():
self.logger.debug('Run JS lib: %s' % js_lib)
result = js_script.run()
if not func or type(func) != str:
return result
if fargs and not isinstance(fargs, (list, tuple)):
raise ArgumentError(
'The "fargs" must be list or tuple')
if func not in js_context.locals:
raise JSFunctionNotExists(
'Function "%s" not exists in JS context' % func)
# Convert to JS objects
for i in range(len(fargs)):
fargs[i] = self._get_js_obj(js_context, fargs[i])
# Convert to Python objects
return self._get_py_obj(js_context,
js_context.locals[func](*fargs))
def __init__(my):
with PyV8.JSLocker():
my.ctx = PyV8.JSContext(GlobalContext())
my.ctx.enter()
my.init()
my.ctx.leave()
# s = 'code='+s[4:]
s = s[8:-9]
# print type(s), s
# win.evalScript(s)
with win.context as ctxt:
ctxt.eval(s)
print '****$$$$$$$******', win.location.href
url = win.location.href
r = {'go_url': url}
l = set()
l.add(url)
return r, l
pyv8_jslocker = PyV8.JSLocker()
def parse_go_url(nodes, name, values, item):
print '+++go url', nodes
if not nodes or len(nodes) < 1:
return
node = nodes[0]
jscode = node.text_content()
# print 'js content' ,jscode
# print type(jscode)
jscode = u'code='+jscode[4:]
# print type(jscode),jscode
url = None
def eval_pyv8(self, script):
with PyV8.JSLocker():
with PyV8.JSContext() as rt:
return rt.eval(script)
