def get_response(url, plugins):
""" Return response with har and detected software. """
lua_script = create_lua_script(plugins)
lua = urllib.parse.quote_plus(lua_script)
page_url = '{}/execute?url={}&lua_source={}'.format(SPLASH_URL, url, lua)
try:
with docker_container():
logger.debug('[+] Sending request to Splash instance')
res = requests.get(page_url)
except DockerStartError:
logger.error(
"There was an error running splash container. "
"It's possible that previous splash container didn't finish well, "
"please verify and stop any other splash instance to avoid port issues."
)
sys.exit(0)
logger.debug('[+] Response received')
json_data = res.json()
if res.status_code in ERROR_STATUS_CODES:
raise SplashError(json_data['description'])
softwares = json_data['softwares']
har = get_valid_har(json_data['har'])
logger.debug('[+] Detected %(n)d softwares', {'n': len(softwares)})
logger.debug('[+] Final HAR has %(n)d valid entries', {'n': len(har)})
return {'har': har, 'softwares': softwares}
def get_response(url, plugins, timeout=SPLASH_TIMEOUT):
"""
Return response with HAR, inline scritps and software detected by JS matchers.
:rtype: dict
"""
lua_script = create_lua_script(plugins)
lua = urllib.parse.quote_plus(lua_script)
page_url = f"{SPLASH_URL}/execute?url={url}&timeout={timeout}&lua_source={lua}"
try:
with docker_container():
logger.debug("[+] Sending request to Splash instance ..")
# Force timeout in the request handler too
res = requests.get(page_url, timeout=timeout)
except requests.exceptions.ConnectionError:
raise SplashError(
"Could not connect to Splash server {}".format(SPLASH_URL))
except requests.exceptions.ReadTimeout:
raise SplashError("Connection to Splash server timed out")
logger.debug("[+] Response received")
json_data = res.json()
if res.status_code in ERROR_STATUS_CODES:
raise SplashError(get_splash_error(json_data))
softwares = json_data["softwares"]
scripts = json_data["scripts"].values()
har = get_valid_har(json_data["har"])
js_error = get_evaljs_error(json_data)
if js_error:
logger.warning(f"[-] Failed to eval JS matchers: {js_error}")
else:
logger.debug("[+] Detected %(n)d softwares from the DOM",
{"n": len(softwares)})
logger.debug("[+] Detected %(n)d scripts from the DOM",
{"n": len(scripts)})
logger.debug("[+] Final HAR has %(n)d valid entries", {"n": len(har)})
return {"har": har, "scripts": scripts, "softwares": softwares}
def get_response(url, plugins, timeout=SPLASH_TIMEOUT):
"""
Return response with HAR, inline scritps and software detected by JS matchers.
:rtype: dict
"""
lua_script = create_lua_script(plugins)
lua = urllib.parse.quote_plus(lua_script)
page_url = f'{SPLASH_URL}/execute?url={url}&timeout={timeout}&lua_source={lua}'
try:
with docker_container():
logger.debug('[+] Sending request to Splash instance')
res = requests.get(page_url)
except requests.exceptions.ConnectionError:
raise SplashError(
"Could not connect to Splash server {}".format(SPLASH_URL))
logger.debug('[+] Response received')
json_data = res.json()
if res.status_code in ERROR_STATUS_CODES:
raise SplashError(get_splash_error(json_data))
softwares = json_data['softwares']
scripts = json_data['scripts'].values()
har = get_valid_har(json_data['har'])
js_error = get_evaljs_error(json_data)
if js_error:
logger.debug('[+] WARNING: failed to eval JS matchers: %(n)s',
{'n': js_error})
else:
logger.debug('[+] Detected %(n)d softwares from the DOM',
{'n': len(softwares)})
logger.debug('[+] Detected %(n)d scripts from the DOM',
{'n': len(scripts)})
logger.debug('[+] Final HAR has %(n)d valid entries', {'n': len(har)})
return {'har': har, 'scripts': scripts, 'softwares': softwares}