async def manual_submit(request, user):
if request.method == 'GET':
return render_template('manual_submit.html',
user=user,
autoResults=json.dumps(data))
else:
return json_resp(
{'list_id': BestsellerList.from_json(request.json, user).id})
def quit_response(session_id, error=False):
resp = {
"sessionId": session_id,
"state": "error" if error else "success",
"hCode": 225874638,
"value": None,
"class": "org.openqa.selenium.remote.Response",
"status": 0
}
status = 500 if error else 200
if error:
resp['message'] = ''
return json_resp(resp, status=status)
async def query_driver(request, driver_url):
if '//' in driver_url:
driver_url = driver_url.replace('//', '/')
request_type = driver_request_type(driver_url, request.method)
if request_type == NEW_DRIVER:
# old: reuse_session = json.loads(body_str).get('reuse_session')
body_str = request.body.decode()
success, new_created, driver_dict = await app_logic.launch_driver(
body_str)
if not success:
return json_resp(driver_dict['creation_resp_json'], status=500)
return json_resp(driver_dict['creation_resp_json'], 200)
elif request_type == QUIT_COMMAND:
selenium_id = get_session_id(driver_url, request.body.decode())
await app_logic.quit_driver(selenium_id)
return quit_response(selenium_id)
return await app_logic.proxy_selenium_request(request, driver_url)
async def proxy_selenium_request(self, request, driver_url):
body_str = request.body.decode()
selenium_id = get_session_id(driver_url, body_str)
container = self.drivers[selenium_id]['container']
request_session = self.drivers[selenium_id]['requests_session']
if random.randint(
0,
10) == 9: # for efficiency, we do this 1 in every 10 requests
self.drivers['last_command_time'] = datetime.datetime.now()
url = 'http://' + container + ':' + PORT + '/' + driver_url
status_code, resp_json = await do_selenium_request_async(
request, request_session, url)
if status_code != 200:
print('warning: selenium request gave status: %s' % status_code)
return json_resp(resp_json, status=status_code)
def _new_driver_resp_error(session_id):
resp = {
"value": {
"stacktrace": "org.openqa.selenium.WebDriverException: Process unexpectedly closed with status: 1\nBuild info: version: '3.6.0', revision: '6fbf3ec767', time: '2017-09-27T16:15:40.131Z'\nSystem info: host: '1bab8fee6817', ip: '172.16.0.174', os.name: 'Linux', os.arch: 'amd64', os.version: '4.4.62-hyper', java.version: '1.8.0_131'\nDriver info: driver.version: unknown\nremote stacktrace: stack backtrace:\n 0: 0x4f388c - backtrace::backtrace::trace::h736111741fa0878e\n 1: 0x4f38c2 - backtrace::capture::Backtrace::new::h63b8a5c0787510c9\n 2: 0x442d88 - webdriver::error::WebDriverError::new::hea6d4dbf778b2b24\n 3: 0x44a8c3 - geckodriver::marionette::MarionetteHandler::create_connection::hf8b6061dba65cdd0\n 4: 0x42ac91 - <webdriver::server::Dispatcher<T, U>>::run::hba9181b5aacf8f04\n 5: 0x402c59 - std::sys_common::backtrace::__rust_begin_short_backtrace::h19de262639927233\n 6: 0x40c065 - std::panicking::try::do_call::h6c1659fc4d01af51\n 7: 0x5e38ec - panic_unwind::__rust_maybe_catch_panic\n at /checkout/src/libpanic_unwind/lib.rs:98\n 8: 0x420d32 - <F as alloc::boxed::FnBox<A>>::call_box::h953e5f59694972c5\n 9: 0x5dc00b - alloc::boxed::{{impl}}::call_once<(),()>\n at /checkout/src/liballoc/boxed.rs:661\n - std::sys_common::thread::start_thread\n at /checkout/src/libstd/sys_common/thread.rs:21\n - std::sys::imp::thread::{{impl}}::new::thread_start\n at /checkout/src/libstd/sys/unix/thread.rs:84\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)\n\tat sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)\n\tat sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)\n\tat java.lang.reflect.Constructor.newInstance(Constructor.java:423)\n\tat org.openqa.selenium.remote.W3CHandshakeResponse.lambda$new$0(W3CHandshakeResponse.java:57)\n\tat org.openqa.selenium.remote.W3CHandshakeResponse.lambda$getResponseFunction$2(W3CHandshakeResponse.java:104)\n\tat org.openqa.selenium.remote.ProtocolHandshake.lambda$createSession$24(ProtocolHandshake.java:359)\n\tat java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:193)\n\tat java.util.Spliterators$ArraySpliterator.tryAdvance(Spliterators.java:958)\n\tat java.util.stream.ReferencePipeline.forEachWithCancel(ReferencePipeline.java:126)\n\tat java.util.stream.AbstractPipeline.copyIntoWithCancel(AbstractPipeline.java:498)\n\tat java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:485)\n\tat java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:471)\n\tat java.util.stream.FindOps$FindOp.evaluateSequential(FindOps.java:152)\n\tat java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)\n\tat java.util.stream.ReferencePipeline.findFirst(ReferencePipeline.java:464)\n\tat org.openqa.selenium.remote.ProtocolHandshake.createSession(ProtocolHandshake.java:362)\n\tat org.openqa.selenium.remote.server.ServicedSession$Factory.apply(ServicedSession.java:185)\n\tat org.openqa.selenium.remote.server.ActiveSessionFactory.lambda$createSession$16(ActiveSessionFactory.java:171)\n\tat java.util.Optional.map(Optional.java:215)\n\tat org.openqa.selenium.remote.server.ActiveSessionFactory.createSession(ActiveSessionFactory.java:171)\n\tat org.openqa.selenium.remote.server.commandhandler.BeginSession.execute(BeginSession.java:72)\n\tat org.openqa.selenium.remote.server.WebDriverServlet.lambda$handle$0(WebDriverServlet.java:232)\n\tat java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)\n\tat java.util.concurrent.FutureTask.run(FutureTask.java:266)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\tat java.lang.Thread.run(Thread.java:748)\n",
"stackTrace": [{
"fileName": "ThreadPoolExecutor.java",
"methodName": "run",
"className": "java.util.concurrent.ThreadPoolExecutor$Worker",
"lineNumber": 624
}, {
"fileName": "Thread.java",
"methodName": "run",
"className": "java.lang.Thread",
"lineNumber": 748
}],
"message": "Exception when trying to launch driver",
"error": "unknown error"
},
"status": 13
}
# original "message" value: "Process unexpectedly closed with status: 1\nBuild info: version: '3.6.0', revision: '6fbf3ec767', time: '2017-09-27T16:15:40.131Z'\nSystem info: host: '1bab8fee6817', ip: '172.16.0.174', os.name: 'Linux', os.arch: 'amd64', os.version: '4.4.62-hyper', java.version: '1.8.0_131'\nDriver info: driver.version: unknown\nremote stacktrace: stack backtrace:\n 0: 0x4f388c - backtrace::backtrace::trace::h736111741fa0878e\n 1: 0x4f38c2 - backtrace::capture::Backtrace::new::h63b8a5c0787510c9\n 2: 0x442d88 - webdriver::error::WebDriverError::new::hea6d4dbf778b2b24\n 3: 0x44a8c3 - geckodriver::marionette::MarionetteHandler::create_connection::hf8b6061dba65cdd0\n 4: 0x42ac91 - <webdriver::server::Dispatcher<T, U>>::run::hba9181b5aacf8f04\n 5: 0x402c59 - std::sys_common::backtrace::__rust_begin_short_backtrace::h19de262639927233\n 6: 0x40c065 - std::panicking::try::do_call::h6c1659fc4d01af51\n 7: 0x5e38ec - panic_unwind::__rust_maybe_catch_panic\n at /checkout/src/libpanic_unwind/lib.rs:98\n 8: 0x420d32 - <F as alloc::boxed::FnBox<A>>::call_box::h953e5f59694972c5\n 9: 0x5dc00b - alloc::boxed::{{impl}}::call_once<(),()>\n at /checkout/src/liballoc/boxed.rs:661\n - std::sys_common::thread::start_thread\n at /checkout/src/libstd/sys_common/thread.rs:21\n - std::sys::imp::thread::{{impl}}::new::thread_start\n at /checkout/src/libstd/sys/unix/thread.rs:84"
return json_resp(resp, 500)
def get_err_response(data, msg='error', code=400):
return json_resp({'data': data, 'code': code, 'msg': msg})
def get_ok_response(data):
return json_resp({'data': data, 'code': 200})
async def query_driver(request, driver_url):
if '//' in driver_url:
driver_url = driver_url.replace('//', '/')
request_type = driver_request_type(driver_url, request.method)
body_str = request.body.decode()
selenium_id = None
if request_type != NEW_DRIVER:
selenium_id = get_session_id(driver_url, body_str)
if request_type == NEW_DRIVER:
reuse_session = json.loads(body_str).get('reuse_session') # often None
start = datetime.datetime.now()
success, new_created, driver_dict = await app_logic.launch_driver(body_str, reuse_session or None)
# success, driver_dict = await loop.run_in_executor( # or: await asyncio.wait_for(future, timeout, loop=loop)
# None, app_logic.launch_driver, (reuse_session or None)
# )
end = datetime.datetime.now()
print('launch_driver took: %s ' % (end-start))
if success:
return json_resp(driver_dict['creation_resp_json'], 200)
raise SanicException('driver launch failed', 500)
# elif request_type == GET_COMMAND:
# pass
# too complex, lets leave this out
# retry_count = 0
# while app_logic.drivers[driver_id]['state'] == 'PAGE_GET_IN_PROGRESS' and retry_count < 100:
# await asyncio.sleep(0.2); retry_count += 1
#
# if app_logic.drivers[driver_id]['state'] == 'PAGE_GET_SUCCESS':
# return page_get_response(app_logic.drivers[driver_id]['selenium_session_id'])
# elif app_logic.drivers[driver_id]['state'] == 'PAGE_GET_FAILED':
# return page_get_response(app_logic.drivers[driver_id]['selenium_session_id'], 'error')
# otherwise, continue as normal and proxy request below
if request_type == QUIT_COMMAND:
await app_logic.quit_driver(selenium_id)
return quit_response(selenium_id)
container_name = app_logic.drivers[selenium_id]['container']
url = 'http://' + container_name + ':5555/' + driver_url
sess = app_logic.drivers[selenium_id]['requests_session']
# resp = await loop.run_in_executor(
# None, do_selenium_request, (request, sess, url)
# )
# if request.method == 'POST':
# resp = await loop.run_in_executor(None, sess.post, (url, request.body))
# # resp = sess.post(url, data=request.body)
# elif request.method == 'GET':
# resp = await loop.run_in_executor(None, sess.get, (url, request.body))
# resp = sess.get(url, params=dict(request.args))
# elif request.method == 'DELETE':
# resp = sess.delete(url)
# to read: http://mahugh.com/2017/05/23/http-requests-asyncio-aiohttp-vs-requests/
# to read: https://gist.github.com/snehesht/c8ef95850c550dc47126
# need an asyn version of requests.get() (possibility: https://stackoverflow.com/questions/22190403/how-could-i-use-requests-in-asyncio and client example here: https://aiohttp.readthedocs.io/en/stable/)
# todo: I think this is what it needs: https://aiohttp.readthedocs.io/en/stable/client_reference.html
resp = await loop.run_in_executor(None, do_selenium_request, request, sess, url)
if resp.status_code != 200:
print('warning: selenium request gave status: %s' % resp.status_code)
return HTTPResponse(resp.content.decode(), status=resp.status_code, content_type="application/json")