async def test_get_event_listener():
browser = aiochrome.Browser()
await close_all_tabs(browser)
tab = await browser.new_tab()
async def request_will_be_sent(**kwargs):
await tab.stop()
await tab.start()
tab.Network.requestWillBeSent = request_will_be_sent
await tab.Network.enable()
try:
await tab.Page.navigate(url="chrome://newtab/")
except aiochrome.UserAbortException:
pass
if not await tab.wait(timeout=5):
assert False, "never get here"
assert tab.Network.requestWillBeSent == request_will_be_sent
tab.Network.requestWillBeSent = None
assert not tab.get_listener("Network.requestWillBeSent")
# notice this
assert tab.Network.requestWillBeSent != tab.get_listener(
"Network.requestWillBeSent")
await tab.stop()
async def test_reuse_tab_error():
browser = aiochrome.Browser()
await close_all_tabs(browser)
tab = await browser.new_tab()
async def request_will_be_sent(**kwargs):
await tab.stop()
await tab.start()
tab.Network.requestWillBeSent = request_will_be_sent
await tab.Network.enable()
try:
await tab.Page.navigate(url="chrome://newtab/")
except aiochrome.UserAbortException:
pass
if not await tab.wait(timeout=5):
assert False, "never get here"
try:
await tab.Page.navigate(url="http://www.fatezero.org")
assert False, "never get here"
except aiochrome.RuntimeException:
pass
await tab.stop()
async def test_status():
browser = aiochrome.Browser()
await close_all_tabs(browser)
tab = await browser.new_tab()
assert tab.status == aiochrome.Tab.status_initial
async def request_will_be_sent(**kwargs):
await tab.stop()
tab.Network.requestWillBeSent = request_will_be_sent
assert tab.status == aiochrome.Tab.status_initial
await tab.start()
await tab.Network.enable()
assert tab.status == aiochrome.Tab.status_started
try:
await tab.Page.navigate(url="chrome://newtab/")
except aiochrome.UserAbortException:
pass
if not await tab.wait(timeout=5):
assert False, "never get here"
await tab.stop()
assert tab.status == aiochrome.Tab.status_stopped
async def test_browser_new():
browser = aiochrome.Browser()
await close_all_tabs(browser)
await browser.new_tab()
tabs = await browser.list_tab()
assert len(tabs) == 1
async def test_invalid_params():
browser = aiochrome.Browser()
await close_all_tabs(browser)
tab = await browser.new_tab()
await tab.start()
try:
await tab.Page.navigate()
assert False, "never get here"
except aiochrome.CallMethodException:
pass
try:
await tab.Page.navigate("http://www.fatezero.org")
assert False, "never get here"
except aiochrome.CallMethodException:
pass
try:
await tab.Page.navigate(invalid_params="http://www.fatezero.org")
assert False, "never get here"
except aiochrome.CallMethodException:
pass
try:
await tab.Page.navigate(url="http://www.fatezero.org",
invalid_params=123)
except aiochrome.CallMethodException:
assert False, "never get here"
await tab.stop()
async def main():
browser = aiochrome.Browser()
await close_all_tabs(browser)
tabs = []
for i in range(len(urls)):
tabs.append(await browser.new_tab())
for i, tab in enumerate(tabs):
eh = EventHandler(browser, tab)
tab.Page.frameStartedLoading = eh.frame_started_loading
tab.Page.frameStoppedLoading = eh.frame_stopped_loading
await tab.start()
await tab.Page.stopLoading()
await tab.Page.enable()
await tab.Page.navigate(url=urls[i])
for tab in tabs:
await tab.wait(60)
await tab.stop()
await browser.close_tab(tab.id)
print('Done')
async def main():
# create a browser instance
browser = aiochrome.Browser(url="http://127.0.0.1:9222")
# create a tab
tab = await browser.new_tab()
# register callback if you want
async def request_will_be_sent(**kwargs):
print("loading: %s" % kwargs.get('request').get('url'))
tab.Network.requestWillBeSent = request_will_be_sent
# start the tab
await tab.start()
# call method
await tab.Network.enable()
# call method with timeout
await tab.Page.navigate(url="https://github.com/fate0/aiochrome",
_timeout=5)
# wait for loading
await tab.wait(5)
# stop the tab (stop handle events and stop recv message from chrome)
await tab.stop()
# close tab
await browser.close_tab(tab)
async def list(host, port, secure):
"""list all the available targets/tabs"""
url = "%s://%s:%s" % ("https" if secure else "http", host, port)
try:
browser = aiochrome.Browser(url)
click.echo(json.dumps(await browser.list_tab(), cls=JSONTabEncoder, indent=4))
except Exception as e:
click.echo(e)
async def test_browser_tabs_map():
browser = aiochrome.Browser()
await close_all_tabs(browser)
tab = await browser.new_tab()
assert tab in await browser.list_tab()
await browser.close_tab(tab)
assert tab not in await browser.list_tab()
async def close(host, port, secure, id):
"""close a target/tab by id"""
url = "%s://%s:%s" % ("https" if secure else "http", host, port)
try:
browser = aiochrome.Browser(url)
click.echo(await browser.close_tab(id))
except Exception as e:
click.echo(e)
async def new(host, port, secure, url="about:blank"):
"""create a new target/tab"""
_url = "%s://%s:%s" % ("https" if secure else "http", host, port)
try:
browser = aiochrome.Browser(_url)
click.echo(json.dumps(await browser.new_tab(url), cls=JSONTabEncoder, indent=4))
except Exception as e:
click.echo(e)
async def version(host, port, secure):
"""show the browser version"""
url = "%s://%s:%s" % ("https" if secure else "http", host, port)
try:
browser = aiochrome.Browser(url)
click.echo(json.dumps(await browser.version(), indent=4))
except Exception as e:
click.echo(e)
async def test_browser_activate_tab():
browser = aiochrome.Browser()
await close_all_tabs(browser)
tabs = []
for i in range(10):
tabs.append(await browser.new_tab())
for tab in tabs:
await browser.activate_tab(tab)
async def test_invalid_method():
browser = aiochrome.Browser()
await close_all_tabs(browser)
tab = await browser.new_tab()
await tab.start()
try:
await tab.Page.NotExistMethod()
assert False, "never get here"
except aiochrome.CallMethodException:
pass
await tab.stop()
async def test_set_wrong_listener():
browser = aiochrome.Browser()
await close_all_tabs(browser)
tab = await browser.new_tab()
await tab.start()
try:
tab.Network.requestWillBeSent = "test"
assert False, "never get here"
except aiochrome.RuntimeException:
pass
await tab.stop()
async def test_normal_callmethod():
browser = aiochrome.Browser()
await close_all_tabs(browser)
tab = await browser.new_tab()
await tab.start()
result = await tab.Page.navigate(url="http://www.fatezero.org")
assert result['frameId']
await asyncio.sleep(1)
result = await tab.Runtime.evaluate(expression="document.domain")
assert result['result']['type'] == 'string'
assert result['result']['value'] == 'www.fatezero.org'
await tab.stop()
async def test_browser_new_100_tabs():
browser = aiochrome.Browser()
await close_all_tabs(browser)
tabs = []
for i in range(100):
tabs.append(await browser.new_tab())
await asyncio.sleep(1)
assert len(await browser.list_tab()) == 100
for tab in tabs:
await browser.close_tab(tab)
await asyncio.sleep(1)
assert len(await browser.list_tab()) == 0
async def test_callback_exception():
browser = aiochrome.Browser()
await close_all_tabs(browser)
tab = await browser.new_tab()
async def request_will_be_sent(**kwargs):
raise Exception("test callback exception")
await tab.start()
tab.Network.requestWillBeSent = request_will_be_sent
await tab.Network.enable()
await tab.Page.navigate(url="chrome://newtab/")
if await tab.wait(timeout=3):
assert False, "never get here"
await tab.stop()
async def test_use_callable_class_event_listener():
browser = aiochrome.Browser()
await close_all_tabs(browser)
tab = await browser.new_tab()
await tab.start()
tab.Network.requestWillBeSent = CallableClass(tab)
await tab.Network.enable()
try:
await tab.Page.navigate(url="chrome://newtab/")
except aiochrome.UserAbortException:
pass
if not await tab.wait(timeout=5):
assert False, "never get here"
await tab.stop()
async def main():
browser = aiochrome.Browser(url="http://127.0.0.1:9222")
tab = await browser.new_tab()
async def request_will_be_sent(**kwargs):
print("loading: %s" % kwargs.get('request').get('url'))
tab.set_listener("Network.requestWillBeSent", request_will_be_sent)
await tab.start()
await tab.call_method("Network.enable")
await tab.call_method("Page.navigate",
url="https://github.com/fate0/aiochrome",
_timeout=5)
await tab.wait(5)
await tab.stop()
await browser.close_tab(tab)
async def test_call_method_timeout():
browser = aiochrome.Browser()
await close_all_tabs(browser)
tab = await browser.new_tab()
await tab.start()
await tab.Page.navigate(url="chrome://newtab/", _timeout=5)
try:
await tab.Page.navigate(url="http://www.fatezero.org", _timeout=0.8)
except aiochrome.TimeoutException:
pass
try:
await tab.Page.navigate(url="http://www.fatezero.org", _timeout=0.005)
except aiochrome.TimeoutException:
pass
await tab.stop()
async def test_del_all_event_listener():
browser = aiochrome.Browser()
await close_all_tabs(browser)
tab = await browser.new_tab()
test_list = []
async def request_will_be_sent(**kwargs):
test_list.append(1)
tab.del_all_listeners()
await tab.start()
tab.Network.requestWillBeSent = request_will_be_sent
await tab.Network.enable()
await tab.Page.navigate(url="chrome://newtab/")
if await tab.wait(timeout=5):
assert False, "never get here"
assert len(test_list) == 1
await tab.stop()
async def test_set_event_listener():
browser = aiochrome.Browser()
await close_all_tabs(browser)
tabs = await new_multi_tabs(browser, 10)
async def request_will_be_sent(tab, **kwargs):
await tab.stop()
for tab in tabs:
await tab.start()
tab.Network.requestWillBeSent = functools.partial(request_will_be_sent, tab)
await tab.Network.enable()
try:
await tab.Page.navigate(url="chrome://newtab/")
except aiochrome.UserAbortException:
pass
for tab in tabs:
if not await tab.wait(timeout=5):
assert False, "never get here"
await tab.stop()
async def test_chome_version():
browser = aiochrome.Browser()
await close_all_tabs(browser)
browser_version = await browser.version()
assert isinstance(browser_version, dict)
async def new_browsertab(port, aioloop):
@attr.s(slots=True)
@add_defaults
class BrowserTab(object):
log = attr.ib()
log_demo_site_ohlc = attr.ib()
log_demo_site_sentiment = attr.ib()
log_demo_site_winperc = attr.ib()
log_demo_site_margin = attr.ib()
symbol = attr.ib()
symbols = attr.ib()
candles = attr.ib()
quotes = attr.ib()
account_sum = attr.ib()
count = attr.ib()
tf = attr.ib()
skip_log = attr.ib()
browser = attr.ib()
_tab = attr.ib()
_port = attr.ib()
_aioloop = attr.ib()
# @timeit
async def async_get_noticeable_data(self, caller_name, **kwargs):
try:
s = kwargs["response"]["payloadData"]
j = json.loads(s)
if not isinstance(j, list):
self.log.warning(f"{caller_name}: Scalar instead of list '{j}'")
j = [j]
return j
except Exception:
self.log.exception(f'{caller_name}: Decode error: {kwargs}')
raise
async def async_websocket_frame_received(self, **kwargs):
if self.log.isEnabledFor(logging.DEBUG):
logging.debug(f"websocket_frame_received: {kwargs}")
dispatch = {
**{i: None for i in [3, 90]},
52: self.async_got_52_account_sum,
70: self.async_got_70_symbol_list,
1: self.async_got_1_quote,
2: self.async_got_2_ohlc,
4: self.async_got_4_quote_ohlc,
72: self.async_got_72_winperc_list,
73: self.async_got_73_sentiment_list,
80: self.async_got_80_got_margins,
95: self.async_got_95_ack_symbol,
}
log_info = { # forced standard data logging for items in list
i: 1 for i in [None]
}
log_debug = { # forced debug data logging for items in list
i: 1 for i in [None]
}
j = await self.async_get_noticeable_data("websocket_frame_received", **kwargs)
for row in j:
e = int(row["e"])
del row["e"]
if e in dispatch:
coro = dispatch[e]
if coro is not None:
await coro(e, row)
if e in log_info:
self.log.info(f"{e} (DEBUG): {row}")
if e not in dispatch or e in log_debug:
eventloop.create_task(self.async_debug_frame(e, row))
# @timeit
async def async_websocket_frame_sent(self, **kwargs):
j = await self.async_get_noticeable_data("websocket_frame_sent", **kwargs)
self.log.info("websocket_frame_sent: %s", j)
# @timeit
async def async_websocket_created(self, **kwargs):
self.log.debug("async_websocket_created AS IS: %s", kwargs)
# @timeit
async def async_websocket_will_send_handshake_request(self, **kwargs):
self.log.debug("websocket_will_send_handshake_request: %s", kwargs)
# @timeit
async def async_websocket_handshake_response_received(self, **kwargs):
self.log.info("websocket_handshake_response_received: %s", kwargs)
# @timeit
async def async_websocket_closed(self, **kwargs):
self.log.debug("websocket_closed AS IS: %s", kwargs)
async def async_go(self, url="https://demo_site.com/platform", timeout=5):
await self._tab.call_method("Page.navigate", url=url, _timeout=timeout)
# shortcut for the convenience
# @timeit
async def async_debug_frame(self, e, j, keep_intact=True):
"""
:param e: opcode
:param j: data
:param keep_intact: process the copy, not original data (slows execution)
:return:
"""
if e in self.skip_log:
return
if keep_intact:
j = copy.deepcopy(j)
if e in (3, 4):
for d in j['d']:
if 'quotes' in d:
d['quotes'] = d['quotes'][:2]
if 'candles' in d:
d['candles'] = d['candles'][:2]
elif e in (70, 73):
j['d'] = j['d'][:2]
elif e in (80,):
for d in j['d']:
d['d'] = d['d'][:2]
# simplify a little
if e in (1, 2, 52, 72, 111): # 70. 73
j = j['d'][0]
# logging.debug(f"{e}: {j}")
self.log.debug(f"{e}: %s", j)
# @timeit
async def async_got_1_quote(self, e, j):
try:
jj = j["d"][0]
t = jj['t']
# assert name == current_pair, f'Quote for {name} instead of {current_pair} : {j}'
if self.quotes is None: # if not hasattr(self, 'quotes'):
# noinspection PyAttributeOutsideInit
self.quotes = pd.DataFrame(jj, columns=['q'], index=[t])
else:
self.quotes.at[t] = {'q': jj['q']}
if e not in self.skip_log:
self.log.debug(f"{e}: Quote: {jj['j']}")
# self.log.debug(f"Quote data size: {len(self.quotes.index)}")
except Exception:
self.log.exception(f"{e}: Wrong data format: {j}")
raise
# @timeit
async def async_got_2_ohlc(self, e, j):
cols = ['open', 'high', 'low', 'close']
try:
jj = j["d"][0]
t = jj['t']
if self.candles is None: # if not hasattr(self, 'candles'):
# noinspection PyAttributeOutsideInit
self.candles = pd.DataFrame(jj, columns=cols, index=[t])
else:
self.log_demo_site_ohlc.info(
f"{jj['t']},{jj['p']},{jj['open']},{jj['high']},{jj['low']},{jj['close']}")
if e not in self.skip_log:
self.log.debug(f"{e}: Candle: {{k: jj[k] for k in cols}}")
# self.log.debug(f"Candles size: {len(self.candles.index)}")
except Exception:
self.log.exception(f"{e}: Wrong data format: {j}")
raise
@timeit
async def async_got_4_quote_ohlc(self, e, j):
skip_log = e in self.skip_log
try:
if j["d"] is None:
self.log.debug(f"{e}: No data")
return
alen = len(j["d"])
assert alen == 1, f"{e}: array length is more than 1"
uuid = f', uuid: {j["uuid"]}' if "uuid" in j else ''
jj = j["d"][0]
# current_pair = self.symbol["pair"]
# assert name == current_pair, f'The data are not for the current pair ' \
# f'{current_pair}:{self.symbol["uuid"]} - {j}'
# noinspection PyAttributeOutsideInit
self.tf = jj['tf']
# noinspection PyAttributeOutsideInit
self.symbol = jj["p"]
# self.log_demo_site_ohlc.info(f"; [{self.symbol}]")
sub_uid = f', sub_uid: {jj["sub_uid"]}' if "sub_uid" in jj else ''
if not skip_log:
self.log.info(f"{e}: Pair: {self.symbol}, timeframe: {self.tf}{uuid}{sub_uid}")
with LogTimer('PD.CANDLES', level=logging.DEBUG):
if "candles" in jj: # 6 ms
df = pd.DataFrame(jj["candles"])
df.set_index('t', inplace=True)
df_csv = df.copy()
df_csv.sort_index(inplace=True)
df_csv["symbol"] = self.symbol
self.log_demo_site_ohlc.info(f"; BATCH BEGIN")
self.log_demo_site_ohlc.info(
df_csv.to_csv(
header=False,
columns=['symbol', 'open', 'high', 'low', 'close'],
line_terminator="\n",
quoting=csv.QUOTE_NONE,
))
self.log_demo_site_ohlc.info(f"; BATCH END")
if self.candles is None:
self.candles = df
else:
self.candles = df
if not skip_log:
self.log.debug(f"{e}: Candles: {len(df.index)}")
await asyncio.sleep(0)
else:
# noinspection PyAttributeOutsideInit
self.candles = None
with LogTimer('PD.QUOTES', level=logging.DEBUG):
if "quotes" in jj: # 8 ms
df = pd.DataFrame(jj["quotes"])
df.set_index(['t'], inplace=True)
if self.quotes is None:
self.quotes = df
else:
self.quotes = pd.concat([self.quotes, df]).drop_duplicates()
if not skip_log:
self.log.debug(f"{e}: Quotes: {len(df.index)}")
else:
# noinspection PyAttributeOutsideInit
self.quotes = None
except Exception:
self.log.exception(f"{e}: Wrong data format: {j}")
raise
# @timeit
async def async_got_52_account_sum(self, e, j):
try:
val = j["d"][0]["value"]
# noinspection PyAttributeOutsideInit
self.account_sum = val
if e not in self.skip_log:
self.log.debug(f"{e}: Account remainder: {val}")
except Exception:
self.log.exception(f"{e}: Wrong data format: {j}")
raise
# @timeit
async def async_got_70_symbol_list(self, e, j):
try:
df = pd.DataFrame(j["d"]) # 90 ms
await asyncio.sleep(0)
df.set_index('name', inplace=True)
# noinspection PyAttributeOutsideInit
self.symbols = df
if e not in self.skip_log:
keys = sorted(df.index.values.tolist())
self.log.debug(f"{e}: Pairs received: {keys!s}")
except Exception:
self.log.exception(f"{e}: Wrong data format: {j}")
raise
# @timeit
async def async_got_72_winperc_list(self, e, j):
if self.symbols is None:
self.log.warning(f"{e}: Margin got before the pairs list: {j}")
return
try:
t = int(time.time())
jd = j["d"]
self.log_demo_site_winperc.info(
"\n".join(((f"{t},{row['pair']},{row['winperc']}" for row in jd)))
)
for jj in jd:
self.symbols.at[jj["pair"], "winperc"] = jj['winperc']
if e not in self.skip_log:
self.log.info(f"{e}: Margin: {j['d']}")
except Exception:
self.log.exception(f"{e}: Wrong data format: {j}")
raise
@timeit
async def async_got_73_sentiment_list(self, e, j):
if self.symbols is None:
self.log.warning(f"{e}: Sentiments got before the pairs list: {j}")
return
try:
t = int(time.time())
jd = j["d"]
self.log_demo_site_sentiment.info(
"\n".join(((f"{t},{row['pair']},{row['sentiment']}" for row in jd)))
)
existing_keys_dict = {k: 1 for k in self.symbols.index.values.tolist()}
existing_keys_data_list = [row for row in jd if row['pair'] in existing_keys_dict]
keys = [row['pair'] for row in existing_keys_data_list]
values = [[row['sentiment']] for row in existing_keys_data_list]
self.symbols.loc[keys, 'sentiment'] = values
non_existing_keys_data_list = [row for row in jd if row['pair'] not in existing_keys_dict]
symbols_dict = {k: None for k in list(self.symbols.columns)}
for k in non_existing_keys_data_list:
symbols_dict["sentiment"] = k['sentiment']
self.symbols.at[k["pair"]] = symbols_dict
pass
if e not in self.skip_log:
self.log.info(f"{e}: Sentiments: {j['d']}")
except Exception:
self.log.exception(f"{e}: Wrong data format: {j}")
raise
def async_got_80_got_margins(self, e, j):
try:
self.log_demo_site_margin.info(j)
except Exception:
self.log.exception(f"{e}: Wrong data format: {j}")
raise
async def async_got_95_ack_symbol(self, e, j):
try:
self.symbol = j["d"][0]["pair"]
except Exception:
self.log.exception(f"Wrong data format (e): {j}")
raise
browsertab = BrowserTab(
port=port,
aioloop=aioloop,
skip_log={i: 1 for i in [1, 2, 72, 73]},
log=logging.getLogger('BrowserTab'),
log_demo_site_ohlc=logging.getLogger("demo_site_ohlc"),
log_demo_site_sentiment=logging.getLogger("demo_site_sentiment"),
log_demo_site_winperc=logging.getLogger("demo_site_winperc"),
log_demo_site_margin=logging.getLogger("demo_site_margin"),
browser=aiochrome.Browser(url=f"http://127.0.0.1:{port!s}", loop=aioloop)
)
tab = await browsertab.browser.new_tab() # if newtab else await browsertab._browser.list_tab()[0]
browsertab._tab = tab
tab.set_listener("Network.webSocketFrameSent", browsertab.async_websocket_frame_sent)
tab.set_listener("Network.webSocketFrameReceived", browsertab.async_websocket_frame_received)
await tab.start()
await tab.call_method("Network.enable")
# handlers
# https://chromedevtools.github.io/devtools-protocol/tot/Network#event-webSocketClosed
# chrome://net-internals/#events&q=type:SOCKET%20is:active
await browsertab.async_go()
pass
