async def test_get_event_listener(): browser = aiochrome.Browser() await close_all_tabs(browser) tab = await browser.new_tab() async def request_will_be_sent(**kwargs): await tab.stop() await tab.start() tab.Network.requestWillBeSent = request_will_be_sent await tab.Network.enable() try: await tab.Page.navigate(url="chrome://newtab/") except aiochrome.UserAbortException: pass if not await tab.wait(timeout=5): assert False, "never get here" assert tab.Network.requestWillBeSent == request_will_be_sent tab.Network.requestWillBeSent = None assert not tab.get_listener("Network.requestWillBeSent") # notice this assert tab.Network.requestWillBeSent != tab.get_listener( "Network.requestWillBeSent") await tab.stop()
async def test_reuse_tab_error(): browser = aiochrome.Browser() await close_all_tabs(browser) tab = await browser.new_tab() async def request_will_be_sent(**kwargs): await tab.stop() await tab.start() tab.Network.requestWillBeSent = request_will_be_sent await tab.Network.enable() try: await tab.Page.navigate(url="chrome://newtab/") except aiochrome.UserAbortException: pass if not await tab.wait(timeout=5): assert False, "never get here" try: await tab.Page.navigate(url="http://www.fatezero.org") assert False, "never get here" except aiochrome.RuntimeException: pass await tab.stop()
async def test_status(): browser = aiochrome.Browser() await close_all_tabs(browser) tab = await browser.new_tab() assert tab.status == aiochrome.Tab.status_initial async def request_will_be_sent(**kwargs): await tab.stop() tab.Network.requestWillBeSent = request_will_be_sent assert tab.status == aiochrome.Tab.status_initial await tab.start() await tab.Network.enable() assert tab.status == aiochrome.Tab.status_started try: await tab.Page.navigate(url="chrome://newtab/") except aiochrome.UserAbortException: pass if not await tab.wait(timeout=5): assert False, "never get here" await tab.stop() assert tab.status == aiochrome.Tab.status_stopped
async def test_browser_new(): browser = aiochrome.Browser() await close_all_tabs(browser) await browser.new_tab() tabs = await browser.list_tab() assert len(tabs) == 1
async def test_invalid_params(): browser = aiochrome.Browser() await close_all_tabs(browser) tab = await browser.new_tab() await tab.start() try: await tab.Page.navigate() assert False, "never get here" except aiochrome.CallMethodException: pass try: await tab.Page.navigate("http://www.fatezero.org") assert False, "never get here" except aiochrome.CallMethodException: pass try: await tab.Page.navigate(invalid_params="http://www.fatezero.org") assert False, "never get here" except aiochrome.CallMethodException: pass try: await tab.Page.navigate(url="http://www.fatezero.org", invalid_params=123) except aiochrome.CallMethodException: assert False, "never get here" await tab.stop()
async def main(): browser = aiochrome.Browser() await close_all_tabs(browser) tabs = [] for i in range(len(urls)): tabs.append(await browser.new_tab()) for i, tab in enumerate(tabs): eh = EventHandler(browser, tab) tab.Page.frameStartedLoading = eh.frame_started_loading tab.Page.frameStoppedLoading = eh.frame_stopped_loading await tab.start() await tab.Page.stopLoading() await tab.Page.enable() await tab.Page.navigate(url=urls[i]) for tab in tabs: await tab.wait(60) await tab.stop() await browser.close_tab(tab.id) print('Done')
async def main(): # create a browser instance browser = aiochrome.Browser(url="http://127.0.0.1:9222") # create a tab tab = await browser.new_tab() # register callback if you want async def request_will_be_sent(**kwargs): print("loading: %s" % kwargs.get('request').get('url')) tab.Network.requestWillBeSent = request_will_be_sent # start the tab await tab.start() # call method await tab.Network.enable() # call method with timeout await tab.Page.navigate(url="https://github.com/fate0/aiochrome", _timeout=5) # wait for loading await tab.wait(5) # stop the tab (stop handle events and stop recv message from chrome) await tab.stop() # close tab await browser.close_tab(tab)
async def list(host, port, secure): """list all the available targets/tabs""" url = "%s://%s:%s" % ("https" if secure else "http", host, port) try: browser = aiochrome.Browser(url) click.echo(json.dumps(await browser.list_tab(), cls=JSONTabEncoder, indent=4)) except Exception as e: click.echo(e)
async def test_browser_tabs_map(): browser = aiochrome.Browser() await close_all_tabs(browser) tab = await browser.new_tab() assert tab in await browser.list_tab() await browser.close_tab(tab) assert tab not in await browser.list_tab()
async def close(host, port, secure, id): """close a target/tab by id""" url = "%s://%s:%s" % ("https" if secure else "http", host, port) try: browser = aiochrome.Browser(url) click.echo(await browser.close_tab(id)) except Exception as e: click.echo(e)
async def new(host, port, secure, url="about:blank"): """create a new target/tab""" _url = "%s://%s:%s" % ("https" if secure else "http", host, port) try: browser = aiochrome.Browser(_url) click.echo(json.dumps(await browser.new_tab(url), cls=JSONTabEncoder, indent=4)) except Exception as e: click.echo(e)
async def version(host, port, secure): """show the browser version""" url = "%s://%s:%s" % ("https" if secure else "http", host, port) try: browser = aiochrome.Browser(url) click.echo(json.dumps(await browser.version(), indent=4)) except Exception as e: click.echo(e)
async def test_browser_activate_tab(): browser = aiochrome.Browser() await close_all_tabs(browser) tabs = [] for i in range(10): tabs.append(await browser.new_tab()) for tab in tabs: await browser.activate_tab(tab)
async def test_invalid_method(): browser = aiochrome.Browser() await close_all_tabs(browser) tab = await browser.new_tab() await tab.start() try: await tab.Page.NotExistMethod() assert False, "never get here" except aiochrome.CallMethodException: pass await tab.stop()
async def test_set_wrong_listener(): browser = aiochrome.Browser() await close_all_tabs(browser) tab = await browser.new_tab() await tab.start() try: tab.Network.requestWillBeSent = "test" assert False, "never get here" except aiochrome.RuntimeException: pass await tab.stop()
async def test_normal_callmethod(): browser = aiochrome.Browser() await close_all_tabs(browser) tab = await browser.new_tab() await tab.start() result = await tab.Page.navigate(url="http://www.fatezero.org") assert result['frameId'] await asyncio.sleep(1) result = await tab.Runtime.evaluate(expression="document.domain") assert result['result']['type'] == 'string' assert result['result']['value'] == 'www.fatezero.org' await tab.stop()
async def test_browser_new_100_tabs(): browser = aiochrome.Browser() await close_all_tabs(browser) tabs = [] for i in range(100): tabs.append(await browser.new_tab()) await asyncio.sleep(1) assert len(await browser.list_tab()) == 100 for tab in tabs: await browser.close_tab(tab) await asyncio.sleep(1) assert len(await browser.list_tab()) == 0
async def test_callback_exception(): browser = aiochrome.Browser() await close_all_tabs(browser) tab = await browser.new_tab() async def request_will_be_sent(**kwargs): raise Exception("test callback exception") await tab.start() tab.Network.requestWillBeSent = request_will_be_sent await tab.Network.enable() await tab.Page.navigate(url="chrome://newtab/") if await tab.wait(timeout=3): assert False, "never get here" await tab.stop()
async def test_use_callable_class_event_listener(): browser = aiochrome.Browser() await close_all_tabs(browser) tab = await browser.new_tab() await tab.start() tab.Network.requestWillBeSent = CallableClass(tab) await tab.Network.enable() try: await tab.Page.navigate(url="chrome://newtab/") except aiochrome.UserAbortException: pass if not await tab.wait(timeout=5): assert False, "never get here" await tab.stop()
async def main(): browser = aiochrome.Browser(url="http://127.0.0.1:9222") tab = await browser.new_tab() async def request_will_be_sent(**kwargs): print("loading: %s" % kwargs.get('request').get('url')) tab.set_listener("Network.requestWillBeSent", request_will_be_sent) await tab.start() await tab.call_method("Network.enable") await tab.call_method("Page.navigate", url="https://github.com/fate0/aiochrome", _timeout=5) await tab.wait(5) await tab.stop() await browser.close_tab(tab)
async def test_call_method_timeout(): browser = aiochrome.Browser() await close_all_tabs(browser) tab = await browser.new_tab() await tab.start() await tab.Page.navigate(url="chrome://newtab/", _timeout=5) try: await tab.Page.navigate(url="http://www.fatezero.org", _timeout=0.8) except aiochrome.TimeoutException: pass try: await tab.Page.navigate(url="http://www.fatezero.org", _timeout=0.005) except aiochrome.TimeoutException: pass await tab.stop()
async def test_del_all_event_listener(): browser = aiochrome.Browser() await close_all_tabs(browser) tab = await browser.new_tab() test_list = [] async def request_will_be_sent(**kwargs): test_list.append(1) tab.del_all_listeners() await tab.start() tab.Network.requestWillBeSent = request_will_be_sent await tab.Network.enable() await tab.Page.navigate(url="chrome://newtab/") if await tab.wait(timeout=5): assert False, "never get here" assert len(test_list) == 1 await tab.stop()
async def test_set_event_listener(): browser = aiochrome.Browser() await close_all_tabs(browser) tabs = await new_multi_tabs(browser, 10) async def request_will_be_sent(tab, **kwargs): await tab.stop() for tab in tabs: await tab.start() tab.Network.requestWillBeSent = functools.partial(request_will_be_sent, tab) await tab.Network.enable() try: await tab.Page.navigate(url="chrome://newtab/") except aiochrome.UserAbortException: pass for tab in tabs: if not await tab.wait(timeout=5): assert False, "never get here" await tab.stop()
async def test_chome_version(): browser = aiochrome.Browser() await close_all_tabs(browser) browser_version = await browser.version() assert isinstance(browser_version, dict)
async def new_browsertab(port, aioloop): @attr.s(slots=True) @add_defaults class BrowserTab(object): log = attr.ib() log_demo_site_ohlc = attr.ib() log_demo_site_sentiment = attr.ib() log_demo_site_winperc = attr.ib() log_demo_site_margin = attr.ib() symbol = attr.ib() symbols = attr.ib() candles = attr.ib() quotes = attr.ib() account_sum = attr.ib() count = attr.ib() tf = attr.ib() skip_log = attr.ib() browser = attr.ib() _tab = attr.ib() _port = attr.ib() _aioloop = attr.ib() # @timeit async def async_get_noticeable_data(self, caller_name, **kwargs): try: s = kwargs["response"]["payloadData"] j = json.loads(s) if not isinstance(j, list): self.log.warning(f"{caller_name}: Scalar instead of list '{j}'") j = [j] return j except Exception: self.log.exception(f'{caller_name}: Decode error: {kwargs}') raise async def async_websocket_frame_received(self, **kwargs): if self.log.isEnabledFor(logging.DEBUG): logging.debug(f"websocket_frame_received: {kwargs}") dispatch = { **{i: None for i in [3, 90]}, 52: self.async_got_52_account_sum, 70: self.async_got_70_symbol_list, 1: self.async_got_1_quote, 2: self.async_got_2_ohlc, 4: self.async_got_4_quote_ohlc, 72: self.async_got_72_winperc_list, 73: self.async_got_73_sentiment_list, 80: self.async_got_80_got_margins, 95: self.async_got_95_ack_symbol, } log_info = { # forced standard data logging for items in list i: 1 for i in [None] } log_debug = { # forced debug data logging for items in list i: 1 for i in [None] } j = await self.async_get_noticeable_data("websocket_frame_received", **kwargs) for row in j: e = int(row["e"]) del row["e"] if e in dispatch: coro = dispatch[e] if coro is not None: await coro(e, row) if e in log_info: self.log.info(f"{e} (DEBUG): {row}") if e not in dispatch or e in log_debug: eventloop.create_task(self.async_debug_frame(e, row)) # @timeit async def async_websocket_frame_sent(self, **kwargs): j = await self.async_get_noticeable_data("websocket_frame_sent", **kwargs) self.log.info("websocket_frame_sent: %s", j) # @timeit async def async_websocket_created(self, **kwargs): self.log.debug("async_websocket_created AS IS: %s", kwargs) # @timeit async def async_websocket_will_send_handshake_request(self, **kwargs): self.log.debug("websocket_will_send_handshake_request: %s", kwargs) # @timeit async def async_websocket_handshake_response_received(self, **kwargs): self.log.info("websocket_handshake_response_received: %s", kwargs) # @timeit async def async_websocket_closed(self, **kwargs): self.log.debug("websocket_closed AS IS: %s", kwargs) async def async_go(self, url="https://demo_site.com/platform", timeout=5): await self._tab.call_method("Page.navigate", url=url, _timeout=timeout) # shortcut for the convenience # @timeit async def async_debug_frame(self, e, j, keep_intact=True): """ :param e: opcode :param j: data :param keep_intact: process the copy, not original data (slows execution) :return: """ if e in self.skip_log: return if keep_intact: j = copy.deepcopy(j) if e in (3, 4): for d in j['d']: if 'quotes' in d: d['quotes'] = d['quotes'][:2] if 'candles' in d: d['candles'] = d['candles'][:2] elif e in (70, 73): j['d'] = j['d'][:2] elif e in (80,): for d in j['d']: d['d'] = d['d'][:2] # simplify a little if e in (1, 2, 52, 72, 111): # 70. 73 j = j['d'][0] # logging.debug(f"{e}: {j}") self.log.debug(f"{e}: %s", j) # @timeit async def async_got_1_quote(self, e, j): try: jj = j["d"][0] t = jj['t'] # assert name == current_pair, f'Quote for {name} instead of {current_pair} : {j}' if self.quotes is None: # if not hasattr(self, 'quotes'): # noinspection PyAttributeOutsideInit self.quotes = pd.DataFrame(jj, columns=['q'], index=[t]) else: self.quotes.at[t] = {'q': jj['q']} if e not in self.skip_log: self.log.debug(f"{e}: Quote: {jj['j']}") # self.log.debug(f"Quote data size: {len(self.quotes.index)}") except Exception: self.log.exception(f"{e}: Wrong data format: {j}") raise # @timeit async def async_got_2_ohlc(self, e, j): cols = ['open', 'high', 'low', 'close'] try: jj = j["d"][0] t = jj['t'] if self.candles is None: # if not hasattr(self, 'candles'): # noinspection PyAttributeOutsideInit self.candles = pd.DataFrame(jj, columns=cols, index=[t]) else: self.log_demo_site_ohlc.info( f"{jj['t']},{jj['p']},{jj['open']},{jj['high']},{jj['low']},{jj['close']}") if e not in self.skip_log: self.log.debug(f"{e}: Candle: {{k: jj[k] for k in cols}}") # self.log.debug(f"Candles size: {len(self.candles.index)}") except Exception: self.log.exception(f"{e}: Wrong data format: {j}") raise @timeit async def async_got_4_quote_ohlc(self, e, j): skip_log = e in self.skip_log try: if j["d"] is None: self.log.debug(f"{e}: No data") return alen = len(j["d"]) assert alen == 1, f"{e}: array length is more than 1" uuid = f', uuid: {j["uuid"]}' if "uuid" in j else '' jj = j["d"][0] # current_pair = self.symbol["pair"] # assert name == current_pair, f'The data are not for the current pair ' \ # f'{current_pair}:{self.symbol["uuid"]} - {j}' # noinspection PyAttributeOutsideInit self.tf = jj['tf'] # noinspection PyAttributeOutsideInit self.symbol = jj["p"] # self.log_demo_site_ohlc.info(f"; [{self.symbol}]") sub_uid = f', sub_uid: {jj["sub_uid"]}' if "sub_uid" in jj else '' if not skip_log: self.log.info(f"{e}: Pair: {self.symbol}, timeframe: {self.tf}{uuid}{sub_uid}") with LogTimer('PD.CANDLES', level=logging.DEBUG): if "candles" in jj: # 6 ms df = pd.DataFrame(jj["candles"]) df.set_index('t', inplace=True) df_csv = df.copy() df_csv.sort_index(inplace=True) df_csv["symbol"] = self.symbol self.log_demo_site_ohlc.info(f"; BATCH BEGIN") self.log_demo_site_ohlc.info( df_csv.to_csv( header=False, columns=['symbol', 'open', 'high', 'low', 'close'], line_terminator="\n", quoting=csv.QUOTE_NONE, )) self.log_demo_site_ohlc.info(f"; BATCH END") if self.candles is None: self.candles = df else: self.candles = df if not skip_log: self.log.debug(f"{e}: Candles: {len(df.index)}") await asyncio.sleep(0) else: # noinspection PyAttributeOutsideInit self.candles = None with LogTimer('PD.QUOTES', level=logging.DEBUG): if "quotes" in jj: # 8 ms df = pd.DataFrame(jj["quotes"]) df.set_index(['t'], inplace=True) if self.quotes is None: self.quotes = df else: self.quotes = pd.concat([self.quotes, df]).drop_duplicates() if not skip_log: self.log.debug(f"{e}: Quotes: {len(df.index)}") else: # noinspection PyAttributeOutsideInit self.quotes = None except Exception: self.log.exception(f"{e}: Wrong data format: {j}") raise # @timeit async def async_got_52_account_sum(self, e, j): try: val = j["d"][0]["value"] # noinspection PyAttributeOutsideInit self.account_sum = val if e not in self.skip_log: self.log.debug(f"{e}: Account remainder: {val}") except Exception: self.log.exception(f"{e}: Wrong data format: {j}") raise # @timeit async def async_got_70_symbol_list(self, e, j): try: df = pd.DataFrame(j["d"]) # 90 ms await asyncio.sleep(0) df.set_index('name', inplace=True) # noinspection PyAttributeOutsideInit self.symbols = df if e not in self.skip_log: keys = sorted(df.index.values.tolist()) self.log.debug(f"{e}: Pairs received: {keys!s}") except Exception: self.log.exception(f"{e}: Wrong data format: {j}") raise # @timeit async def async_got_72_winperc_list(self, e, j): if self.symbols is None: self.log.warning(f"{e}: Margin got before the pairs list: {j}") return try: t = int(time.time()) jd = j["d"] self.log_demo_site_winperc.info( "\n".join(((f"{t},{row['pair']},{row['winperc']}" for row in jd))) ) for jj in jd: self.symbols.at[jj["pair"], "winperc"] = jj['winperc'] if e not in self.skip_log: self.log.info(f"{e}: Margin: {j['d']}") except Exception: self.log.exception(f"{e}: Wrong data format: {j}") raise @timeit async def async_got_73_sentiment_list(self, e, j): if self.symbols is None: self.log.warning(f"{e}: Sentiments got before the pairs list: {j}") return try: t = int(time.time()) jd = j["d"] self.log_demo_site_sentiment.info( "\n".join(((f"{t},{row['pair']},{row['sentiment']}" for row in jd))) ) existing_keys_dict = {k: 1 for k in self.symbols.index.values.tolist()} existing_keys_data_list = [row for row in jd if row['pair'] in existing_keys_dict] keys = [row['pair'] for row in existing_keys_data_list] values = [[row['sentiment']] for row in existing_keys_data_list] self.symbols.loc[keys, 'sentiment'] = values non_existing_keys_data_list = [row for row in jd if row['pair'] not in existing_keys_dict] symbols_dict = {k: None for k in list(self.symbols.columns)} for k in non_existing_keys_data_list: symbols_dict["sentiment"] = k['sentiment'] self.symbols.at[k["pair"]] = symbols_dict pass if e not in self.skip_log: self.log.info(f"{e}: Sentiments: {j['d']}") except Exception: self.log.exception(f"{e}: Wrong data format: {j}") raise def async_got_80_got_margins(self, e, j): try: self.log_demo_site_margin.info(j) except Exception: self.log.exception(f"{e}: Wrong data format: {j}") raise async def async_got_95_ack_symbol(self, e, j): try: self.symbol = j["d"][0]["pair"] except Exception: self.log.exception(f"Wrong data format (e): {j}") raise browsertab = BrowserTab( port=port, aioloop=aioloop, skip_log={i: 1 for i in [1, 2, 72, 73]}, log=logging.getLogger('BrowserTab'), log_demo_site_ohlc=logging.getLogger("demo_site_ohlc"), log_demo_site_sentiment=logging.getLogger("demo_site_sentiment"), log_demo_site_winperc=logging.getLogger("demo_site_winperc"), log_demo_site_margin=logging.getLogger("demo_site_margin"), browser=aiochrome.Browser(url=f"http://127.0.0.1:{port!s}", loop=aioloop) ) tab = await browsertab.browser.new_tab() # if newtab else await browsertab._browser.list_tab()[0] browsertab._tab = tab tab.set_listener("Network.webSocketFrameSent", browsertab.async_websocket_frame_sent) tab.set_listener("Network.webSocketFrameReceived", browsertab.async_websocket_frame_received) await tab.start() await tab.call_method("Network.enable") # handlers # https://chromedevtools.github.io/devtools-protocol/tot/Network#event-webSocketClosed # chrome://net-internals/#events&q=type:SOCKET%20is:active await browsertab.async_go() pass