def process(): filepath = '/Users/alberthan/VSCodeProjects/HDLogger/youtube-dl/logs/03.pickled_states_hex.tracer.log' tp = TraceProcessor(filepath) df = tp.dataframe.drop('attrs', 1) tp.level1 tp.level_1 sts = tp.pickleable_states maxstacklen = df['stacklen'].max() # 56 try: df[df['stacklen'].apply(lambda cell: cell <= maxstacklen)] except: exc = sys.exc_info() tb = traceback.format_exception(*sys.exc_info()) wf(tb, 'logs/catchall.tb.log', 'w') sp = stackprinter.format(sys.exc_info()) wf(sp, 'logs/catchall.sp.log', 'w') try: df[df['stacklen'].apply(lambda cell: cell <= maxstacklen)] except Exception as e: exc = e tb = traceback.format_exception(*e) wf(tb, 'logs/named.tb.log', 'w') sp = stackprinter.format(e) wf(sp, 'logs/named.sp.log', 'w') wf(tp.level1, 'logs/process.level1.log', 'a') wf(tp.level_1, 'logs/process.level_1.log', 'a')
def parse_instrument(response): map_to_standard = settings.SYMBOL_MAP_TO_STANDARD["KRAKEN"] try: key = list(response.keys())[0] response = response[key] except Exception as e: logging.error(stackprinter.format(e, style="darkbg2")) try: parsed_instrument = { "symbol": map_to_standard[key], "low": response["l"][0], "high": response["h"][0], "vwap": response["p"][0], "last": response["c"][0], "volume": response["v"][0], "trdCount": response["t"][0], "bestAsk": { response["a"][0]: response["a"][2] }, "bestBid": { response["b"][0]: response["b"][2] }, "prevLow": response["l"][1], "prevHigh": response["h"][1], "prevVwap": response["p"][1], "prevVolume": response["v"][1], "prevTrdCount": response["t"][1] } except Exception as e: logging.error(stackprinter.format(e, style="darkbg2")) return parsed_instrument
async def publish_data(self, data: dict, feed: str, redis_pool): """message needs to be json loadedy str, make sure we have the correct keys """ channel = f"ws:private:data:{self.exchange}:{feed}" try: ws_data = data_models_map[feed](data=data, channel_name=feed) except ValidationError as e: logging.error(stackprinter.format(e, style="darkbg2")) try: self.feed_counters[channel] += 1 update_chan = f"data:update:{self.exchange}:{feed}" data_to_publish = ws_data.dict() data_to_publish = data_to_publish["data"] await redis_pool.publish(update_chan, ujson.dumps(data_to_publish)) except KeyError: self.feed_counters[channel] = 0 snapshot_chan = f"data:snapshot:{self.exchange}:{feed}" data_to_publish = ws_data.dict() data_to_publish = data_to_publish["data"] await redis_pool.publish(snapshot_chan, ujson.dumps(data_to_publish)) except Exception as e: logging.error(stackprinter.format(e, style="darkbg2"))
async def startup_balances(redis_instance): try: exch_balances = await CurrentBalance.all().values() # returns a list of dicts of the format {id: int , holdings:{}, exchange_id: int} # if CurrentBalance table is empty: if not exch_balances: logging.warning(f"{exch_balances} CurrentBalance DBTable is empty") try: exchanges = await Exchange.all().values() # returns a list of dicts of format {exchange_id: int, name: str} if not exchanges: await instantiate_exchange_table(exchanges) else: for exchange in exchanges: await instantiate_balance_table(exchange) except Exception as e: logging.error(stackprinter.format(e, style="darkbg2")) # if CurrentBalance table is not empty: # this code will not execute if balance table is initially empty # since exch_value will be == [] # we need to fetch the balance values from db again after they are updated : exch_balances = await CurrentBalance.all().values() for balance in exch_balances: await update_balance_holdings(balance) await update_balance_positions(balance) except Exception as e: logging.error(stackprinter.format(e, style="darkbg2")) raw_dictlist = await CurrentBalance.all().values() # returns list of dicts of format {id: int, holdings: dict, positions: dict, exchange_id: int} #iterate over raw dictlist, for each item pull out exchange_name from exchange id updated_balances = { settings.EXCHANGE_NAME_FROM_IDS[dic["exchange_id"]]: dic for dic in raw_dictlist } # separate steps to better visualise how to set up the dict comprehension # new_dict = {} # for dic in raw_dictlist: # exchange_name = settings.EXCHANGE_NAME_FROM_IDS[dic["exchange_id"]] # new_dict[exchange_name]=dic try: redis_instance.set("balances", json.dumps(updated_balances)) except Exception as e: logging.error(stackprinter.format(e, style="darkbg2"))
async def log_exc_to_db(logger: object, msg: Exception): # ErrorLog being an ORM Model : if not isinstance(msg, str): stack = stackprinter.format(msg) logger.exception(stackprinter.format(msg, style="darkbg2")) if isinstance(msg, Exception): json = {"error": str(msg)} else: return await ErrorLog.create(json=json, stack=stack)
def parse_ohlc(response): try: key = list(response.keys())[0] except Exception as e: logging.error(stackprinter.format(e, style="darkbg2")) try: parsed_ohlc = [parse_single_ohlc(item, key) for item in response[key]] except Exception as e: logging.error(stackprinter.format(e, style="darkbg2")) return parsed_ohlc
def parse_orders_to_list(response, symbol): """Parse open or closed orders into unified format and validate data against model. Args: response (typing.Any): raw response from kraken to open_orders or closed_orders query smymbol (noobit.PAIR): filter out only orders for a given symbol Returns: parsed_orders (list): list containing unvalidated noobit.Order items Note: parsed data is to be validated by corresponding api method in noobit.exchanges.base.rest.api """ #! this is not correct anymore since we parse it after, #! so shoud return empty dict maybe if response is None: return OrdersList(data=[]) # response["result"] from kraken will be indexed with "open" or "closed" key according to request # check if we received a response for closed/open/single order try: key = list(response.keys())[0] except Exception as e: logging.error(stackprinter.format(e, style="darkbg2")) if key == "open": response = response["open"] elif key == "closed": response = response["closed"] else: # # the request is for a single order filtered by ID # response = response[key] pass try: parsed_orders = [ parse_single_order(key, info) for key, info in response.items() ] except Exception as e: logging.error(stackprinter.format(e, style="darkbg2")) #! validation has to take place in the base API file : we dont want to leave this reponsability to user implementation # try: # validated_data = OrdersList(data=parsed_orders) # except ValidationError as e: # raise e # return validated_data return parsed_orders
def aggregate_open_positions(response: dict): """Aggregate open_positions by side and pair Args: response (dict): data received from get_open_positions request Returns: dict: 2 keys cost (dict) : position cost (in quote currency) aggregated by side and pair format {"long" : {<pair>:<cost>}, "short": {...}} volume (dict) : position volume (in base currency) aggregated by side and pair format {"long" : {<pair>:<vol>}, "short": {...}} """ aggregated = {"cost": {"long": {}, "short": {}}, "volume": {"long": {}, "short": {}} } for pos_id, pos_info in response.items(): pair = pos_info["pair"] side = pos_info["type"] if side == "buy": try: aggregated["volume"]["long"][pair] += pos_info["vol"] except KeyError: aggregated["volume"]["long"][pair] = pos_info["vol"] except Exception as e: logging.error(stackprinter.format(e, style="darkbg2")) try: aggregated["cost"]["long"][pair] += pos_info["cost"] except KeyError: aggregated["cost"]["long"][pair] = pos_info["cost"] except Exception as e: logging.error(stackprinter.format(e, style="darkbg2")) if side == "sell": try: aggregated["volume"]["short"][pair] += pos_info["vol"] except KeyError: aggregated["volume"]["short"][pair] = pos_info["vol"] except Exception as e: logging.error(stackprinter.format(e, style="darkbg2")) try: aggregated["cost"]["short"][pair] += pos_info["cost"] except KeyError: aggregated["cost"]["short"][pair] = pos_info["cost"] except Exception as e: logging.error(stackprinter.format(e, style="darkbg2")) return aggregated
def make_eval_able(self, s): """returns a python object""" try: py_obj = eval(s) return py_obj except SyntaxError as exc: if len(s) < 200: return s else: with open('el155', 'a') as f: f.write(stackprinter.format()) f.write(stackprinter.format(exc)) f.write('\n----\n' + s + '\n') raise SystemExit
def _load_all_env_keys(self): """ Loads all keys from env file into cls.env_keys_dq Notes: In .env file, keys should contain : API Key : <exchange-name> & "key" API Secret : <exchange-name> & "secret" """ try: env_list_of_tuples = [] env_dict = list(dict(os.environ).items()) for k, v in env_dict: if self.exchange.upper() in k: if "KEY" in k: tuple_api_key = v env_secret_key = k.replace("KEY", "SECRET") tuple_secret_key = os.environ[env_secret_key] env_list_of_tuples.append( (tuple_api_key, tuple_secret_key)) self._set_class_var( deque(env_list_of_tuples, maxlen=len(env_list_of_tuples))) except Exception as e: logging.error(stackprinter.format(e, style="darkbg2"))
def main(**kwargs): # This wrapper prevents click from suppressing errors. try: sys.exit(datahub(standalone_mode=False, **kwargs)) except click.exceptions.Abort: # Click already automatically prints an abort message, so we can just exit. sys.exit(1) except click.ClickException as error: error.show() sys.exit(1) except Exception as exc: kwargs = {} sensitive_cause = SensitiveError.get_sensitive_cause(exc) if sensitive_cause: kwargs = {"show_vals": None} exc = sensitive_cause logger.error( stackprinter.format( exc, line_wrap=MAX_CONTENT_WIDTH, truncate_vals=10 * MAX_CONTENT_WIDTH, suppressed_paths=[r"lib/python.*/site-packages/click/"], **kwargs, )) logger.info( f"DataHub CLI version: {datahub_package.__version__} at {datahub_package.__file__}" ) logger.info( f"Python version: {sys.version} at {sys.executable} on {platform.platform()}" ) logger.info(f"GMS config {get_gms_config()}") sys.exit(1)
async def get_raw_trades(self, pair: list, since: int = None, retries: int = 0) -> dict: """Get raw trades data (not validated against data model) Args: pair (list): asset pair to get trade data for since (int): return trade data since given id (optional. exclusive) Returns: dict : 2 keys data : array of <price>, <volume>, <timestamp> last : array of <price>, <volume>, <timestamp> """ data = {"pair": pair, "since": since} response = await self.query_public(method="trades", data=data, retries=retries) # response is of format : # {"XXBTZUSD":[ # ["8862.70000","0.96511005",1583604462.0163,"b","l",""], # ................ # ["8895.20000","0.10000000",1583610370.7438,"b","m",""] # ], # "last":"1583610370743780877"} try: data = response[pair[0].upper()] last = response["last"] except Exception as e: logging.error(stackprinter.format(e, style="darkbg2")) return {"data": data, "last": last}
async def get_raw_spread(self, pair: list, since: int = None, retries: int = 0) -> dict: """ Get raw spread data (not validated against data model) Args: retries (int): pair (list): asset pair to get spread data for since (int): return spread data since given id (optional. inclusive) Returns: dict : keys data (list) : array of entries <time>, <bid>, <ask> last (Decimal) : id to be used as since when polling for new data """ data = {"pair": pair, "since": since} response = await self.query_public(method="spread", data=data, retries=retries) # response is of format : # {"XXBTZUSD":[ # [1583610484,"8887.90000","8889.60000"], # ................ # [1583610677,"8886.40000","8886.50000"] # ], # "last":1583610677} try: data = response[pair[0].upper()] last = response["last"] except Exception as e: logging.error(stackprinter.format(e, style="darkbg2")) return {"data": data, "last": last}
def _get_stack(exception, engine="better_exceptions"): assert engine in ["stackprinter", "better_exceptions"] if engine == "better_exceptions": return "".join(better_exceptions.format_exception(*exception)) elif engine == "stackprinter": return stackprinter.format(exception)
def setup(): import sys import os sys.path.insert(0, '/Users/alberthan/VSCodeProjects/HDLogger') hd_tracer = hdTracer(vars_to_watch) oldout, olderr = sys.stdout, sys.stderr sys.stdout = newout = io.StringIO() sys.stderr = newerr = io.StringIO() sys.settrace(hd_tracer.trace_dispatch) try: youtube_dl.main() # except SystemExit as err: # wf(stackprinter.format(err), 'logs/system_exit.expected_error.log', 'a') # print('--- expected SystemExit: \x1b[1;32mSuccess\x1b[0m') except: wf(stackprinter.format(sys.exc_info()), 'logs/error.except.main.log', 'a') raise finally: # hd_tracer.varswatcher.write_var_history() output, error = newout.getvalue(), newerr.getvalue() print(output) wf(output, 'logs/stdout.finally.log', 'a') wf(error, 'logs/stderr.finally.log', 'a') return (inspect.currentframe(), globals(), locals())
def _run_sketch(sketch_path, classpath, exit_if_error): if not jvm.is_jvm_running(): if classpath: jvm.add_classpath(classpath) jvm.add_jars(sketch_path.parent / 'jars') set_imported_mode(True) import py5 if (py5.is_running() if callable(py5.is_running) else py5.is_running): print( 'You must exit the currently running sketch before running another sketch.') return None py5_options_str = str( [f'--{o}' for o in py5_options]) if py5_options else 'None' sketch_args_str = str(sketch_args) with open(sketch_path, 'r', encoding='utf8') as f: sketch_code = _CODE_FRAMEWORK.format( f.read(), exit_if_error, py5_options_str, sketch_args_str) # does the code parse? if not, display an error message try: sketch_ast = ast.parse( sketch_code, filename=sketch_path, mode='exec') except IndentationError as e: msg = f'There is an indentation problem with your code on line {e.lineno}:\n' arrow_msg = f'--> {e.lineno} ' msg += f'{arrow_msg}{e.text}' msg += ' ' * (len(arrow_msg) + e.offset) + '^' print(msg) return except Exception as e: msg = stackprinter.format(e) m = re.search(r'^SyntaxError:', msg, flags=re.MULTILINE) if m: msg = msg[m.start(0):] msg = 'There is a problem with your code:\n' + msg print(msg) return problems = parsing.check_reserved_words(sketch_code, sketch_ast) if problems: msg = 'There ' + ('is a problem' if len(problems) == 1 else f'are {len(problems)} problems') + ' with your Sketch code' msg += '\n' + '=' * len(msg) + '\n' + '\n'.join(problems) print(msg) return sketch_compiled = compile( parsing.transform_py5_code(sketch_ast), filename=sketch_path, mode='exec') sys.path.extend([str(sketch_path.absolute().parent), os.getcwd()]) py5_ns = dict() py5_ns.update(py5.__dict__) py5_ns['__file__'] = str(original_sketch_path) exec(sketch_compiled, py5_ns)
def str_implies_container_BAK(self, s, depth, c=False): """returns newly formatted string | False ..args: s - must be a repr! """ depth += 2 join_str = f"\n{' '*depth}" # if isinstance(s,list) or isinstance(s,dict): # return self.dispatch_container_type(s,depth=depth,c=c) if len(s) < 80: return s elif "[" in s or "{" in s: py_obj = self.make_eval_able(s) if isinstance(py_obj, str): ns = py_obj if not c else py_obj return ns elif isinstance(py_obj, list): ns = self.process_list( py_obj, depth=depth) if not c else self.process_list( py_obj, depth=depth, c=True) nl = ["[", ns, "]"] return join_str.join(nl) elif isinstance(py_obj, dict): ns = self.process_dict( py_obj, depth=depth) if not c else self.process_dict( py_obj, depth=depth, c=True) nl = ["{", ns, "}"] return join_str.join(nl) else: with open('el04', 'a') as f: f.write(stackprinter.format()) f.write(s) raise SystemExit else: assert isinstance(s, str), info(s) return s
def setup_debug3(): try: youtube_dl.main() except: wf(stackprinter.format(sys.exc_info()), 'logs/error.except.main3.log', 'a') raise
def test_exception_formatting(): from source import Hovercraft try: Hovercraft().eels except: msg_plain = stackprinter.format() msg_color = stackprinter.format(style='darkbg') lines = msg_plain.split('\n') assert lines[0].endswith('eels') assert lines[-1] == 'Exception: ahoi!' print(msg_plain) print(msg_color)
def kraken_format_pair(normalized_pair: str): try: n_base, n_quote = normalized_pair.split("-") return f"X{n_base.upper()}Z{n_quote.upper()}" except Exception as e: logging.error(stackprinter.format(e, style="darkbg2"))
def format_record(record: dict) -> str: """ Custom format for loguru loggers. Uses pformat for log any data like request/response body during debug. Works with logging if loguru handler it. Example: >>> payload = [{"users":[{"name": "Nick", "age": 87, "is_active": True}, {"name": "Alex", "age": 27, "is_active": True}], "count": 2}] >>> logger.bind(payload=).debug("users payload") >>> [ { 'count': 2, >>> 'users': [ {'age': 87, 'is_active': True, 'name': 'Nick'}, >>> {'age': 27, 'is_active': True, 'name': 'Alex'}]}] """ format_string = LOGURU_FORMAT + "\n" if record["extra"].get("payload") is not None: record["extra"]["payload"] = pformat(record["extra"]["payload"], indent=2, compact=True, width=120) format_string += "\n<level>{extra[payload]}</level>" if record["exception"] is not None: record["extra"]["stack"] = stackprinter.format(record["exception"]) format_string += "{extra[stack]}\n" return format_string
async def subscribe(self, ping_interval: int, ping_timeout: int): """Subscribe to websocket """ self.ws = await websockets.connect(uri=self.ws_uri, ping_interval=ping_interval, ping_timeout=ping_timeout) ws_token = await self.api.get_websocket_auth_token() for feed in self.feeds: try: data = { "event": "subscribe", "subscription": { "name": feed, "token": ws_token['token'] } } payload = ujson.dumps(data) await self.ws.send(payload) await asyncio.sleep(0.1) except Exception as e: logging.error(stackprinter.format(e, style="darkbg2"))
async def serve(self, ping_interval: int = 60, ping_timeout: int = 30): async with websockets.connect(uri=self.ws_uri, ping_interval=ping_interval, ping_timeout=ping_timeout) as ws: ws_token = await self.api.get_websocket_auth_token() for feed in self.feeds: try: data = { "event": "subscribe", "subscription": { "name": feed, "token": ws_token['token'] } } payload = ujson.dumps(data) await ws.send(payload) await asyncio.sleep(0.1) except Exception as e: logging.error(stackprinter.format(e, style="darkbg2")) # async for msg in ws: # # await self.ws_msg_handler(msg) # print(msg) while 1: msg = await ws.recv() print(msg)
async def subscribe(self, ping_interval: int, ping_timeout: int): """Subscribe to websocket """ self.ws = await websockets.connect(uri=self.ws_uri, ping_interval=ping_interval, ping_timeout=ping_timeout) for feed in self.feeds: try: data = { "event": "subscribe", "pair": self.pairs, "subscription": { "name": feed } } if feed == "ohlc": data["subscription"]["interval"] = self.timeframe if feed == "book": data["subscription"]["depth"] = self.depth payload = ujson.dumps(data) await self.ws.send(payload) await asyncio.sleep(0.1) except Exception as e: logging.error(stackprinter.format(e, style="darkbg2"))
def format_exception( e: typing.Union[Exception, typing.Tuple[typing.Any, typing.Any, typing.Any]] ) -> typing.List[str]: tb = [] skip = 0 lines = stackprinter.format( e, line_wrap=0, truncate_vals=2000, suppressed_paths=SUPPRESSED_PATHS).splitlines() for i, line in enumerate(lines): if not skip: for s in SKIP_TB: if s.search(line): # if it's just a two line traceback we skip it if lines[i + 1].startswith(' ') and lines[ i + 2].startswith('File'): skip = 2 continue if skip: skip -= 1 continue tb.append(line) return tb
def fix_init(self, elm): new_filepath = Path(elm.filepath).stem assert new_filepath is not None, elm if "__init__" in new_filepath: lno = int(elm.line_number) def trunc(f): return f[lno - 3:lno + 3] def func(e): return e.filepath e, d, p, y = trnctd_fls = [trunc(f) for f in self.initfiles] try: if lno > 61: new_filepath = "__init__" elif e and any([func(elm) in itm for itm in e]): new_filepath = "e.__init__" elif d and any([func(elm) in itm for itm in d]): new_filepath = "d.__init__" elif p and any([func(elm) in itm for itm in p]): new_filepath = "p.__init__" elif y and any([func(elm) in itm for itm in y]): new_filepath = "__init__" except Exception as exc: tb = stackprinter.format(exc) print('The front fell off.' + tb) st() if new_filepath is None: st() return new_filepath
def main(**kwargs): # This wrapper prevents click from suppressing errors. try: sys.exit(datahub(standalone_mode=False, **kwargs)) except click.exceptions.Abort: # Click already automatically prints an abort message, so we can just exit. sys.exit(1) except click.ClickException as error: error.show() sys.exit(1) except Exception as exc: kwargs = {} sensitive_cause = SensitiveError.get_sensitive_cause(exc) if sensitive_cause: kwargs = {"show_vals": None} exc = sensitive_cause logger.error( stackprinter.format( exc, line_wrap=MAX_CONTENT_WIDTH, truncate_vals=10 * MAX_CONTENT_WIDTH, suppressed_paths=[r"lib/python.*/site-packages/click/"], **kwargs, )) sys.exit(1)
def __init__(self, price: float): try: price = Decimal(price) self.price = price.quantize(Decimal('0.01')) except Exception as e: print(stackprinter.format(e, style="darkbg2")) self.validated_data = self._validate()
def parse_single_position(key, value): info = value map_to_standard = settings.SYMBOL_MAP_TO_STANDARD["KRAKEN"] try: parsed_info = { "orderID": info["ordertxid"], "symbol": map_to_standard[info["pair"]], "currency": map_to_standard[info["pair"]].split("-")[1], "side": info["type"], "ordType": info["ordertype"], "clOrdID": None, "cashMargin": "margin", "ordStatus": "new", "workingIndicator": True, "transactTime": info["time"] * 10**9, "grossTradeAmt": info["cost"], "orderQty": info["vol"], "cashOrderQty": info["cost"], "cumQty": info["vol_closed"], "leavesQty": Decimal(info["vol"]) - Decimal(info["vol_closed"]), "marginRatio": Decimal(info["margin"]) / Decimal(info["cost"]), "marginAmt": info["margin"], "commission": info["fee"], "price": Decimal(info["cost"]) / Decimal(info["vol"]), "avgPx": None, # we need to request <docacls> to get this value "unrealisedPnL": info.get("net", None), "text": { "misc": info["misc"], "flags": info["oflags"] } } except Exception as e: logging.error(stackprinter.format(e, style="darkbg2")) return parsed_info # KRAKEN RESPONSE FORMAT (FROM DOC) # <position_txid> = open position info # ordertxid = order responsible for execution of trade # pair = asset pair # time = unix timestamp of trade # type = type of order used to open position (buy/sell) # ordertype = order type used to open position # cost = opening cost of position (quote currency unless viqc set in oflags) # fee = opening fee of position (quote currency) # vol = position volume (base currency unless viqc set in oflags) # vol_closed = position volume closed (base currency unless viqc set in oflags) # margin = initial margin (quote currency) # value = current value of remaining position (if docalcs requested. quote currency) # net = unrealized profit/loss of remaining position (if docalcs requested. quote currency, quote currency scale) # misc = comma delimited list of miscellaneous info # oflags = comma delimited list of order flags # viqc = volume in quote currency
def parse_orderbook(response): map_to_standard = settings.SYMBOL_MAP_TO_STANDARD["KRAKEN"] try: key = list(response.keys())[0] response = response[key] except Exception as e: logging.error(stackprinter.format(e, style="darkbg2")) try: parsed_orderbook = { "sendingTime": time.time_ns(), "symbol": map_to_standard[key], "asks": {item[0]: item[1] for item in response["asks"]}, "bids": {item[0]: item[1] for item in response["bids"]} } except Exception as e: logging.error(stackprinter.format(e, style="darkbg2")) return parsed_orderbook # KRAKEN RESPONSE FORMAT (DOC) # Result: array of pair name and market depth # <pair_name> = pair name # asks = ask side array of array entries(<price>, <volume>, <timestamp>) # bids = bid side array of array entries(<price>, <volume>, <timestamp>) # KRAKEN EXAMPLE RESULT # {"XXBTZUSD":{ # "asks":[ # ["9294.60000","1.615",1588792807], # ["9295.00000","0.306",1588792808] # ], # "bids":[ # ["9289.50000","1.179",1588792808], # ["9289.40000","0.250",1588792808], # ["9226.30000","2.879",1588792742]] # } # }