def aggregate_factory(ex_name, symbol, src_res, dst_res): """ aggregate data from src_res to dst_res, it reads metadata from the dst, and aggregate available src_data up to date """ src_len = int(pd.Timedelta(dst_res) / pd.Timedelta(src_res)) logger = general.create_logger("compress_" + dst_res) src = mongo._get_lib(src_res) dst = mongo._get_lib(dst_res) end = pd.Timestamp("now").floor(dst_res) start = end - pd.Timedelta(dst_res) rng = DateRange(start=start, end=end, interval=CLOSED_OPEN) try: db_sym = ex_name + "/" + symbol logger.info(db_sym) # check dst overlap if dst.has_symbol(db_sym): dst_end = pd.Timestamp(dst.read_metadata(db_sym).metadata["end"]) if dst_end >= start: logger.warning("avoid overwriting [" + start.strftime("%Y-%m-%d %H:%M:%S.%f")) return else: start = max(dst_end + pd.Timedelta(dst_res), start) rng = DateRange(start=start, end=end, interval=CLOSED_OPEN) else: start = pd.Timestamp("2019-01-01") rng = DateRange(start=start, end=end, interval=CLOSED_OPEN) # read src data dfsrc = src.read(db_sym, date_range=rng).data # validate src data if dfsrc.empty: logger.warning("empty data at " + start.strftime("%Y-%m-%d %H:%M:%S.%f")) return if len(dfsrc) < src_len: logger.warning("potential data loss at " + start.strftime("%Y-%m-%d %H:%M:%S.%f")) # aggregate src data to dst resolution dfdst = compress.aggr(dfsrc, dst_res) # validate dst data an save if not dst.has_symbol(db_sym): dst.write(db_sym, dfdst) else: dst.append(db_sym, dfdst) # update metadata for end timestamp dst.write_metadata(db_sym, {"end": dfdst.index[-1]}) logger.info("trades converted at " + start.strftime("%Y-%m-%d %H:%M:%S.%f")) except Exception as e: print(e)
def __init__(self, exchange_id, resolution): assert (exchange_id in config.SUPPORT_EXCHANGE) self._exchange_id = exchange_id self._imp = config.SUPPORT_EXCHANGE[self._exchange_id] assert (resolution in config.RESOLUTION_PD2CCXT) self._resolution = resolution self._data_dst = mongo.get_lib(self._resolution) self._logger = general.create_logger(self._exchange_id + "_" + self._resolution) # REST call interval, in second self._pull_interval = 10 self._running = False self.init_meta()
def aggregate_raw_exchange(ex_name: "exchange name"): """ This function aggregates raw trades of a given exchange. The symbols to be processed are set in config file of the exchange. :return: """ logger = general.create_logger("aggregate_raw_exchange_" + ex_name) _1min = "1min" ts_1min = pd.Timedelta(_1min) end = pd.Timestamp("now").floor(_1min) start = end - ts_1min str_start = start.strftime("%Y-%m-%d %H:%M:00") src = mongo.get_lib(ex_name, "raw") dst = mongo.get_lib(ex_name, _1min) for symbol in EXCHANGE_SYMBOLS[ex_name]: try: df = src.read(symbol, date_range=DateRange(start, end, CLOSED_OPEN)).data prev_close = get_prev_close(ex_name=ex_name, symbol=symbol, timestamp=start - ts_1min, resolution=_1min) df_agg = compress.aggregate_raw(parsed_df=df, prev_close=prev_close, resolution=_1min) dst.append(symbol, df_agg) dst.write_metadata(symbol, metadata={ "last update": start, "prev-close": df_agg["close"].last() }) except Exception as e: logger.warning(symbol + " raw aggregation failed at " + str_start) logger.warning(e)
def __init__(self, exchange_id, on_message_fnc=None): assert (exchange_id in config.config) self._config = config.config[exchange_id] assert ("url" in self._config) preset_fnc = getattr(on_msg_fncs, exchange_id + "_on_message") if preset_fnc: self._on_message_fnc = preset_fnc if on_message_fnc: self._on_message_fnc = on_message_fnc super().__init__(url=self._config["url"], on_open=self.on_open, on_message=self.on_message, on_error=self.on_error, on_close=self.on_close) self.subbed_count = 0 self._data_dst = mongo.get_lib("trades") self._logger = general.create_logger(self._config["exchange_id"]) self._keep_running = True
def mongo_buffer(delete_raw=False): logger = general.create_logger("mongo_buffer") end = pd.Timestamp("now").floor("1min") start = end - pd.Timedelta("1min") rng = DateRange(start, end, CLOSED_OPEN) prev_min = start - pd.Timedelta("1min") prev_min = int(prev_min.value/1e6) for each in TRADE_EX: bins = [] try: libname = each + "/trades" lib = mongo._get_lib(libname) sym = 'btc/usdt' if each == "coinbasepro": sym = 'btc/usd' df = lib.read(sym, date_range=rng).data pprice = helper.get_close(prev_min, each) logger.info(each) # check empty if df.empty: logger.warning("empty data at " + start.strftime("%Y-%m-%d %H:%M:00")) # compress bins = compress.aggregate_raw(df, pprice) print (bins["buy"], bins["sell"]) # save compressed db_sym = each+"/XBTUSD" mongo._get_lib("1min").append(db_sym, bins) # delete raw if delete_raw: mongo.delete_cache(each, mongo.get_engine(TRADE_WSS_PATH), logger=logger, start=start, end=end) logger.info(each + "trades converted at " + start.strftime("%Y-%m-%d %H:%M:00")) except Exception as e: print (e)
def save_trades_to_file(offset=0): engine = get_engine(config.TRADE_WSS_PATH) logger = general.create_logger("save_trades_to_file") end = pd.Timestamp("now").floor("1D") - offset * pd.Timedelta("1D") start = end - pd.Timedelta("1D") for ex_name in config.TRADE_EX: try: expr = "SELECT * FROM " + ex_name + \ " WHERE time >= '" + start.strftime("%Y-%m-%d %H:%M:00") + "'" + \ "AND time < '" + end.strftime("%Y-%m-%d %H:%M:00") + "'" df = pd.read_sql_query(expr, engine) fname = ex_name + "_" + start.strftime("%Y-%m-%d") full_fname = config.XBT_CSV_PATH + fname + ".csv" if Path(full_fname).exists(): logger.warning(full_fname + " existed, renamed") fname = fname + "_01" full_fname = config.XBT_CSV_PATH + fname + ".csv" df.to_csv(full_fname) delete_cache(ex_name, engine, logger, end, start) logger.info(full_fname + " has been saved and deleted from cache") except Exception as e: print("not saved")
import urllib import pandas as pd from btc_panel.utils import general logger = general.create_logger(__file__) def BIDASK_URL(ex_name, reso="minute", win_len="6h"): return 'https://data.bitcoinity.org/export_data.csv?bp=10&bu=c¤cy=USD&data_type=bidask_sum' \ + '&exchange=' + ex_name \ + '&r=' + reso \ + '&t=m×pan=' + win_len def SPREAD_URL(reso="minute", win_len="6h"): return 'https://data.bitcoinity.org/export_data.csv?c=e¤cy=USD&data_type=spread&f=m10' \ + '&r=' + reso \ + '&st=log&t=l×pan=' + win_len @general.exception(logger) def update_db_bidask(ex_name, engine): url = BIDASK_URL(ex_name=ex_name) ret = urllib.request.urlretrieve(url) df_bidask = pd.read_csv(ret[0]) table_name = ex_name + '/bidask10' if table_name not in engine.table_names():