def aggregate_factory(ex_name, symbol, src_res, dst_res):
"""
aggregate data from src_res to dst_res, it reads metadata from the dst, and aggregate
available src_data up to date
"""
src_len = int(pd.Timedelta(dst_res) / pd.Timedelta(src_res))
logger = general.create_logger("compress_" + dst_res)
src = mongo._get_lib(src_res)
dst = mongo._get_lib(dst_res)
end = pd.Timestamp("now").floor(dst_res)
start = end - pd.Timedelta(dst_res)
rng = DateRange(start=start, end=end, interval=CLOSED_OPEN)
try:
db_sym = ex_name + "/" + symbol
logger.info(db_sym)
# check dst overlap
if dst.has_symbol(db_sym):
dst_end = pd.Timestamp(dst.read_metadata(db_sym).metadata["end"])
if dst_end >= start:
logger.warning("avoid overwriting [" +
start.strftime("%Y-%m-%d %H:%M:%S.%f"))
return
else:
start = max(dst_end + pd.Timedelta(dst_res), start)
rng = DateRange(start=start, end=end, interval=CLOSED_OPEN)
else:
start = pd.Timestamp("2019-01-01")
rng = DateRange(start=start, end=end, interval=CLOSED_OPEN)
# read src data
dfsrc = src.read(db_sym, date_range=rng).data
# validate src data
if dfsrc.empty:
logger.warning("empty data at " +
start.strftime("%Y-%m-%d %H:%M:%S.%f"))
return
if len(dfsrc) < src_len:
logger.warning("potential data loss at " +
start.strftime("%Y-%m-%d %H:%M:%S.%f"))
# aggregate src data to dst resolution
dfdst = compress.aggr(dfsrc, dst_res)
# validate dst data an save
if not dst.has_symbol(db_sym):
dst.write(db_sym, dfdst)
else:
dst.append(db_sym, dfdst)
# update metadata for end timestamp
dst.write_metadata(db_sym, {"end": dfdst.index[-1]})
logger.info("trades converted at " +
start.strftime("%Y-%m-%d %H:%M:%S.%f"))
except Exception as e:
print(e)
def __init__(self, exchange_id, resolution):
assert (exchange_id in config.SUPPORT_EXCHANGE)
self._exchange_id = exchange_id
self._imp = config.SUPPORT_EXCHANGE[self._exchange_id]
assert (resolution in config.RESOLUTION_PD2CCXT)
self._resolution = resolution
self._data_dst = mongo.get_lib(self._resolution)
self._logger = general.create_logger(self._exchange_id + "_" +
self._resolution)
# REST call interval, in second
self._pull_interval = 10
self._running = False
self.init_meta()
def aggregate_raw_exchange(ex_name: "exchange name"):
"""
This function aggregates raw trades of a given exchange. The symbols
to be processed are set in config file of the exchange.
:return:
"""
logger = general.create_logger("aggregate_raw_exchange_" + ex_name)
_1min = "1min"
ts_1min = pd.Timedelta(_1min)
end = pd.Timestamp("now").floor(_1min)
start = end - ts_1min
str_start = start.strftime("%Y-%m-%d %H:%M:00")
src = mongo.get_lib(ex_name, "raw")
dst = mongo.get_lib(ex_name, _1min)
for symbol in EXCHANGE_SYMBOLS[ex_name]:
try:
df = src.read(symbol,
date_range=DateRange(start, end, CLOSED_OPEN)).data
prev_close = get_prev_close(ex_name=ex_name,
symbol=symbol,
timestamp=start - ts_1min,
resolution=_1min)
df_agg = compress.aggregate_raw(parsed_df=df,
prev_close=prev_close,
resolution=_1min)
dst.append(symbol, df_agg)
dst.write_metadata(symbol,
metadata={
"last update": start,
"prev-close": df_agg["close"].last()
})
except Exception as e:
logger.warning(symbol + " raw aggregation failed at " + str_start)
logger.warning(e)
def __init__(self, exchange_id, on_message_fnc=None):
assert (exchange_id in config.config)
self._config = config.config[exchange_id]
assert ("url" in self._config)
preset_fnc = getattr(on_msg_fncs, exchange_id + "_on_message")
if preset_fnc:
self._on_message_fnc = preset_fnc
if on_message_fnc:
self._on_message_fnc = on_message_fnc
super().__init__(url=self._config["url"],
on_open=self.on_open,
on_message=self.on_message,
on_error=self.on_error,
on_close=self.on_close)
self.subbed_count = 0
self._data_dst = mongo.get_lib("trades")
self._logger = general.create_logger(self._config["exchange_id"])
self._keep_running = True
def mongo_buffer(delete_raw=False):
logger = general.create_logger("mongo_buffer")
end = pd.Timestamp("now").floor("1min")
start = end - pd.Timedelta("1min")
rng = DateRange(start, end, CLOSED_OPEN)
prev_min = start - pd.Timedelta("1min")
prev_min = int(prev_min.value/1e6)
for each in TRADE_EX:
bins = []
try:
libname = each + "/trades"
lib = mongo._get_lib(libname)
sym = 'btc/usdt'
if each == "coinbasepro":
sym = 'btc/usd'
df = lib.read(sym, date_range=rng).data
pprice = helper.get_close(prev_min, each)
logger.info(each)
# check empty
if df.empty:
logger.warning("empty data at " + start.strftime("%Y-%m-%d %H:%M:00"))
# compress
bins = compress.aggregate_raw(df, pprice)
print (bins["buy"], bins["sell"])
# save compressed
db_sym = each+"/XBTUSD"
mongo._get_lib("1min").append(db_sym, bins)
# delete raw
if delete_raw:
mongo.delete_cache(each, mongo.get_engine(TRADE_WSS_PATH), logger=logger, start=start, end=end)
logger.info(each + "trades converted at " + start.strftime("%Y-%m-%d %H:%M:00"))
except Exception as e:
print (e)
def save_trades_to_file(offset=0):
engine = get_engine(config.TRADE_WSS_PATH)
logger = general.create_logger("save_trades_to_file")
end = pd.Timestamp("now").floor("1D") - offset * pd.Timedelta("1D")
start = end - pd.Timedelta("1D")
for ex_name in config.TRADE_EX:
try:
expr = "SELECT * FROM " + ex_name + \
" WHERE time >= '" + start.strftime("%Y-%m-%d %H:%M:00") + "'" + \
"AND time < '" + end.strftime("%Y-%m-%d %H:%M:00") + "'"
df = pd.read_sql_query(expr, engine)
fname = ex_name + "_" + start.strftime("%Y-%m-%d")
full_fname = config.XBT_CSV_PATH + fname + ".csv"
if Path(full_fname).exists():
logger.warning(full_fname + " existed, renamed")
fname = fname + "_01"
full_fname = config.XBT_CSV_PATH + fname + ".csv"
df.to_csv(full_fname)
delete_cache(ex_name, engine, logger, end, start)
logger.info(full_fname + " has been saved and deleted from cache")
except Exception as e:
print("not saved")
import urllib
import pandas as pd
from btc_panel.utils import general
logger = general.create_logger(__file__)
def BIDASK_URL(ex_name, reso="minute", win_len="6h"):
return 'https://data.bitcoinity.org/export_data.csv?bp=10&bu=c¤cy=USD&data_type=bidask_sum' \
+ '&exchange=' + ex_name \
+ '&r=' + reso \
+ '&t=m×pan=' + win_len
def SPREAD_URL(reso="minute", win_len="6h"):
return 'https://data.bitcoinity.org/export_data.csv?c=e¤cy=USD&data_type=spread&f=m10' \
+ '&r=' + reso \
+ '&st=log&t=l×pan=' + win_len
@general.exception(logger)
def update_db_bidask(ex_name, engine):
url = BIDASK_URL(ex_name=ex_name)
ret = urllib.request.urlretrieve(url)
df_bidask = pd.read_csv(ret[0])
table_name = ex_name + '/bidask10'
if table_name not in engine.table_names():