def rt_arrow():
try:
logger.info("<rt_arrow>")
adb_data = rock.rocks("tsdbrocks", read_only=True).get("tsdb_data")
adb_data["adb_timestamp"] = int(time.time())
rock.rocks("adbrocks").put("adb_data", adb_data)
logger.info("</rt_arrow>")
except Exception as ex:
logger.exception(ex, "Root rt_arrow exception handler:")
time.sleep(2)
def ts_rt():
"""
Pump rt data down the pipeline.
"""
logger.info("<ts_rt>")
global rtdb_keys
tsdb_data = {}
for key in rtdb_keys:
tsdb_data[key] = rock.rocks("rtdb", read_only=True).get(key)
tsdb_data["tsdb_timestamp"] = int(time.time())
rock.rocks("tsdbrocks").put("tsdb_data", tsdb_data)
logger.info("</ts_rt>")
def ts_hourly():
logger.info("<ts_hourly>")
now = datetime.datetime.now().replace(minute=0, second=0, microsecond=0)
for key in rtdb_keys:
try:
val = rock.rocks("rtdb", read_only=True).get(key)
fname = f"/data/tsdb/hourly/{key}.parq"
if os.path.exists(fname):
df = pd.read_parquet(fname)
nvals = len(df)
if type(val) == dict:
nowpie = val
nowpie["date"] = now
else:
nowpie = {"date": now, "value": val}
df = df.append(nowpie, ignore_index=True)
df.date = pd.to_datetime(df.date)
df = df.dropna()
df.drop_duplicates(subset=["date"], keep="first", inplace=True)
if len(df) != nvals:
racoon.to_parquet(df, fname)
else:
logger.info("Create: " + fname)
if type(val) == dict:
nowpie = {k: [v] for k, v in val.items()}
nowpie["date"] = [now]
else:
nowpie = {"date": [now], "value": [val]}
df = pd.DataFrame(nowpie)
df.date = pd.to_datetime(df.date)
racoon.to_parquet(df, fname)
except Exception as ex:
logger.info(f"ts_hourly.exception.{key}")
logger.exception(ex)
logger.info("</ts_hourly>")
def get_adb_data():
"""
Rate limit the hit squad.
"""
adb_rocks = rock.rocks("adbrocks", read_only=True)
adb_data = adb_rocks.get("adb_data")
adb_data["color_index"] = adb_rocks.get("color_index")
return adb_data
def test_hammer_read_only(self):
N = 60
try:
started = time.time()
os.makedirs("/data/" + TestRock.DBNAME, mode=0o770, exist_ok=True)
for idx in range(N):
rock.rocks(TestRock.DBNAME).put(f"key{idx}", idx)
test_result = True
for idx in range(N):
val = rock.rocks(TestRock.DBNAME, read_only=True).get(f"key{idx}")
test_result = test_result and (val == idx)
if not test_result:
print(idx, val)
elapsed = time.time() - started
if elapsed >= 8:
print("8 second timeout on testrocks. ;?")
assert False
except Exception as ex:
print(ex)
raise ex
def day_arrow():
"""
Daily spot model.
"""
logger.info("<day_spot_model>")
fn = "/data/tsdb/daily/spot_xaubtc.parq"
df = pd.read_parquet(fn)
df.columns = ["date", "spot"]
df["time"] = range(len(df))
df["log1p_spot"] = df.spot.apply(np.log1p)
aa, bb, cc, dd, ee = np.polyfit(df.time, df.log1p_spot, 4)
labels = [f"spot_model_{x}" for x in range(6)]
log_pads = [-0.80, -0.40, 0.0, 0.5, 1, 1.2]
exp_pads = [0.662, 0.496, 0.248, -0.244, -1.044, -1.512]
#log_pads = [-0.80, -0.40, 0.0, 0.5, 1.0, 1.1]
#exp_pads = [0.66, 0.5, 0.25, -0.2, -1.0, -1.1]
for label, lpad, epad in zip(labels, log_pads, exp_pads):
log1p_spot_model = aa * df.time**4 + bb * df.time**3 + cc * df.time**2 + dd * df.time + ee
log1p_spot_model += lpad
spot_model = np.exp(log1p_spot_model) - 1
spot_model += epad
df[f"{label}"] = spot_model
df[f"{label}"] = df[f"{label}"].apply(lambda x: max(x, 0.001))
spot_model_bands = df[labels].tail(1).values.tolist()[0]
spot = df.spot.iloc[-1]
spot_model_band = stats.closest(spot_model_bands, spot)
color_index = spot_model_bands.index(spot_model_band)
rock.rocks("adbrocks").put("spot_model_bands", spot_model_bands)
rock.rocks("adbrocks").put("color_index", color_index)
racoon.to_csv(df, "/data/adbcsv/spot_model_xaubtc_daily.csv")
logger.info("</day_spot_model>")
def peek():
"""
Interactive debugging in prod...
"""
return rock.rocks("adbrocks", read_only=True).get("adb_data")
def write_rt():
"""
Write real-time data.
"""
try:
# Gold Spots:
spots_usdxau = get_spots("gold")
#logger.info("spots_usdxau" + str(spots_usdxau))
spot_usdxau = stats.robust_mean(spots_usdxau)
# Bitcoin Spots:
spots_usdbtc = get_spots("bitcoin")
#logger.info("spots_usdbtc" + str(spots_usdbtc))
spot_usdbtc = stats.robust_mean(spots_usdbtc)
# Average Spot:
spot_xaubtc = spot_usdbtc / spot_usdxau
spot_btcxau = 1.0 / spot_xaubtc
# Purge terrorism units:
spots_xaubtc = [x / spot_usdxau for x in spots_usdbtc]
spots_btcxau = [1.0 / x for x in spots_xaubtc]
# System Stats:
blockchain_info = bitcoin.bitcoincli.blockchaininfo()
bitcoin_info = bitcoin.bitcoincli.info()
blockchain_stats = bitcoin.blockchain.stats()
# Storage:
dats = []
keys = []
for source, source_spot in zip(gold.spot_source_names, spots_btcxau):
key = f"{source}_spot_btcxau"
keys += [key]
dats += [(key, source_spot)]
for source, source_spot in zip(bitcoin.spot_source_names,
spots_xaubtc):
key = f"{source}_spot_xaubtc"
keys += [key]
dats += [(key, source_spot)]
extra_dats = [
("spot_usdxau", spot_usdxau),
("spot_usdbtc", spot_usdbtc),
("spot_btcxau", spot_btcxau),
("spot_xaubtc", spot_xaubtc),
("blockchain_info", blockchain_info),
("bitcoin_info", bitcoin_info),
("blockchain_stats", blockchain_stats),
("rtdb_timestamp", int(time.time())),
]
dats += extra_dats
keys += [x[0] for x in extra_dats]
dats += [
("keys", keys),
]
for dat in dats:
#logger.info("rtdb.put: " + dat[0])
try:
rock.rocks("rtdb").put(dat[0], dat[1])
except rock.rocksdb.errors.RocksIOError as ex:
pass
except Exception as ex:
time.sleep(2)
logger.exception(ex)
def peek():
"""
Interactive debugging in prod...
"""
return rock.rocks("rtdb", read_only=True).get("spot_xaubtc")
"""
import datetime
import math
import os
import time
import numpy as np
import pandas as pd
from util import logger
from util import racoon
from util import rock
from util import stats
tsdb_keys = list(rock.rocks("tsdbrocks").get("tsdb_data").keys())
gold_spot_keys = [x for x in tsdb_keys if "_spot_btcxau" in x]
bitcoin_spot_keys = [x for x in tsdb_keys if "_spot_xaubtc" in x]
keys = tsdb_keys.copy()
def gold_minute_arrow():
"""
Maintain minutely spot aggregates for gold.
"""
global gold_spot_keys
logger.info("<gold_spots_minutely>")
gold_spot_keys = gold_spot_keys.copy()
df = pd.read_parquet(f"/data/tsdb/minutely/{gold_spot_keys[0]}.parq")
import pandas as pd
import schedule
from util import logger
from util import racoon
from util import rock
def peek():
"""
Interactive debugging in prod...
"""
return rock.rocks("tsdbrocks", read_only=True).get("tsdb_data")
rtdb_keys = rock.rocks("rtdb", read_only=True).get("keys")
periods = ["minutely", "hourly", "daily"]
for period in periods:
path = f"/data/tsdb/{period}"
if not os.path.exists(path):
os.makedirs(path, mode=0o770, exist_ok=True)
def ts_rt():
"""
Pump rt data down the pipeline.
"""
logger.info("<ts_rt>")
global rtdb_keys
tsdb_data = {}
