def process_signal_ticker(ticker, ref_ticker, maturity_suffix):
table_name = "signals" + "_" + ref_ticker + "_" + maturity_suffix
DB = Client(dbc)
list_tables = DB.list_tables()
# create table
if table_name not in list_tables:
DB.create_table(table_name, "ticker", "S", True, "trade_date", "S")
DB.add_index(table_name, "reverse", "trade_date", "S", "ticker", "S")
signal_table = Table(dbr.Table(table_name))
#batch calc_signals
selected_signals = select_signals(maturity_suffix,
param_rviv.pct_prefix_list,
param_rviv.median_prefix_list,
param_rviv.proba_1_prefix_list)
ref_data = query_rviv(ref_ticker)
df = calc_signals(ticker, ref_data, selected_signals)
signal_table.put_df_batch(df)
return df
def create_env(self, env_name, db_dict, db_schema):
(dbr, dbc, s3r, s3c, bucket) = self.init_aws_resources()
DB = Client(dbc)
DB.create_table(self.envs_table_name, "name", "S")
table_envs = Table(dbr.Table(self.envs_table_name))
env_item = {"name": env_name, "db_dict": db_dict}
table_envs.put_item(env_item)
for schema in db_schema.values():
DB.create_table(schema[0], schema[1], schema[2], schema[3],
schema[4], schema[5])
#DB.add_index("data", "reverse", "trade_date", "S", "ref", "S")
# DB.add_index("ticker_expis", "env-expi-index", "env", "S", "expi", "S")
# DB.add_index("ticker_expis", "env-ticker-index", "env", "S", "ticker", "S")
return dbr, dbc, s3r, s3c, bucket, db_dict
def select_env(self, env_name):
(dbr, dbc, s3r, s3c, bucket) = self.init_aws_resources()
DB = Client(dbc)
tables = DB.list_tables()
if "envs" not in tables:
raise ValueError("You need to create envs table")
table_envs = Table(dbr.Table(self.envs_table_name))
existing_envs = table_envs.scan()
names = [e["name"] for e in existing_envs]
if env_name in names:
env = table_envs.get_item("name", env_name)
db_dict = env["db_dict"]
return dbr, dbc, s3r, s3c, bucket, db_dict
else:
raise ValueError("You need to create this env")
def load_ticker_df(table_name, ticker, start, end):
items = Table(dbr.Table(table_name)).query_range_between(
"ticker", ticker, "trade_date", start, end)
dates = [i["trade_date"] for i in items]
if start in dates:
return pd.DataFrame(items)
else:
return pd.DataFrame([])
def process_signal_all(ref_ticker, maturity_suffix):
table_name = "signals" + "_" + ref_ticker + "_" + maturity_suffix
DB = Client(dbc)
list_tables = DB.list_tables()
# create table
if table_name not in list_tables:
DB.create_table(table_name, "ticker", "S", True, "trade_date", "S")
DB.add_index(table_name, "reverse", "trade_date", "S", "ticker", "S")
signal_table = Table(dbr.Table(table_name))
#batch calc_signals
selected_signals = select_signals(maturity_suffix,
param_rviv.pct_prefix_list,
param_rviv.median_prefix_list,
param_rviv.proba_1_prefix_list)
ref_data = query_rviv(ref_ticker)
te_table = Table(dbr.Table(db_dict["vols_table"]))
tickers = list(set([item["ticker"] for item in te_table.scan()]))
for t in tickers:
df = calc_signals(t, ref_data, selected_signals)
#write them
signal_table.put_df_batch(df)
def add_table(self,
env_name,
table_key,
table_name,
hash_name,
hash_type='S',
has_range=False,
range_name=None,
range_type='S'):
(dbr, dbc, s3r, s3c, bucket) = self.init_aws_resources()
DB = Client(dbc)
tables = DB.list_tables()
if "envs" not in tables:
raise ValueError("You need to create envs table")
table_envs = Table(dbr.Table(self.envs_table_name))
existing_envs = table_envs.scan()
names = [e["name"] for e in existing_envs]
if env_name in names:
env = table_envs.get_item("name", env_name)
db_dict = env["db_dict"]
if table_name in db_dict.values():
raise ValueError("This table exists already")
else:
try:
DB.create_table(table_name, hash_name, hash_type,
has_range, range_name, range_type)
except:
print("unable to create table")
db_dict[table_key] = table_name
env["db_dict"] = db_dict
table_envs.put_item(env)
else:
raise ValueError("This env does not exist")
def process_ts_ticker(ticker, strikes, maturity_suffix, create_tables=False):
errors_table = Table(dbr.Table("ts_errors"))
table_names = [
derive_table_name(strike, maturity_suffix) for strike in strikes
]
if create_tables:
DB = Client(dbc)
list_tables = DB.list_tables()
for table_name in table_names:
if table_name not in list_tables:
DB.create_table(table_name, "ticker", "S", True, "trade_date",
"S")
DB.add_index(table_name, "reverse", "trade_date", "S",
"ticker", "S")
# batch calc_store_ts
try:
output, _ = ticker_bo_ts(ticker, strikes, maturity_suffix,
all_dates[0], all_dates[-1],
timeserie.strike_change_threshold)
for i, strike in enumerate(strikes):
try:
output[strike]["ticker"] = ticker
ts_table = Table(dbr.Table(table_names[i]))
ts_table.put_df_batch(output[strike])
except Exception as ee:
errors_table.put_item({
"ticker": ticker,
"error": str(strike),
"exception": str(ee)
})
# print(str(ee))
except Exception as e:
errors_table.put_item({
"ticker": ticker,
"error": "ts_calc",
"exception": str(e)
})
def select_signal_sequence(date, table_name, list_signals, list_num_stocks,
list_largest, excluding_list):
signals = pd.DataFrame(
Table(dbr.Table(table_name)).query_index("reverse", "trade_date",
date))
if len(signals) > 0:
for i, s in enumerate(list_signals):
signals = signals[~signals.ticker.isin(excluding_list)]
signals = signals.sort_values(
by=s, ascending=~list_largest[i]).head(list_num_stocks[i])
return list(signals.ticker)
else:
return []
import pandas as pd
from src.library.dynamo.Client import Client
from src.library.dynamo.Table import Table
from src.library.helpers.dates import get_dates
from src.library.helpers.general import bring_columns_first
from src.library.osmv.Osmv import Osmv
from src.library.params import param
from src.library.params import rviv as param_rviv
osmv = Osmv(param.IS_LOCAL, param.BUCKET_NAME)
(dbr, dbc, s3r, s3c, bucket, db_dict) = osmv.select_env(param.ENV_USED)
(list_of_dates, list_obj_dates) = get_dates()
DB = Client(dbc)
data_table = Table(dbr.Table(db_dict["data_table"]))
vols_table = Table(dbr.Table(db_dict["vols_table"]))
rv_table = Table(dbr.Table(db_dict["rv_table"]))
iv_table = Table(dbr.Table(db_dict["iv_table"]))
iv_windows = param_rviv.IV_WINDOWS
AF = param_rviv.AF
rv_windows = {("rv_" + k): int(AF / 365 * v) for k, v in iv_windows.items()}
rv_cols = list(rv_windows.keys())
def query_rviv(ticker):
rv = pd.DataFrame(rv_table.query("ticker", ticker)).set_index("trade_date")
iv = pd.DataFrame(iv_table.query("ticker", ticker)).set_index("trade_date")
df = pd.concat((rv, iv), axis=1)
def batch_write_ivs():
te_table = Table(dbr.Table(db_dict["vols_table"]))
tickers = list(set([item["ticker"] for item in te_table.scan()]))
for t in tickers:
lw_date = get_last_written_date(t)
write_ticker_iv_date(t, lw_date)
import pandas as pd
import src.library.params.param as param
from src.library.dynamo.Client import Client
from src.library.dynamo.Table import Table
from src.library.helpers.dates import get_dates
from src.library.osmv.Osmv import Osmv
from src.library.params import rviv as param_rviv
osmv = Osmv(param.IS_LOCAL, param.BUCKET_NAME)
(dbr, dbc, s3r, s3c, bucket, db_dict) = osmv.select_env(param.ENV_USED)
(list_of_dates, list_obj_dates) = get_dates()
DB = Client(dbc)
vols_table = Table(dbr.Table(db_dict["vols_table"]))
iv_table = Table(dbr.Table(db_dict["iv_table"]))
iv_windows = param_rviv.IV_WINDOWS
list_iv_matus = list(iv_windows.values())
def query_vols(ticker, date=None):
if date is None:
return vols_table.query("ticker", ticker)
else:
return vols_table.query_range_gt_than("ticker", ticker, "trade_date",
date)
def transform_iv_dict(r):
import src.library.params.param as param
from src.library.dynamo.Table import Table
from src.library.helpers.dates import get_dates
from src.library.helpers.general import df_decimal_to_float
from src.library.importer.filters import filter_first_refs
from src.library.importer.filters import filter_new_refs
from src.library.importer.tweak import calc_pnl
from src.library.importer.tweak import tweak
from src.library.osmv.Osmv import Osmv
from src.library.signals.vol import populate_expis
osmv = Osmv(param.IS_LOCAL, param.BUCKET_NAME)
(dbr, dbc, s3r, s3c, bucket, db_dict) = osmv.select_env(param.ENV_USED)
(list_of_dates, list_obj_dates) = get_dates()
data_table = Table(dbr.Table(db_dict["data_table"]))
vols_table = Table(dbr.Table(db_dict["vols_table"]))
refs_table = Table(dbr.Table(db_dict["state_refs_table"]))
te_table = Table(dbr.Table(db_dict["ticker_expis_table"]))
errors_table = Table(dbr.Table(db_dict["errors_table"]))
import logging
import tracemalloc
log_format = "%(asctime)s::%(levelname)s::%(name)s::" \
"%(filename)s::%(lineno)d::%(message)s"
logging.basicConfig(filename='logs.log', level='INFO', format=log_format)
logger = logging.getLogger('server_logger')
from datetime import datetime
import pandas as pd
import src.library.params.param as param
from src.library.dynamo.Table import Table
from src.library.osmv.Osmv import Osmv
from src.library.params import columns
osmv = Osmv(param.IS_LOCAL, param.BUCKET_NAME)
(dbr, dbc, s3r, s3c, bucket, db_dict) = osmv.select_env(param.ENV_USED)
splits_table = Table(dbr.Table(db_dict["splits_table"]))
def stock_split_ref(df, stock_split, ref, prev_df):
strike = float(ref[(ref.rfind('-')+1):len(ref)])
available_strikes = set(df.strike)
adj_strike = find_closest_strike(strike, stock_split, available_strikes)
df_k = df[df.strike == adj_strike]
prev_df_k = prev_df[prev_df.strike == strike]
df_k = calc_pnl(prev_df_k, df_k, stock_split)
return df_k, adj_strike
def calc_pnl(prev_df_k, df_k, stock_split):
col_moves = ["spot", "i_rate", "c_mid", "p_mid", "yte", "c_iv_mid", "p_iv_mid", "osmv_vol"]
ss_moves = ["spot", "c_mid", "p_mid"]
df_prev = pd.DataFrame(prev_df_k).set_index("expi")
df_prev.columns = [str(col) + '_prev' for col in df_prev.columns]
df = pd.DataFrame(df_k).set_index("expi")
from src.library.dynamo.Table import Table
from src.library.osmv.Osmv import Osmv
from src.library.params import param
osmv = Osmv(param.IS_LOCAL, param.BUCKET_NAME)
(dbr, dbc, s3r, s3c, bucket, db_dict) = osmv.select_env(param.ENV_USED)
state_refs_table = Table(dbr.Table(db_dict["state_refs_table"]))
te_table = Table(dbr.Table(db_dict["ticker_expis_table"]))
def list_tickers(expi=None):
if expi:
items = te_table.query_index("env-expi-index", "env", param.ENV_USED,
True, "expi", expi)
else:
items = te_table.query_index("env-expi-index", "env", param.ENV_USED)
return sorted(set([item["ticker"] for item in items]))
def list_expis(ticker=None):
if ticker:
items = te_table.query_index("env-ticker-index", "env", param.ENV_USED,
True, "ticker", ticker)
else:
items = te_table.query_index("env-ticker-index", "env", param.ENV_USED)
return sorted(set([item["expi"] for item in items]))
# paginator = s3c.get_paginator('list_objects_v2')
# pages = paginator.paginate(Bucket='osmv-2', Prefix='prefix')
# for page in pages:
for obj in bucket.objects.all():
tmp = obj.key.encode('utf-8')[-12:-4]
list_obj_dates.append(tmp)
for elt in range(len(list_obj_dates)):
var = list_obj_dates[elt]
tmp = (datetime.datetime.strptime(var.decode("utf-8")), "%Y%m%d").date()
list_of_dates.append(tmp)
#quandl_rebal_dates = rebal_fixed_strike(list_of_dates)
data_table = Table(dbr.Table(db_dict["data_table"]))
threshold_default = timeserie.strike_change_threshold
keep_cols = timeserie.KEEP_COLS
common_cols = timeserie.COMMON_TS_COLS
expi_cols = timeserie.EXPI_TS_COLS
alpha_cols = timeserie.ALPHA_TS_COLS
base_cols = timeserie.BASE_COLS
expi_threshold = timeserie.expi_threshold
max_distance_one_expi = timeserie.max_distance_one_expi
delta_threshold = timeserie.delta_threshold
bidask_cap = timeserie.bidask_cap