def print_split_report(self, stk, dt, ratio):
print(f'{stk} {dt} {ratio}')
try:
s_date = stxcal.move_busdays(dt, -10)
e_date = stxcal.move_busdays(dt, 10)
ts = StxTS(stk, s_date, e_date)
ts.set_day(e_date, c=-1)
print(f'{ts.df}')
except:
logging.error(f'print_split_report failed for {stk}, {dt}, '
f'{ratio}')
tb.print_exc()
def __init__(self, max_atm_price=5.0, num_stx=150):
self.tbl_name = 'eods'
self.opt_tbl_name = 'opt_cache'
self.ldr_tbl_name = 'leaders'
self.setup_tbl_name = 'setups'
self.exclude_tbl_name = 'exclusions'
self.sql_create_opt_tbl = "CREATE TABLE {0:s} ("\
'expiry date NOT NULL,'\
'und character varying(16) NOT NULL,'\
'cp character varying(1) NOT NULL,'\
'strike numeric(10,2) NOT NULL,'\
'dt date NOT NULL,'\
'bid numeric(10,2),'\
'ask numeric(10,2),'\
'volume integer,'\
'PRIMARY KEY (expiry,und,cp,strike,dt)'\
')'.format(self.opt_tbl_name)
self.sql_create_ldr_tbl = "CREATE TABLE {0:s} ("\
"exp date NOT NULL,"\
"stk varchar(16) NOT NULL,"\
"activity integer DEFAULT NULL,"\
"opt_spread integer DEFAULT NULL,"\
"atm_price numeric(6,2) DEFAULT NULL,"\
"PRIMARY KEY (exp,stk)"\
")".format(self.ldr_tbl_name)
self.sql_create_setup_tbl = "CREATE TABLE {0:s} ("\
"stk varchar(8) NOT NULL,"\
"dt date NOT NULL,"\
"setup varchar(80) NOT NULL,"\
"PRIMARY KEY (stk,dt)"\
")".format(self.setup_tbl_name)
self.sql_create_exclude_tbl = "CREATE TABLE {0:s} ("\
"stk varchar(8) NOT NULL,"\
"PRIMARY KEY (stk)"\
")".format(self.exclude_tbl_name)
stxdb.db_create_missing_table(self.opt_tbl_name,
self.sql_create_opt_tbl)
stxdb.db_create_missing_table(self.ldr_tbl_name,
self.sql_create_ldr_tbl)
stxdb.db_create_missing_table(self.setup_tbl_name,
self.sql_create_setup_tbl)
stxdb.db_create_missing_table(self.exclude_tbl_name,
self.sql_create_exclude_tbl)
# calculate the last date for which we have historical options
prev_year = datetime.datetime.now().date().year - 1
self.last_opt_date = stxcal.move_busdays(
'{0:d}-12-31'.format(prev_year), 0)
self.ts_dct = {}
self.start_date = '1985-01-01'
self.end_date = stxcal.move_busdays(
str(datetime.datetime.now().date()), 1)
self.max_atm_price = max_atm_price
self.num_stx = num_stx
def eow_job(self):
print('247 end of week job')
print('247 end of week job')
ana_date = '2002-04-19'
crt_date = '2002-07-19'
max_dt_q = stxdb.db_read_cmd('select max(dt) from leaders')
if max_dt_q[0][0] is not None:
ana_date = str(max_dt_q[0][0])
crt_date = str(datetime.datetime.now().date())
self.end_date = stxcal.move_busdays(
str(datetime.datetime.now().date()), 1)
while ana_date <= crt_date:
self.get_liq_leaders(ana_date)
self.get_opt_spread_leaders(ana_date)
ana_date = stxcal.move_busdays(stxcal.next_expiry(ana_date), 0)
def test_4_move_busdays(self):
res1 = stxcal.move_busdays(self.dt2, -1)
res2 = stxcal.move_busdays(self.dt2, 1)
res3 = stxcal.move_busdays(self.dt2, 0)
res4 = stxcal.move_busdays(self.dt3, -1)
res5 = stxcal.move_busdays(self.dt3, 0)
res6 = stxcal.move_busdays(self.dt3, 1)
res7 = stxcal.move_busdays(self.dt1, 7)
res8 = stxcal.move_busdays(self.dt5, -7)
self.assertTrue((res1 == self.dt1) and (res2 == '2016-12-19') and
(res3 == self.dt2) and (res4 == self.dt2) and
(res5 == self.dt2) and (res6 == res2) and
(res7 == self.dt5) and (res8 == self.dt1))
def find_all_liquid_stocks_as_of(self, selected_date):
res = []
q = "select * from eods where date = '{0:s}'".format(selected_date)
df = pd.read_sql(q, stxdb.db_get_cnx())
print('Found {0:d} stocks'.format(len(df)))
df['rg'] = df['hi'] - df['lo']
df_1 = df.query('volume>1000 & c>30 & rg>0.015*c')
stx = df_1['stk'].tolist()
print('Found {0:d} leaders'.format(len(stx)))
start_date = stxcal.move_busdays(selected_date, -60)
print('start_date is: {0:s}'.format(str(start_date)))
ixx = 0
for stk in stx:
ixx += 1
ts = StxTS(stk, start_date, selected_date)
# adjust the whole thing for splits, etc.
ts.set_day(str(ts.df.index[-1].date()))
ts.df['hi_1'] = ts.df['hi'].shift(1)
ts.df['lo_1'] = ts.df['lo'].shift(1)
ts.df['rg'] = ts.df['hi'] - ts.df['lo']
ts.df['act'] = ts.df['volume'] * ts.df['c']
ts.df['avg_v'] = ts.df['volume'].rolling(50).mean()
ts.df['avg_c'] = ts.df['c'].rolling(50).mean()
ts.df['avg_rg'] = ts.df['rg'].rolling(50).mean()
ts.df['avg_act'] = ts.df['act'].rolling(50).mean()
rec = ts.df.ix[-1]
if rec.avg_v > 2000 and rec.avg_c > 40 and \
rec.avg_act > 100000 and rec.avg_rg > 0.015 * rec.avg_c:
res.append(stk)
sc = StxCandles(stk)
setup_ts = sc.calculate_setups(sd=start_date)
setups = [
'gap', 'marubozu', 'hammer', 'doji', 'engulfing',
'piercing', 'harami', 'star', 'engulfharami', 'three_m',
'three_in', 'three_out', 'up_gap_two_crows'
]
with open('/home/cma/setups/{0:s}.csv'.format(stk), 'w') as f:
for index, row in setup_ts.df.iterrows():
f.write('{0:s};'.format(str(index.date())))
for setup in setups:
if row[setup] != 0:
f.write(' {0:s}: {1:.0f} '.format(
setup.upper(), row[setup]))
f.write('\n')
if ixx == len(stx) or ixx % 50 == 0:
print('Processed {0:d} leaders'.format(ixx))
print('Found {0:d} super leaders'.format(len(res)))
return res
def get_opts(stx=None):
stxdb.db_create_missing_table('options', sql_create_options)
stx_list = [] if stx is None else stx.split(',')
dt = stxcal.move_busdays(datetime.strftime(datetime.now(), '%Y-%m-%d'), 0)
l_exps = stxcal.long_expiries()
c = pycurl.Curl()
c.setopt(pycurl.SSL_VERIFYPEER, 0)
c.setopt(pycurl.SSL_VERIFYHOST, 0)
first = True
for stk in stx_list:
y_exps = get_chain(c, stk, l_exps[0], dt, 'w' if first else 'a')
first = False
exps = [val for val in l_exps if val in y_exps]
for exp in exps[1:]:
get_chain(c, stk, exp, dt, 'a')
stxdb.db_upload_file(file_name, 'options', 5)
def get_liq_leaders(self, ana_date, min_act=80000, min_rcr=0.015):
stk_list = stxdb.db_read_cmd(
"select distinct stk from eods where "
"date='{0:s}' order by stk".format(ana_date))
all_stocks = [
s[0] for s in stk_list if re.match(r'^[A-Za-z]', str(s[0]))
]
print('Found {0:d} stocks for {1:s}'.format(len(all_stocks), ana_date))
next_exp = stxcal.next_expiry(ana_date)
next_exp_busday = stxcal.move_busdays(next_exp, 0)
num_stx = 0
num = 0
liq_leaders = []
for s in all_stocks:
num += 1
ts = self.ts_dct.get(s)
if ts is None:
ts = StxTS(s, self.start_date, self.end_date)
ts.set_day(str(ts.df.index[-1].date()))
ts.df['activity'] = ts.df['volume'] * ts.df['c']
ts.df['avg_act'] = ts.df['activity'].rolling(50).mean()
ts.df['rg'] = ts.df['hi'] - ts.df['lo']
ts.df['avg_rg'] = ts.df['rg'].rolling(50).mean()
ts.df['rg_c_ratio'] = ts.df['avg_rg'] / ts.df['c']
self.ts_dct[s] = ts
num_stx += 1
stk_act = [s]
if self.is_liq_leader(ts, ana_date, min_act, min_rcr, stk_act):
liq_leaders.append(stk_act)
if num % 1000 == 0 or num == len(all_stocks):
print('Processed {0:d} stocks, found {1:d} liquidity leaders'.
format(num, len(liq_leaders)))
print('Found {0:d} liquidity leaders for {1:s}'.format(
len(liq_leaders), ana_date))
print('Loaded {0:d} stocks for {1:s}'.format(num_stx, ana_date))
cnx = stxdb.db_get_cnx()
with cnx.cursor() as crs:
for ldr in liq_leaders:
crs.execute(
'insert into leaders(exp,stk,activity,opt_spread) values '
+ crs.mogrify('(%s,%s,%s,%s)',
[next_exp, ldr[0],
int(ldr[1]), -1000]) +
'on conflict do nothing')
def get_indices(self):
indices = {'^DJI': '^DJI', '^SPX': '^SPX', '^NDQ': '^COMP'}
end_date = stxcal.move_busdays(str(datetime.datetime.now().date()), 0)
for stooq_name, db_name in indices.items():
db_cmd = "select max(date) from eods where stk='{0:s}' and " \
"volume > 1".format(db_name)
res_db = stxdb.db_read_cmd(db_cmd)
start_date = stxcal.next_busday(str(res_db[0][0]))
res = requests.get(
'https://stooq.com/q/d/l?s={0:s}&d1={1:s}&d2={2:s}'.
format(stooq_name,
start_date.replace('-', ''),
end_date.replace('-', '')))
lines = res.text.split('\n')
for line in lines[1:]:
tokens = line.split(',')
with open('{0:s}.csv'.format(db_name), 'w') as f:
f.write(res.text)
logging.info('{0:s}: {1:d}'.format(stooq_name, res.status_code))
def analyze(self, exp):
# 1. Select all the leaders for that expiry
# 2. Run StxJL for each leader, for each factor
# ldr_list = self.get_leaders(
setup_df = pd.DataFrame(columns=['date', 'stk', 'setup'])
factors = [1.0, 1.5, 2.0]
q1 = "select min(dt), max(dt) from leaders where exp='{0:s}'".format(
exp)
date_list = stxdb.db_read_cmd(q1)
exp_dt = datetime.datetime.strptime(exp, '%Y-%m-%d')
jls_date = stxcal.next_busday('{0:d}-01-01'.format(exp_dt.year))
jle_date = stxcal.move_busdays(exp, 0)
q2 = "select distinct stk from leaders where exp='{0:s}'".format(exp)
ldr_list = stxdb.db_read_cmd(q2)
for ldr in ldr_list:
s_date = str(date_list[0][0])
e_date = str(date_list[0][1])
stk = ldr[0]
ts = StxTS(stk, jls_date, jle_date)
for ixx in range(1, 5):
ts.df['hi_{0:d}'.format(ixx)] = ts.df['hi'].shift(ixx)
ts.df['lo_{0:d}'.format(ixx)] = ts.df['lo'].shift(ixx)
jl_list = []
for factor in factors:
jl = StxJL(ts, factor)
jl.jl(s_date)
jl_list.append(jl)
while s_date < e_date:
setup_df = self.setups(ts, jl_list, setup_df)
ts.next_day()
for jl in jl_list:
jl.nextjl()
s_date = stxcal.next_busday(s_date)
print('Finished {0:s}'.format(stk))
setup_df = setup_df.sort_values(by=['date', 'setup', 'stk'])
for _, row in setup_df.iterrows():
print('{0:s} {1:12s} {2:s}'.format(row['date'], row['stk'],
row['setup']))
def get_high_activity(self, dt, df):
eight_days_ago = stxcal.move_busdays(dt, -8)
df['d_8'] = eight_days_ago
def hiactfun(r):
qha = sql.Composed([
sql.SQL('select * from jl_setups where dt between '),
sql.Literal(r['d_8']),
sql.SQL(' and '),
sql.Literal(r['dt']),
sql.SQL(' and stk='),
sql.Literal(r['stk']),
sql.SQL(' and abs(score) > 100 and setup in ('),
sql.SQL(',').join([
sql.Literal('Gap'),
sql.Literal('SC'),
sql.Literal('RDay')
]),
sql.SQL(')')
])
db_df = pd.read_sql(qha, stxdb.db_get_cnx())
return db_df['score'].sum() if len(db_df) > 0 else 0
df['hi_act'] = df.apply(hiactfun, axis=1)
default=Datafeed.stooq)
parser.add_argument('-d',
'--data_dir',
help='download directory for EOD files',
type=str,
default=os.path.join(os.getenv('HOME'), 'Downloads'))
args = parser.parse_args()
logging.basicConfig(
format='%(asctime)s %(levelname)s [%(filename)s:%(lineno)d] - '
'%(message)s',
datefmt='%Y-%m-%d %H:%M:%S',
level=logging.INFO)
logging.info('Getting index (S&P500, Nasdaq, Dow Jones) quotes')
si = StxIndex()
index_end_date = stxcal.current_busdate(hr=9)
index_start_date = stxcal.move_busdays(index_end_date, -5)
for idx in ['^GSPC', '^IXIC', '^DJI']:
try:
si.get_quote(idx, index_start_date, index_end_date)
except:
logging.error(f'Get index quote failed for {idx}')
tb.print_exc()
sdf = StxDatafeed()
res = stxdb.db_read_cmd("SELECT dt FROM analyses WHERE "
"analysis='eod_datafeed'")
start_date = str(res[0][0]) if res else '2000-01-01'
logging.info('The start date is: {0:s}'.format(start_date))
res = stxdb.db_read_cmd("SELECT MAX(dt) FROM dividends")
splits_start_date = str(res[0][0]) if res else '2000-01-01'
args = parser.parse_args()
data_dir = os.getenv('DOWNLOAD_DIR')
logging.basicConfig(
format='%(asctime)s %(levelname)s [%(filename)s:%(lineno)d] - '
'%(message)s',
datefmt='%Y-%m-%d %H:%M:%S',
level=logging.INFO
)
seod = StxEOD(data_dir)
if args.stooq:
s_date_sq = '2018-03-12'
e_date_sq = '2018-03-29'
seod.parseeodfiles(s_date_sq, e_date_sq)
sys.exit(0)
# Handle default EODData stream
# 1. Get the last date for which eod data is available in the database
# res = stxdb.db_read_cmd("select max(dt) from eods where oi=0")
res = stxdb.db_read_cmd("select dt from analyses where "
"analysis='eod_datafeed'")
start_date = stxcal.next_busday(str(res[0][0]))
# 2. get the last trading date
end_date = stxcal.move_busdays(str(datetime.datetime.now().date()), 0)
# 3. Find out if files are available for the dates
# 4. if the files are available, upload them
batch_load = True if args.batch else False
seod.load_eoddata_files(start_date, end_date, batch=batch_load)
res = stxdb.db_read_cmd("select max(dt) from dividends")
start_date = stxcal.next_busday(str(res[0][0]))
seod.handle_splits(start_date)
def ana_report(self, stk, start_date, end_date):
res = '<table><tr>'
jl_start_date = stxcal.move_busdays(end_date, -8)
# add the A/D setups table
res += '<td><table>'
qad = sql.Composed([
sql.SQL('select * from jl_setups where dt between '),
sql.Literal(start_date),
sql.SQL(' and '),
sql.Literal(end_date),
sql.SQL(' and setup in ('),
sql.SQL(',').join(
[sql.Literal('Gap'),
sql.Literal('SC'),
sql.Literal('RDay')]),
sql.SQL(') and abs(score) >= 100 and stk='),
sql.Literal(stk),
sql.SQL(' order by dt, direction, setup')
])
df_ad = pd.read_sql(qad, stxdb.db_get_cnx())
for _, row in df_ad.iterrows():
res += '<tr><td>{}</td><td>{}</td><td>{}</td><td>{}</td>'\
'</tr>'.format(row['dt'].strftime('%b %d'), row['setup'],
row['direction'], row['score'])
res += '</td></table>'
# add the JL setups table
res += '<td><table>'
qjl = sql.Composed([
sql.SQL('select * from jl_setups where dt between '),
sql.Literal(jl_start_date),
sql.SQL(' and '),
sql.Literal(end_date),
sql.SQL(' and setup in ('),
sql.SQL(',').join([
sql.Literal('JL_B'),
sql.Literal('JL_P'),
sql.Literal('JL_SR')
]),
sql.SQL(') and stk='),
sql.Literal(stk),
sql.SQL(' order by dt, direction, setup, factor')
])
df_jl = pd.read_sql(qjl, stxdb.db_get_cnx())
for _, row in df_jl.iterrows():
res += '<tr><td>{}</td><td>{}</td><td>{}</td><td>{}</td>'\
'<td>{}</td></tr>'.format(row['dt'].strftime('%b %d'),
row['setup'], row['direction'],
row['factor'], row['score'])
res += '</table></td>'
# add the candlesticks setups table
res += '<td><table>'
qcs = sql.Composed([
sql.SQL('select * from jl_setups where dt between '),
sql.Literal(start_date),
sql.SQL(' and '),
sql.Literal(end_date),
sql.SQL(' and setup in ('),
sql.SQL(',').join([
sql.Literal('EngHarami'),
sql.Literal('Cbs'),
sql.Literal('3out'),
sql.Literal('3'),
sql.Literal('Kicking'),
sql.Literal('Piercing'),
sql.Literal('Engulfing'),
sql.Literal('Star')
]),
sql.SQL(') and stk='),
sql.Literal(stk),
sql.SQL(' order by dt, direction, setup')
])
df_cs = pd.read_sql(qcs, stxdb.db_get_cnx())
for _, row in df_cs.iterrows():
res += '<tr><td>{}</td><td>{}</td><td>{}</td></tr>'.format(
row['dt'].strftime('%b %d'), row['setup'], row['direction'])
res += '</td></table>'
res += '</tr></table>'
return res
def get_report(self, crt_date, df, do_analyze):
s_date = stxcal.move_busdays(crt_date, -50)
jl_s_date = stxcal.move_busdays(crt_date, -350)
ana_s_date = stxcal.move_busdays(crt_date, -20)
res = []
rsdf = self.get_rs_stx(crt_date)
if do_analyze:
indexes = ['^GSPC', '^IXIC', '^DJI']
for index in indexes:
stk_plot = StxPlot(index, s_date, crt_date)
stk_plot.plot_to_file()
res.append('<h4>{0:s}</h4>'.format(index))
res.append('<img src="/tmp/{0:s}.png" alt="{1:s}">'.format(
index, index))
try:
jl_res = StxJL.jl_report(index, jl_s_date, crt_date, 1.0)
res.append(jl_res)
except:
logging.error('{0:s} JL(1.0) calc failed'.format(index))
tb.print_exc()
try:
jl_res = StxJL.jl_report(index, jl_s_date, crt_date, 2.0)
res.append(jl_res)
except:
logging.error('{0:s} JL(2.0) calc failed'.format(index))
tb.print_exc()
try:
ana_res = self.ana_report(index, ana_s_date, crt_date)
res.append(ana_res)
except:
logging.error('Failed to analyze {0:s}'.format(index))
tb.print_exc()
setup_df = df.merge(rsdf)
up_setup_df = setup_df.query("direction=='U'").copy()
up_setup_df.sort_values(by=['rs'], ascending=False, inplace=True)
down_setup_df = setup_df.query("direction=='D'").copy()
down_setup_df.sort_values(by=['rs'], ascending=True, inplace=True)
res.append('<h3>{0:d} UP Setups</h3>'.format(len(up_setup_df)))
for _, row in up_setup_df.iterrows():
res.extend(
self.setup_report(row, s_date, jl_s_date, ana_s_date,
crt_date))
res.append('<h3>{0:d} DOWN Setups</h3>'.format(len(down_setup_df)))
for _, row in down_setup_df.iterrows():
res.extend(
self.setup_report(row, s_date, jl_s_date, ana_s_date,
crt_date))
if do_analyze:
rsbest = rsdf.query('rs_rank==99').copy()
rsworst = rsdf.query('rs_rank==0').copy()
rsworst.sort_values(by=['rs'], ascending=True, inplace=True)
res.append('<h3>RS Leaders</h3>')
for i, (_, row) in enumerate(rsbest.iterrows()):
res.extend(
self.rs_report(i, row, s_date, jl_s_date, ana_s_date,
crt_date))
res.append('<h3>RS Laggards</h3>')
for i, (_, row) in enumerate(rsworst.iterrows()):
res.extend(
self.rs_report(i, row, s_date, jl_s_date, ana_s_date,
crt_date))
return res
])
stxdb.db_write_cmd(db_cmd)
logging.info(f'Updated {len(date_list)} records for {idx}')
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('-i',
'--index',
type=str,
required=True,
help='Comma-separated list of quoted Indexes')
parser.add_argument('-s',
'--startdate',
type=str,
default=stxcal.move_busdays(
stxcal.current_busdate(hr=9), -5),
help='Start date for quote history')
parser.add_argument('-e',
'--enddate',
type=str,
default=stxcal.current_busdate(hr=9),
help='End date for quote history')
args = parser.parse_args()
logging.basicConfig(
format='%(asctime)s %(levelname)s [%(filename)s:%(lineno)d] - '
'%(message)s',
datefmt='%Y-%m-%d %H:%M:%S',
level=logging.INFO)
idx_list = args.index.split(',')
si = StxIndex()
for idx in idx_list: