def post_processing(df_all, symbols, error_symbols):
if df_all is not None and df_all.shape[0] > 0:
print("saving...", df_all.shape)
#today_str = datetime.datetime.today().strftime('%Y-%m-%d')
dates = list(set(df_all["date"]))
first_day_str = min(dates).strftime('%Y-%m-%d')
yesterday = Td.get_prev_trading_day(min(dates))
yesterday_str = yesterday.strftime("%Y-%m-%d")
yesterday_frame = None
print("yesterday_str", yesterday_str)
if os.path.exists( pickles_dir + os.sep + 'stock.pickle' + yesterday_str):
yesterday_frame = pd.read_pickle( pickles_dir + os.sep + 'stock.pickle' + yesterday_str)
print("yesterday's frame", yesterday_frame.shape, yesterday_frame.columns)
else:
m = mongo_api()
date_filter = construct_day_filter(yesterday)
yesterday_frame = m.read_df('stockcandles', False, '*', [], date_filter, {})
print("yesterday's frame", yesterday_frame.shape, yesterday_frame.columns)
if yesterday_frame is not None and yesterday_frame.shape[0] > 0:
if len(dates) > 1:
df_all["delete_flg"] = df_all.date.apply(lambda x: x.hour == 16)
print("detect multiple dates", df_all.shape)
df_all = df_all[df_all["delete_flg"] == False]
dates = list(np.unique(df_all["date"]))
df_all = update_multiple_date_close(df_all, dates, yesterday_frame, symbols)
else:
df_all = update_close_for_df(symbols, df_all, yesterday_frame)
#m.write_df(df_out, collection_name)
print(df_all.shape, set(df_all.date))
df_all.to_pickle('stock_pickles/stock.pickle_tmp' + "_"+ datetime.datetime.today().strftime("%Y%m%d%H%M%S"))
df_all["delete_flag"] = df_all.date.apply(lambda x: (x.hour == 16))
df_all["keep_flag"] = df_all.date.apply(lambda x: (x.hour == 21))
df_keep = df_all[df_all["keep_flag"] == True]
print("df_keep.shape", df_keep.shape)
df_discard = df_all[df_all["delete_flag"] == True]
print("df_discard.shape", df_discard.shape)
df_keep = drop_columns(df_keep, ["delete_flag", "keep_fJun19 85 C lag"])
df_discard = drop_columns(df_discard, ["delete_flag", "keep_flag"])
m = mongo_api()
if df_keep.shape[0] > 0:
df_keep["date"] = df_keep.date.apply(lambda x: datetime.datetime(x.year, x.month, x.day, 0,0,0))
for d in set(df_keep.date):
filename = compose_file_name(d)
df_keep[df_keep["date"] == d].to_pickle(filename)
m.write_df(df_keep, collection_name)
print("total # of rows written, error symbols", df_keep.shape[0], len(error_symbols))
if df_discard.shape[0] > 0:
for d in set(df_discard.date):
filename = compose_file_name(d) + "_discard"
df_discard[df_discard["date"] == d].to_pickle(filename)
print("total # of rows written for timestamp 16:00:00, error symbols", df_discard.shape[0], len(error_symbols))
def persist_option_hist_file(fileName, symbols):
df = pd.read_pickle(fileName)
df_out = None
num_stats = 0
if df is not None:
m = mongo_api()
df = df.drop("Expiration", axis=1)
if symbols is not None:
df = df.loc[df.UnderlyingSymbol.isin(symbols)]
dates = set(df.data_date)
#print("df.shape", df.shape)
df = df.sort_values(['UnderlyingSymbol', 'data_date'])
sCount = 0
#stock_closing_price = np.min(cur_df["UnderlyingPrice"])
drop_index = df.loc[(df.exp_week != 3) |
(df.days_to_expire > 90)].index
if len(drop_index) > 0:
df.drop(drop_index, in_place=True)
df_inserted = None
total_recs = 0
df_in_optionstats = m.read_df(collection_name, False,
["date", "symbol"], [], {}, {
'symbol': 1,
'date': 1
})
for s in symbols:
cur_dif = insert_one_symbol(df, s, m, df_in_optionstats)
if cur_dif is not None and cur_dif.shape[0] > 1:
m.write_df(cur_dif, collection_name)
total_recs += cur_dif.shape[0]
print("inserted", total_recs)
def main(argv):
d1 = argv[0]
d2 = argv[1]
watch_list_file = argv[2]
symbols = []
if not os.path.exists(watch_list_file):
symbol = argv[2]
symbols = [symbol]
print("symbols", symbols)
else:
symbols = load_watch_lists(watch_list_file)
if not os.path.exists(watch_list_file):
watch_list_file = None
df_date = datetime.datetime.strptime(d2, "%Y%m%d")
symbols = find_missing_symbols(symbols, df_date)
print("symbols after calculating missing", symbols)
df = get_stockcandles_for_day(d1, d2, None, symbols, False)
#df = pd.read_pickle('stock.pickle2020-01-2122')
print("df.shape after get_stockcandles_for_day", df.shape)
df_out = post_processing_for_close(df, df_date)
temp = str(random.randint(51, 100))
file_suffix = df_date.strftime("%Y-%m-%d")
filename = pickles_dir + os.sep + "stock.pickle" + file_suffix + "_" + temp
df_out.to_pickle(filename)
if df[df_out.chg.isna()].shape[0] == 0:
print("persisting... shape is:", df.shape)
m = mongo_api()
m.write_df(df_out, 'stockcandles')
print("saving to " + filename)
def check_num_rows(date_range, symbols, request_type, missing_map):
collection_name = "optionstat"
col_names = ["symbol", "date"]
sort_spec = {"date": 1, "symbol": 1}
if request_type == 'sq':
collection_name = "stockcandles"
start_dt, end_dt = date_range.split(",")
all_hours = Td.get_market_hour(start_dt, end_dt)
dest_dates = set(all_hours["market_day"])
m = mongo_api()
df_all = m.read_df(collection_name, False, col_names, [], {}, sort_spec)
for d in dest_dates:
cur_frame = df_all.loc[df_all.date == d]
diff_symbols = set(symbols) - set(cur_frame.symbol)
formattedD = d.strftime("%Y%m%d")
if len(diff_symbols) == 0:
continue
if missing_map != {} and d in missing_map.keys():
missing_map[formattedD].append(diff_symbols)
else:
missing_map[formattedD] = list(diff_symbols)
if missing_map != {}:
f = open("missing_map_" + request_type + ".json", "w+")
j = json.dumps(missing_map, sort_keys=True, indent=4)
f.write(j)
f.close()
else:
print("foound nothing missing")
def persist_option_dirs(dirName, symbols, pattern):
debug_print("reading", dirName)
m = mongo_api()
num_stats_rec_inserted = 0
for (root, dirs, files) in os.walk(dirName, topdown=True):
if root == 'historical':
continue
files = np.sort(files)
#print("len files", len(files))
for fi in files:
(file_type, d_cur, curr_symbol,
fi) = determine_file_origin(root, fi)
#print("file origin", file_type)
if pattern is not None and fi.find(pattern) == -1:
#print("file not in pattern", fi)
continue
if curr_symbol not in symbols:
debug_print("skipped symbol ", curr_symbol)
continue
if file_type == 'TDA':
print("processing:", fi)
df = pd.read_csv(fi)
x = persist_td_option_file(df, curr_symbol, d_cur, m)
elif file_type == 'OPTIONISTICS':
print("processing:", fi)
df = pd.read_csv(fi)
x = persist_optionistics_file(df, curr_symbol, d_cur, m)
else:
debug_print("skipping", fi)
num_stats_rec_inserted += x
print("number of records inserted to optionhist", num_stats_rec_inserted)
def getPHFromDb(symbol, begin, end):
m = mongo_api()
filter = {'$and': [{'symbol':{'$eq': symbol}}, {'date': {'$gte': begin}}, {'date': {'$lt': end}}]}
print(filter)
df = m.read_df('stockcandles', False, '*', [], {'$and': [{'symbol':{'$eq': symbol}}, {'date': {'$gte': begin}},{'date': {'$lt': end}}]}, {})
df = drop_columns(df, ["_id"])
return df
def update_close_forall(date_range):
m = mongo_api()
d1, d2 = date_range.split(",")
date1 = datetime.datetime.strptime(d1, '%Y%m%d')
date2 = datetime.datetime.strptime(d2, '%Y%m%d')
df_all = m.read_df(
'stockcandles', False, '*', [],
{'$and': [{
'date': {
'$gte': date1
}
}, {
'date': {
'$lte': date2
}
}]}, {})
df_all = df_all.sort_values(["symbol", "date"])
df_out = None
for s in set(df_all.symbol):
df_s = df_all.loc[df_all.symbol == s]
df_s = df_s.sort_values("date")
df_s_first_index = df_s.index[0]
df_s["prev_c"] = df_s["close"].shift(periods=1, fill_value=0)
df_s.loc[df_s_first_index, "prev_c"] = df_s.loc[df_s_first_index,
"close"]
df_s["chg"] = df_s["close"] - df_s["prev_c"]
df_out = append_df(df_out, df_s)
return df_out
def get_swingindex(df):
m = mongo_api()
df_prevFrame = None
i = 0
symbolSet = set(df.symbol)
df_out = None
for s in symbolSet:
df_symbol = df[df.symbol == s]
df_symbol = df_symbol.sort_values("date")
df_symbol = df_symbol.reset_index()
prev_trading_date = Td.get_prev_trading_day(df.iloc[0].date)
df_prevFrameSymbol = None
if df_prevFrame is None:
df_prevFrame = m.read_df('stockcandles', False, '*', [],
{'date': {
'$eq': prev_trading_date
}}, {})
print("read prev day", df_prevFrame.shape)
df_prevFrameSymbol = df_prevFrame[df_prevFrame.symbol == s]
if df_prevFrameSymbol is None or df_prevFrameSymbol.shape[0] == 0:
print("can't find prev day frame:", s)
continue
#print("prev_day symbol", s, df_prevFrameSymbol.shape)
df_symbol["today_chg"] = df_symbol.apply(lambda x: x.close - x.open,
axis=1)
df_symbol["prev_chg"] = df_symbol.today_chg.shift()
min_date = np.min(df_symbol.date)
df_symbol.loc[df_symbol.date == min_date,
"prev_chg"] = df_prevFrameSymbol["close"].iloc[
0] - df_prevFrameSymbol["open"].iloc[0]
df_symbol["weighted_change"] = df_symbol.apply(
lambda x: 50 * (x.chg + 0.5 * x.prev_chg + 0.25 * x.today_chg),
axis=1)
df_symbol["R1"] = df_symbol.apply(
lambda x: np.abs(x.high - x.prev_c) - 0.5 * np.abs(
x.low - x.prev_c) + 0.25 * x.prev_chg,
axis=1)
df_symbol["R2"] = df_symbol.apply(
lambda x: np.abs(x.low - x.prev_c) - 0.5 * np.abs(
x.high - x.prev_c) + 0.25 * x.prev_chg,
axis=1)
df_symbol["R3"] = df_symbol.apply(
lambda x: np.abs(x.high - x.low) + 0.25 * x.prev_chg, axis=1)
df_symbol["R"] = df_symbol.apply(lambda x: np.amax([x.R1, x.R2, x.R3]),
axis=1)
df_K = np.max(
df_symbol.apply(
lambda x: np.amax([x.high - x.prev_c, x.low - x.prev_c]),
axis=1))
df_M = np.max(df_symbol.apply(lambda x: (x.high - x.low), axis=1))
df_symbol["si"] = df_symbol.apply(lambda x:
(x["weighted_change"] / x.R) *
(df_K / df_M),
axis=1)
i += 1
if i % 20 == 0:
print("processed ", i)
df_out = append_df(df_out, df_symbol)
return df_out
def getStockDesc(symbols, descName):
m = mongo_api()
if type(symbols) == str:
symbols = [symbols]
filter = {"symbol": {'$in': symbols}}
df = m.read_df('stock_quotes', False, ['symbol', descName], [], filter,
{"symbol": 1})
assert (df.shape[0] == len(symbols))
i = list(df.columns).index(descName)
return df.iloc[:, i]
def persist_stock_price_history(symbols):
m = mongo_api()
(dest_data, start_date, end_date) = get_market_days()
total_recs = 0
for symbol in symbols:
try:
db_stock_df = m.read_df('stockhist', True, "datetime", [], {'symbol': {'$eq': symbol}} ,{"datetime":1})
if db_stock_df is not None and db_stock_df.shape[0] > 0 and "datetime" in db_stock_df.columns:
db_stock_df["market_day"] = db_stock_df["datetime"].apply(lambda x: datetime.datetime(x.year, x.month, x.day, 0,0,0))
curr_data = set(db_stock_df["market_day"])
diff_date = np.array(list(dest_data - curr_data))
else:
diff_date = np.array(list(dest_data))
diff_date = np.sort(diff_date)
#debug_print("Differentiated dates", len(diff_date))
if len(diff_date) <=0:
continue
m.deleteMany('stockhist', {'symbol': {'$eq': symbol}})
start_datetime = datetime.datetime.strptime(start_date, '%Y-%m-%d')
delta = (datetime.datetime.today() - start_datetime).days + 1
option_params = "{\"resolution\" : \"D\", \"count\": " + str(delta) + "}"
df = getResultByType('price_history', '2048', symbol, option_params)
if df is None:
print("can't get result for symbol", symbol)
continue
df["datetime"] = df.t.apply(lambda x: Td.convert_timestamp_to_time(x, 's'))
df["symbol"] = symbol
#df = df.sort_values(['datetime'])
# make sure we get the same shape
df = df.sort_values('datetime',ascending=True)
market_day = df.datetime.apply(
lambda x: datetime.datetime(x.year, x.month, x.day, 0, 0, 0))
if (len(set(market_day)) < len(dest_data)):
print("length diff", symbol, len(market_day), len(dest_data))
debug_print("read stock history", df.shape)
diff_ts = []
for d in diff_date:
diff_ts.append((np.datetime64(d) - np.datetime64('1970-01-01T00:00:00Z')) / np.timedelta64(1, 's'))
df["ts"] = df.datetime.apply(lambda x: (np.datetime64(x.strftime('%Y-%m-%dT00:00:00Z')) - np.datetime64('1970-01-01T00:00:00Z')) / np.timedelta64(1, 's'))
debug_print("df.ts", df.ts)
debug_print("diff_ts", diff_ts)
df = df[df["ts"].isin(diff_ts)]
debug_print("df.shape after filter", df.shape)
if df.shape[0] > 0:
m.write_df(df, 'stockhist')
total_recs += df.shape[0]
else:
total_recs += 0
except KeyError:
print("error when persist stock price history")
continue
return 0
return total_recs
def read_data(symbol, date_begin, date_end):
m = mongo_api()
date_range = Td.get_market_hour(date_begin, date_end)
print(date_range["market_day"][0], date_range["market_day"][-1])
df = m.read_df('stockcandles', False, '*', [], {'symbol': {
'$eq': symbol
}}, {'date': 1})
df = df[(df.date >= date_range["market_day"][0])
& (df.date <= date_range["market_day"][-1])]
print(df.shape)
return df
def find_dates(df_symbol, symbol, df_in_optionstats):
m = mongo_api()
date_set3 = set([])
date_set2 = set(df_symbol.data_date)
if df_in_optionstats.shape[0] > 0:
date_set3 = set(df_in_optionstats["date"])
diff_2 = []
if date_set3 != date_set2:
diff_2 = date_set2 - date_set3
diff_2 = np.sort(list(diff_2))
del df_in_optionstats
return diff_2
def find_missing_symbols(target_symbols, target_date):
m = mongo_api()
date_filter = construct_day_filter(target_date)
symbol_in_db = m.read_df(collection_name, False, ["symbol", "date"], [],
date_filter, {})
if symbol_in_db.shape[0] > 0:
symbols_missing = set(target_symbols) - set(
symbol_in_db["symbol"].values)
print("symbols_in_db", symbol_in_db.shape, symbols_missing)
return list(symbols_missing)
else:
return target_symbols
def get_daily_stock_for_symbols(symbols, error_symbols, done_symbols, day_range, use_td=False, use_intraday=True):
df_out = None
symbol_count = 0
d = Td.get_prev_trading_day(datetime.datetime.today())
m = mongo_api()
#assert(yesterday_frame.shape[0] == len(symbols))
for symbol in symbols:
if symbol in done_symbols:
continue
try:
print("getting", symbol)
#first we use the finn API
if use_td:
df = get_daily_stock_from_td(symbol)
elif use_intraday:
df = get_daily_stock_for_intraday(symbol, datetime.datetime.today())
else:
df = getResultByType('price_history', None, symbol, "{\"resolution\": \"D\", \"count\":" + day_range + "}")
if df is None and use_td == False:
df = get_daily_stock_from_td(symbol)
else:
df = post_process_ph_df(df, symbol)
if df is None:
print("add to error symbols", symbol)
error_symbols.add(symbol)
continue
#prev_close = yesterday_frame[yesterday_frame["symbol"] == symbol].iloc(0)
#df = calculate_chg(df, prev_close)
df_out = append_df(df_out, df)
symbol_count += 1
if symbol in error_symbols:
error_symbols.remove(symbol)
done_symbols.add(symbol)
if symbol_count % 15 == 0:
print("current processed ", symbol_count, " symbols")
time.sleep(3)
except requests.exceptions.SSLError:
error_symbols.add(symbol)
if len(error_symbols) % 15 == 0:
print("error symbols:", error_symbols)
time.sleep(3)
continue
except json.decoder.JSONDecodeError:
error_symbols.add(symbol)
if len(error_symbols) % 15 == 0:
print("error symbols:", error_symbols)
time.sleep(3)
continue
return df_out
def post_processing_for_close(df, d):
prev_trading_date = Td.get_prev_trading_day(d)
print("prev_trading_date", prev_trading_date)
m = mongo_api()
df_prev= m.read_df('stockcandles', False, '*', [],\
{'$and': [{'year': {'$eq': prev_trading_date.year}},\
{'month':{'$eq': prev_trading_date.month}},\
{'d_index': {'$eq': prev_trading_date.day}}]},{})
print(df_prev.columns)
symbols = set(df_prev["symbol"])
output_df = update_close_for_df(symbols, df, df_prev)
output_df["date"] = output_df.date.apply(
lambda x: datetime.datetime(x.year, x.month, x.day, 0, 0, 0))
return output_df
def getOptionHist(symbol, startDate, endDate):
projectionAttrs = ["date", 'max_vol_call', 'max_vol_put', 'mean_vol_call', 'mean_vol_put', 'mean_oi_call', 'mean_oi_put', 'median_iv_call']
projectionMeasures = []
filter = {'symbol': {'$eq': symbol}}
sortSpec = {'symbol': 1, 'date': 1}
m = mongo_api()
df = m.read_df('optionstat', False, projectionAttrs, projectionMeasures, filter, sortSpec)
if df is not None and df.shape[0] > 0:
print("df.columns", df.columns)
df = df[(df.date >= startDate) & (df.date <= endDate)]
print("OptionHistory shape", df.shape, df.columns)
else:
print("can't find the stats: ", filter, projectionAttrs)
return df
def persist_company_info(symbol):
try:
if db_api.recordExists('stock_quotes', {'symbol': {'$eq': symbol}}):
print("skipped symbol", symbol)
return
quotes = getResultByType('quote', '2048', symbol, {}, True)
df = pd.DataFrame(data= quotes)
df = df.transpose()
df = df.reset_index()
df = df.rename(columns = {"index": "symbol"})
m = mongo_api()
m.write_df(df, 'stock_quotes')
except ValueError:
print("error persist ", symbol)
def read_optionistics_stock(symbols, dirName):
m = mongo_api()
total_recs = 0
df_out = None
for (root,dirs,files) in os.walk(dirName, topdown=True):
for fi in files:
(file_type, d_cur, curr_symbol, fi) = parse_optionistics_filename(root, fi)
print(file_type, fi)
if file_type == 'OPTIONISTICS':
print("reading ", fi)
df = pd.read_csv(fi)
df = persist_optionistics_stock_file(df, curr_symbol, m)
df_out = append_df(df_out, df)
total_recs = df_out.shape[0]
print("total rec inserted", total_recs)
df_out.to_pickle('today_stock.pickle')
return total_recs
def persist_stock_history_from_file(symbols):
# we have two files which has different columns
m = mongo_api()
total_recs = 0
for year in 2015,2016,2017, 2018, 2019:
df = pd.read_pickle(STOCK_HIST_FILE_PATH + os.sep + str(year) + '.pickle_stock')
df = df[df["symbol"].isin(symbols)]
try:
df = calculate_spread(df)
print(df.shape, df.columns)
total_recs += df.shape[0]
m.write_df(df, collection_name)
except KeyError:
print("error when persist stock price history")
continue
return 0
print("total records inserted", total_recs)
return total_recs
def read_one_stock_quote(symbol, currDate, projection):
m = mongo_api()
date_filter = construct_day_filter(currDate)
assert (type(projection) == str)
try:
db_df = m.read_df('stockcandles', False, [projection], [],
{'$and': [{
'symbol': {
'$eq': symbol
}
}, date_filter]}, None)
if db_df.shape is not None and db_df.shape[0] >= 1:
debug_print("found price for ", currDate, " at ", db_df.iloc[0, 1])
return db_df.iloc[0, 1]
else:
print("can't find specific symbol", symbol)
return None
except KeyError:
print("error when reading single stock price history")
return None
def computeOptionHist(symbol, startDate, endDate):
projectionAttrs = ['data_date', 'UnderlyingSymbol']
projectionMeasures = ['Volume', 'OpenInterest']
sortSpec = ['UnderlyingSymbol', 'data_date']
filter = {'$and': [ {'UnderlyingSymbol': {'$in': [symbol]}},{'Type': {'$eq': 'call'}}, {'data_date': {'$gte': startDate}}, \
{'data_date': {'$lte': endDate}}]}
sortSpec = {'symbol': 1, 'data_date': 1}
m = mongo_api()
df_call = m.read_df('optionhist', False, projectionAttrs, projectionMeasures, filter, sortSpec)
filter = {'$and': [{'UnderlyingSymbol': {'$in': [symbol]}}, {'Type': {'$eq': 'put'}}, {'data_date': {'$gte': startDate}}, \
{'data_date': {'$lt': endDate}}]}
df_put = m.read_df('optionhist', False, projectionAttrs, projectionMeasures, filter, sortSpec)
df_call.columns = ['date', 'symbol', 'callvol', 'calloi']
df_put.columns = ['date', 'symbol', 'putvol', 'putoi']
df = df_call
df["putvol"] = df_put.putvol
df["putoi"] = df_put.putoi
debug_print("after computing optionstat history", df.shape, df.columns)
m.write_df(df, 'optionstat')
return df
def persist_daily_stock(day_range, watch_list_file, symbol = None, useIntraDay=False):
skipped_set = {"VIAB", "CBS", "BBT"}
df_all = None
if symbol is None:
symbols = load_watch_lists(watch_list_file)
else:
symbols = [symbol]
day_range = str(day_range)
m = mongo_api()
today = datetime.datetime.today()
date_filter = db_api.construct_day_filter(today)
projection = "date"
#if not db_api.check_persist_timing(m, 'optionstat', projection, date_filter, today):
# return
error_symbols = set()
symbols = np.sort(symbols)
retryCount = 0
symbols_set = set(symbols)
for skip_s in skipped_set:
if skip_s in symbols_set:
symbols_set.remove(skip_s)
symbols = list(symbols_set)
done_symbols = set()
retry_count = 0
while(len(done_symbols) != len(symbols)):
df= get_daily_stock_for_symbols(symbols, error_symbols, done_symbols, day_range, False, useIntraDay)
df_all = append_df(df_all, df)
if (len(error_symbols) == 0 or (df_all is not None and df_all.shape[0] == len(symbols)) or retry_count >= 4):
post_processing(df_all,symbols, error_symbols)
if len(set(df_all.symbol)) + len(skipped_set) < len(symbols):
print("missing", set(symbols) - set(df_all.symbols) - set(skipped_set))
break
else:
#post_processing(df_all, symbols, error_symbols)
print("error symbols, continuing, resetting symbols to error_Symbols", error_symbols)
retry_count = retry_count + 1
def main(argv):
fileName = None
dirName = None
symbol = None
symbols = []
pattern = None
skipped_list = [
"SPY", "USO", "IWM", "QQQ", "DIA", "DOW", "DD", "VIAB", 'KHC', 'UAA',
'HPE', 'ARNC', 'PYPL'
]
try:
opts, args = getopt.getopt(argv, "hf:d:s:w:p:",
["help", "f=", "d=", "s=", "p="])
for opt, arg in opts:
if opt == '-h':
print(
sys.argv[0] +
'-f <filename> -d <dirName> -s <symbol> -w <watch_list> -p <pattern>'
)
sys.exit()
if opt == '-f':
print("getting file", arg)
fileName = arg
if opt == '-d':
dirName = arg
if opt == '-s':
symbol = arg
if symbol.find(",") == -1:
symbols = [symbol]
else:
symbols = load_json_for_symbol(symbol)
if opt == '-p':
pattern = arg
if opt == '-w':
watch_list_file = arg
symbols = load_watch_lists(watch_list_file)
print(symbols)
symbols = list(set(symbols) - set(skipped_list))
if symbols is not None:
symbols = np.sort(symbols)
if len(symbols) == 0:
print("please specifiy symbol as -s or -w")
exit(-1)
today = datetime.datetime.today()
date_filter = construct_day_filter(today)
m = mongo_api()
#if not check_persist_timing(m, collection_name, date_filter, today):
# return
if fileName is not None and os.path.exists(fileName):
persist_option_hist_file(fileName, symbols)
elif fileName is not None and not os.path.exists(fileName):
print("Can't find", fileName)
exit(-1)
if dirName is not None:
try:
persist_option_dirs(dirName, symbols, pattern)
except KeyError:
print("Error when reading option_dirs")
exit(-1)
except TypeError:
print("Error in argument")
exit(-1)
import numpy.ma as ma
from trade_api.tda_api import Td
from trade_api.mongo_api import mongo_api
import os as os
import sys as sys
import json
import csv
import datetime
from analyze_historical import OptionStats
from optionML import getResultByType
import timedelta
from util import debug_print, append_df, load_watch_lists
from trade_api.db_api import read_one_stock_quote, getStockDesc, get_stock_price_history, recordExists
from trade_api.calculate_iv import newton_vol_call, newton_vol_put, newton_vol_call_div, newton_vol_put_div
m = mongo_api()
def readTable(tableName, projection=None):
if projection is None:
projection = '*'
df = m.read_df(tableName, False, projection, [], {}, {})
return df
def getMaxMin(df, colName):
return np.max(df[colName]), np.min(df[colName])
def aggr(df, projCol, expression):
gb = df.groupby(by=projCol, as_index=True)
def getResultByType(request_type, code, symbol, parameters, use_td=False):
result = None
if request_type == 'quote':
#ib_a = init_ib()
#c_list = ib_a.qualifyContracts([symbol])
#p_list = ib_a.reqTickers(c_list)
#for p in p_list:
# print(p.marketPrice())
td = init_td(code)
result = td.get_quotes(symbol)
elif request_type == 'init_token':
init_token()
elif request_type == 'option_chain':
result = getOptions(parameters, symbol, code)
elif request_type == 'price_history':
if code is not None or use_td:
td = init_td(code)
if parameters == '':
parameters = getDefaultPriceHistoryParameters()
parameters = json.loads(parameters)
result = td.get_price_history(symbol, None, None, parameters)
else:
#print(parameters)
f = Finnapi('', 2)
result = f.getCandleStick(symbol, parameters)
elif request_type == 'option_history':
m = mongo_api()
result = m.read_df('optionstats', False, \
[{'date': {'$dateFromParts': {'year': '$year', 'month': '$month', 'day': '$d_index'}}}, 'symbol'], \
['callvol', 'calloi', 'putvol', 'putoi'], \
{'symbol': {'$in': [symbol]}}, \
{'symbol': 1, 'date': 1})
elif request_type == 'stock_profile':
if code is not None or use_td:
td = init_td(code)
result = td.getInstrument(symbol)
elif request_type == 'upgrade':
f = Finnapi('', 2)
result = f.getUpDownGrade(symbol)
result.gradeTime = Td.convert_timestamp_to_time(result.gradeTime, 's')
elif request_type == 'price-target':
f = Finnapi('', 2)
result = f.getPriceTarget(symbol)
elif request_type == 'recommendations':
f = Finnapi('', 2)
result = f.getRecommendations(symbol)
elif request_type == 'patterns':
f = Finnapi('', 2)
if parameters == '':
parameters = {'resolution': 'D'}
result = f.getPatterns(symbol, parameters)
elif request_type == 'techind':
f = Finnapi('', 2)
if parameters == '':
todayTimestamp = date.today()
d1, d2 = gettimestampForPeriod(todayTimestamp, 'D', -10)
parameters = {
'resolution': 'D',
'indicator': 'sma',
'timeperiod': 3,
'from': d2,
'to': d1
}
else:
if parameters != '':
parameters = json.loads(parameters)
if parameters["resolution"] == 'D' or parameters[
"resolution"] == 'M':
todayTimestamp = date.today()
count = parameters["count"]
d1, d2 = gettimestampForPeriod(todayTimestamp,
parameters["resolution"],
-count)
ind_period = parameters["timeperiod"]
indicator = parameters["indicator"]
parameters = {
'resolution': 'D',
'indicator': indicator,
'timeperiod': ind_period,
'from': d2,
'to': d1
}
result = f.getTechInd(symbol, parameters)
elif request_type == 'rev-est' or request_type == 'eps-est' or request_type == 'earnings':
f = Finnapi('', 2)
result = f.getEarnings(symbol, parameters, request_type)
return result
def recordExists(collection_name, filter):
m = mongo_api()
bexist = (m.count(collection_name, filter) > 0)
return bexist
def getStockDesc(symbols, descName):
m = mongo_api()
df = m.read_df('stock_quotes', True, ['symbol', descName], [], {"symbol": {'$in': symbols}}, {"symbol":1})
assert(df.shape[0] == len(symbols))
return df[descName]