def post_processing(df_all, symbols, error_symbols): if df_all is not None and df_all.shape[0] > 0: print("saving...", df_all.shape) #today_str = datetime.datetime.today().strftime('%Y-%m-%d') dates = list(set(df_all["date"])) first_day_str = min(dates).strftime('%Y-%m-%d') yesterday = Td.get_prev_trading_day(min(dates)) yesterday_str = yesterday.strftime("%Y-%m-%d") yesterday_frame = None print("yesterday_str", yesterday_str) if os.path.exists( pickles_dir + os.sep + 'stock.pickle' + yesterday_str): yesterday_frame = pd.read_pickle( pickles_dir + os.sep + 'stock.pickle' + yesterday_str) print("yesterday's frame", yesterday_frame.shape, yesterday_frame.columns) else: m = mongo_api() date_filter = construct_day_filter(yesterday) yesterday_frame = m.read_df('stockcandles', False, '*', [], date_filter, {}) print("yesterday's frame", yesterday_frame.shape, yesterday_frame.columns) if yesterday_frame is not None and yesterday_frame.shape[0] > 0: if len(dates) > 1: df_all["delete_flg"] = df_all.date.apply(lambda x: x.hour == 16) print("detect multiple dates", df_all.shape) df_all = df_all[df_all["delete_flg"] == False] dates = list(np.unique(df_all["date"])) df_all = update_multiple_date_close(df_all, dates, yesterday_frame, symbols) else: df_all = update_close_for_df(symbols, df_all, yesterday_frame) #m.write_df(df_out, collection_name) print(df_all.shape, set(df_all.date)) df_all.to_pickle('stock_pickles/stock.pickle_tmp' + "_"+ datetime.datetime.today().strftime("%Y%m%d%H%M%S")) df_all["delete_flag"] = df_all.date.apply(lambda x: (x.hour == 16)) df_all["keep_flag"] = df_all.date.apply(lambda x: (x.hour == 21)) df_keep = df_all[df_all["keep_flag"] == True] print("df_keep.shape", df_keep.shape) df_discard = df_all[df_all["delete_flag"] == True] print("df_discard.shape", df_discard.shape) df_keep = drop_columns(df_keep, ["delete_flag", "keep_fJun19 85 C lag"]) df_discard = drop_columns(df_discard, ["delete_flag", "keep_flag"]) m = mongo_api() if df_keep.shape[0] > 0: df_keep["date"] = df_keep.date.apply(lambda x: datetime.datetime(x.year, x.month, x.day, 0,0,0)) for d in set(df_keep.date): filename = compose_file_name(d) df_keep[df_keep["date"] == d].to_pickle(filename) m.write_df(df_keep, collection_name) print("total # of rows written, error symbols", df_keep.shape[0], len(error_symbols)) if df_discard.shape[0] > 0: for d in set(df_discard.date): filename = compose_file_name(d) + "_discard" df_discard[df_discard["date"] == d].to_pickle(filename) print("total # of rows written for timestamp 16:00:00, error symbols", df_discard.shape[0], len(error_symbols))
def persist_option_hist_file(fileName, symbols): df = pd.read_pickle(fileName) df_out = None num_stats = 0 if df is not None: m = mongo_api() df = df.drop("Expiration", axis=1) if symbols is not None: df = df.loc[df.UnderlyingSymbol.isin(symbols)] dates = set(df.data_date) #print("df.shape", df.shape) df = df.sort_values(['UnderlyingSymbol', 'data_date']) sCount = 0 #stock_closing_price = np.min(cur_df["UnderlyingPrice"]) drop_index = df.loc[(df.exp_week != 3) | (df.days_to_expire > 90)].index if len(drop_index) > 0: df.drop(drop_index, in_place=True) df_inserted = None total_recs = 0 df_in_optionstats = m.read_df(collection_name, False, ["date", "symbol"], [], {}, { 'symbol': 1, 'date': 1 }) for s in symbols: cur_dif = insert_one_symbol(df, s, m, df_in_optionstats) if cur_dif is not None and cur_dif.shape[0] > 1: m.write_df(cur_dif, collection_name) total_recs += cur_dif.shape[0] print("inserted", total_recs)
def main(argv): d1 = argv[0] d2 = argv[1] watch_list_file = argv[2] symbols = [] if not os.path.exists(watch_list_file): symbol = argv[2] symbols = [symbol] print("symbols", symbols) else: symbols = load_watch_lists(watch_list_file) if not os.path.exists(watch_list_file): watch_list_file = None df_date = datetime.datetime.strptime(d2, "%Y%m%d") symbols = find_missing_symbols(symbols, df_date) print("symbols after calculating missing", symbols) df = get_stockcandles_for_day(d1, d2, None, symbols, False) #df = pd.read_pickle('stock.pickle2020-01-2122') print("df.shape after get_stockcandles_for_day", df.shape) df_out = post_processing_for_close(df, df_date) temp = str(random.randint(51, 100)) file_suffix = df_date.strftime("%Y-%m-%d") filename = pickles_dir + os.sep + "stock.pickle" + file_suffix + "_" + temp df_out.to_pickle(filename) if df[df_out.chg.isna()].shape[0] == 0: print("persisting... shape is:", df.shape) m = mongo_api() m.write_df(df_out, 'stockcandles') print("saving to " + filename)
def check_num_rows(date_range, symbols, request_type, missing_map): collection_name = "optionstat" col_names = ["symbol", "date"] sort_spec = {"date": 1, "symbol": 1} if request_type == 'sq': collection_name = "stockcandles" start_dt, end_dt = date_range.split(",") all_hours = Td.get_market_hour(start_dt, end_dt) dest_dates = set(all_hours["market_day"]) m = mongo_api() df_all = m.read_df(collection_name, False, col_names, [], {}, sort_spec) for d in dest_dates: cur_frame = df_all.loc[df_all.date == d] diff_symbols = set(symbols) - set(cur_frame.symbol) formattedD = d.strftime("%Y%m%d") if len(diff_symbols) == 0: continue if missing_map != {} and d in missing_map.keys(): missing_map[formattedD].append(diff_symbols) else: missing_map[formattedD] = list(diff_symbols) if missing_map != {}: f = open("missing_map_" + request_type + ".json", "w+") j = json.dumps(missing_map, sort_keys=True, indent=4) f.write(j) f.close() else: print("foound nothing missing")
def persist_option_dirs(dirName, symbols, pattern): debug_print("reading", dirName) m = mongo_api() num_stats_rec_inserted = 0 for (root, dirs, files) in os.walk(dirName, topdown=True): if root == 'historical': continue files = np.sort(files) #print("len files", len(files)) for fi in files: (file_type, d_cur, curr_symbol, fi) = determine_file_origin(root, fi) #print("file origin", file_type) if pattern is not None and fi.find(pattern) == -1: #print("file not in pattern", fi) continue if curr_symbol not in symbols: debug_print("skipped symbol ", curr_symbol) continue if file_type == 'TDA': print("processing:", fi) df = pd.read_csv(fi) x = persist_td_option_file(df, curr_symbol, d_cur, m) elif file_type == 'OPTIONISTICS': print("processing:", fi) df = pd.read_csv(fi) x = persist_optionistics_file(df, curr_symbol, d_cur, m) else: debug_print("skipping", fi) num_stats_rec_inserted += x print("number of records inserted to optionhist", num_stats_rec_inserted)
def getPHFromDb(symbol, begin, end): m = mongo_api() filter = {'$and': [{'symbol':{'$eq': symbol}}, {'date': {'$gte': begin}}, {'date': {'$lt': end}}]} print(filter) df = m.read_df('stockcandles', False, '*', [], {'$and': [{'symbol':{'$eq': symbol}}, {'date': {'$gte': begin}},{'date': {'$lt': end}}]}, {}) df = drop_columns(df, ["_id"]) return df
def update_close_forall(date_range): m = mongo_api() d1, d2 = date_range.split(",") date1 = datetime.datetime.strptime(d1, '%Y%m%d') date2 = datetime.datetime.strptime(d2, '%Y%m%d') df_all = m.read_df( 'stockcandles', False, '*', [], {'$and': [{ 'date': { '$gte': date1 } }, { 'date': { '$lte': date2 } }]}, {}) df_all = df_all.sort_values(["symbol", "date"]) df_out = None for s in set(df_all.symbol): df_s = df_all.loc[df_all.symbol == s] df_s = df_s.sort_values("date") df_s_first_index = df_s.index[0] df_s["prev_c"] = df_s["close"].shift(periods=1, fill_value=0) df_s.loc[df_s_first_index, "prev_c"] = df_s.loc[df_s_first_index, "close"] df_s["chg"] = df_s["close"] - df_s["prev_c"] df_out = append_df(df_out, df_s) return df_out
def get_swingindex(df): m = mongo_api() df_prevFrame = None i = 0 symbolSet = set(df.symbol) df_out = None for s in symbolSet: df_symbol = df[df.symbol == s] df_symbol = df_symbol.sort_values("date") df_symbol = df_symbol.reset_index() prev_trading_date = Td.get_prev_trading_day(df.iloc[0].date) df_prevFrameSymbol = None if df_prevFrame is None: df_prevFrame = m.read_df('stockcandles', False, '*', [], {'date': { '$eq': prev_trading_date }}, {}) print("read prev day", df_prevFrame.shape) df_prevFrameSymbol = df_prevFrame[df_prevFrame.symbol == s] if df_prevFrameSymbol is None or df_prevFrameSymbol.shape[0] == 0: print("can't find prev day frame:", s) continue #print("prev_day symbol", s, df_prevFrameSymbol.shape) df_symbol["today_chg"] = df_symbol.apply(lambda x: x.close - x.open, axis=1) df_symbol["prev_chg"] = df_symbol.today_chg.shift() min_date = np.min(df_symbol.date) df_symbol.loc[df_symbol.date == min_date, "prev_chg"] = df_prevFrameSymbol["close"].iloc[ 0] - df_prevFrameSymbol["open"].iloc[0] df_symbol["weighted_change"] = df_symbol.apply( lambda x: 50 * (x.chg + 0.5 * x.prev_chg + 0.25 * x.today_chg), axis=1) df_symbol["R1"] = df_symbol.apply( lambda x: np.abs(x.high - x.prev_c) - 0.5 * np.abs( x.low - x.prev_c) + 0.25 * x.prev_chg, axis=1) df_symbol["R2"] = df_symbol.apply( lambda x: np.abs(x.low - x.prev_c) - 0.5 * np.abs( x.high - x.prev_c) + 0.25 * x.prev_chg, axis=1) df_symbol["R3"] = df_symbol.apply( lambda x: np.abs(x.high - x.low) + 0.25 * x.prev_chg, axis=1) df_symbol["R"] = df_symbol.apply(lambda x: np.amax([x.R1, x.R2, x.R3]), axis=1) df_K = np.max( df_symbol.apply( lambda x: np.amax([x.high - x.prev_c, x.low - x.prev_c]), axis=1)) df_M = np.max(df_symbol.apply(lambda x: (x.high - x.low), axis=1)) df_symbol["si"] = df_symbol.apply(lambda x: (x["weighted_change"] / x.R) * (df_K / df_M), axis=1) i += 1 if i % 20 == 0: print("processed ", i) df_out = append_df(df_out, df_symbol) return df_out
def getStockDesc(symbols, descName): m = mongo_api() if type(symbols) == str: symbols = [symbols] filter = {"symbol": {'$in': symbols}} df = m.read_df('stock_quotes', False, ['symbol', descName], [], filter, {"symbol": 1}) assert (df.shape[0] == len(symbols)) i = list(df.columns).index(descName) return df.iloc[:, i]
def persist_stock_price_history(symbols): m = mongo_api() (dest_data, start_date, end_date) = get_market_days() total_recs = 0 for symbol in symbols: try: db_stock_df = m.read_df('stockhist', True, "datetime", [], {'symbol': {'$eq': symbol}} ,{"datetime":1}) if db_stock_df is not None and db_stock_df.shape[0] > 0 and "datetime" in db_stock_df.columns: db_stock_df["market_day"] = db_stock_df["datetime"].apply(lambda x: datetime.datetime(x.year, x.month, x.day, 0,0,0)) curr_data = set(db_stock_df["market_day"]) diff_date = np.array(list(dest_data - curr_data)) else: diff_date = np.array(list(dest_data)) diff_date = np.sort(diff_date) #debug_print("Differentiated dates", len(diff_date)) if len(diff_date) <=0: continue m.deleteMany('stockhist', {'symbol': {'$eq': symbol}}) start_datetime = datetime.datetime.strptime(start_date, '%Y-%m-%d') delta = (datetime.datetime.today() - start_datetime).days + 1 option_params = "{\"resolution\" : \"D\", \"count\": " + str(delta) + "}" df = getResultByType('price_history', '2048', symbol, option_params) if df is None: print("can't get result for symbol", symbol) continue df["datetime"] = df.t.apply(lambda x: Td.convert_timestamp_to_time(x, 's')) df["symbol"] = symbol #df = df.sort_values(['datetime']) # make sure we get the same shape df = df.sort_values('datetime',ascending=True) market_day = df.datetime.apply( lambda x: datetime.datetime(x.year, x.month, x.day, 0, 0, 0)) if (len(set(market_day)) < len(dest_data)): print("length diff", symbol, len(market_day), len(dest_data)) debug_print("read stock history", df.shape) diff_ts = [] for d in diff_date: diff_ts.append((np.datetime64(d) - np.datetime64('1970-01-01T00:00:00Z')) / np.timedelta64(1, 's')) df["ts"] = df.datetime.apply(lambda x: (np.datetime64(x.strftime('%Y-%m-%dT00:00:00Z')) - np.datetime64('1970-01-01T00:00:00Z')) / np.timedelta64(1, 's')) debug_print("df.ts", df.ts) debug_print("diff_ts", diff_ts) df = df[df["ts"].isin(diff_ts)] debug_print("df.shape after filter", df.shape) if df.shape[0] > 0: m.write_df(df, 'stockhist') total_recs += df.shape[0] else: total_recs += 0 except KeyError: print("error when persist stock price history") continue return 0 return total_recs
def read_data(symbol, date_begin, date_end): m = mongo_api() date_range = Td.get_market_hour(date_begin, date_end) print(date_range["market_day"][0], date_range["market_day"][-1]) df = m.read_df('stockcandles', False, '*', [], {'symbol': { '$eq': symbol }}, {'date': 1}) df = df[(df.date >= date_range["market_day"][0]) & (df.date <= date_range["market_day"][-1])] print(df.shape) return df
def find_dates(df_symbol, symbol, df_in_optionstats): m = mongo_api() date_set3 = set([]) date_set2 = set(df_symbol.data_date) if df_in_optionstats.shape[0] > 0: date_set3 = set(df_in_optionstats["date"]) diff_2 = [] if date_set3 != date_set2: diff_2 = date_set2 - date_set3 diff_2 = np.sort(list(diff_2)) del df_in_optionstats return diff_2
def find_missing_symbols(target_symbols, target_date): m = mongo_api() date_filter = construct_day_filter(target_date) symbol_in_db = m.read_df(collection_name, False, ["symbol", "date"], [], date_filter, {}) if symbol_in_db.shape[0] > 0: symbols_missing = set(target_symbols) - set( symbol_in_db["symbol"].values) print("symbols_in_db", symbol_in_db.shape, symbols_missing) return list(symbols_missing) else: return target_symbols
def get_daily_stock_for_symbols(symbols, error_symbols, done_symbols, day_range, use_td=False, use_intraday=True): df_out = None symbol_count = 0 d = Td.get_prev_trading_day(datetime.datetime.today()) m = mongo_api() #assert(yesterday_frame.shape[0] == len(symbols)) for symbol in symbols: if symbol in done_symbols: continue try: print("getting", symbol) #first we use the finn API if use_td: df = get_daily_stock_from_td(symbol) elif use_intraday: df = get_daily_stock_for_intraday(symbol, datetime.datetime.today()) else: df = getResultByType('price_history', None, symbol, "{\"resolution\": \"D\", \"count\":" + day_range + "}") if df is None and use_td == False: df = get_daily_stock_from_td(symbol) else: df = post_process_ph_df(df, symbol) if df is None: print("add to error symbols", symbol) error_symbols.add(symbol) continue #prev_close = yesterday_frame[yesterday_frame["symbol"] == symbol].iloc(0) #df = calculate_chg(df, prev_close) df_out = append_df(df_out, df) symbol_count += 1 if symbol in error_symbols: error_symbols.remove(symbol) done_symbols.add(symbol) if symbol_count % 15 == 0: print("current processed ", symbol_count, " symbols") time.sleep(3) except requests.exceptions.SSLError: error_symbols.add(symbol) if len(error_symbols) % 15 == 0: print("error symbols:", error_symbols) time.sleep(3) continue except json.decoder.JSONDecodeError: error_symbols.add(symbol) if len(error_symbols) % 15 == 0: print("error symbols:", error_symbols) time.sleep(3) continue return df_out
def post_processing_for_close(df, d): prev_trading_date = Td.get_prev_trading_day(d) print("prev_trading_date", prev_trading_date) m = mongo_api() df_prev= m.read_df('stockcandles', False, '*', [],\ {'$and': [{'year': {'$eq': prev_trading_date.year}},\ {'month':{'$eq': prev_trading_date.month}},\ {'d_index': {'$eq': prev_trading_date.day}}]},{}) print(df_prev.columns) symbols = set(df_prev["symbol"]) output_df = update_close_for_df(symbols, df, df_prev) output_df["date"] = output_df.date.apply( lambda x: datetime.datetime(x.year, x.month, x.day, 0, 0, 0)) return output_df
def getOptionHist(symbol, startDate, endDate): projectionAttrs = ["date", 'max_vol_call', 'max_vol_put', 'mean_vol_call', 'mean_vol_put', 'mean_oi_call', 'mean_oi_put', 'median_iv_call'] projectionMeasures = [] filter = {'symbol': {'$eq': symbol}} sortSpec = {'symbol': 1, 'date': 1} m = mongo_api() df = m.read_df('optionstat', False, projectionAttrs, projectionMeasures, filter, sortSpec) if df is not None and df.shape[0] > 0: print("df.columns", df.columns) df = df[(df.date >= startDate) & (df.date <= endDate)] print("OptionHistory shape", df.shape, df.columns) else: print("can't find the stats: ", filter, projectionAttrs) return df
def persist_company_info(symbol): try: if db_api.recordExists('stock_quotes', {'symbol': {'$eq': symbol}}): print("skipped symbol", symbol) return quotes = getResultByType('quote', '2048', symbol, {}, True) df = pd.DataFrame(data= quotes) df = df.transpose() df = df.reset_index() df = df.rename(columns = {"index": "symbol"}) m = mongo_api() m.write_df(df, 'stock_quotes') except ValueError: print("error persist ", symbol)
def read_optionistics_stock(symbols, dirName): m = mongo_api() total_recs = 0 df_out = None for (root,dirs,files) in os.walk(dirName, topdown=True): for fi in files: (file_type, d_cur, curr_symbol, fi) = parse_optionistics_filename(root, fi) print(file_type, fi) if file_type == 'OPTIONISTICS': print("reading ", fi) df = pd.read_csv(fi) df = persist_optionistics_stock_file(df, curr_symbol, m) df_out = append_df(df_out, df) total_recs = df_out.shape[0] print("total rec inserted", total_recs) df_out.to_pickle('today_stock.pickle') return total_recs
def persist_stock_history_from_file(symbols): # we have two files which has different columns m = mongo_api() total_recs = 0 for year in 2015,2016,2017, 2018, 2019: df = pd.read_pickle(STOCK_HIST_FILE_PATH + os.sep + str(year) + '.pickle_stock') df = df[df["symbol"].isin(symbols)] try: df = calculate_spread(df) print(df.shape, df.columns) total_recs += df.shape[0] m.write_df(df, collection_name) except KeyError: print("error when persist stock price history") continue return 0 print("total records inserted", total_recs) return total_recs
def read_one_stock_quote(symbol, currDate, projection): m = mongo_api() date_filter = construct_day_filter(currDate) assert (type(projection) == str) try: db_df = m.read_df('stockcandles', False, [projection], [], {'$and': [{ 'symbol': { '$eq': symbol } }, date_filter]}, None) if db_df.shape is not None and db_df.shape[0] >= 1: debug_print("found price for ", currDate, " at ", db_df.iloc[0, 1]) return db_df.iloc[0, 1] else: print("can't find specific symbol", symbol) return None except KeyError: print("error when reading single stock price history") return None
def computeOptionHist(symbol, startDate, endDate): projectionAttrs = ['data_date', 'UnderlyingSymbol'] projectionMeasures = ['Volume', 'OpenInterest'] sortSpec = ['UnderlyingSymbol', 'data_date'] filter = {'$and': [ {'UnderlyingSymbol': {'$in': [symbol]}},{'Type': {'$eq': 'call'}}, {'data_date': {'$gte': startDate}}, \ {'data_date': {'$lte': endDate}}]} sortSpec = {'symbol': 1, 'data_date': 1} m = mongo_api() df_call = m.read_df('optionhist', False, projectionAttrs, projectionMeasures, filter, sortSpec) filter = {'$and': [{'UnderlyingSymbol': {'$in': [symbol]}}, {'Type': {'$eq': 'put'}}, {'data_date': {'$gte': startDate}}, \ {'data_date': {'$lt': endDate}}]} df_put = m.read_df('optionhist', False, projectionAttrs, projectionMeasures, filter, sortSpec) df_call.columns = ['date', 'symbol', 'callvol', 'calloi'] df_put.columns = ['date', 'symbol', 'putvol', 'putoi'] df = df_call df["putvol"] = df_put.putvol df["putoi"] = df_put.putoi debug_print("after computing optionstat history", df.shape, df.columns) m.write_df(df, 'optionstat') return df
def persist_daily_stock(day_range, watch_list_file, symbol = None, useIntraDay=False): skipped_set = {"VIAB", "CBS", "BBT"} df_all = None if symbol is None: symbols = load_watch_lists(watch_list_file) else: symbols = [symbol] day_range = str(day_range) m = mongo_api() today = datetime.datetime.today() date_filter = db_api.construct_day_filter(today) projection = "date" #if not db_api.check_persist_timing(m, 'optionstat', projection, date_filter, today): # return error_symbols = set() symbols = np.sort(symbols) retryCount = 0 symbols_set = set(symbols) for skip_s in skipped_set: if skip_s in symbols_set: symbols_set.remove(skip_s) symbols = list(symbols_set) done_symbols = set() retry_count = 0 while(len(done_symbols) != len(symbols)): df= get_daily_stock_for_symbols(symbols, error_symbols, done_symbols, day_range, False, useIntraDay) df_all = append_df(df_all, df) if (len(error_symbols) == 0 or (df_all is not None and df_all.shape[0] == len(symbols)) or retry_count >= 4): post_processing(df_all,symbols, error_symbols) if len(set(df_all.symbol)) + len(skipped_set) < len(symbols): print("missing", set(symbols) - set(df_all.symbols) - set(skipped_set)) break else: #post_processing(df_all, symbols, error_symbols) print("error symbols, continuing, resetting symbols to error_Symbols", error_symbols) retry_count = retry_count + 1
def main(argv): fileName = None dirName = None symbol = None symbols = [] pattern = None skipped_list = [ "SPY", "USO", "IWM", "QQQ", "DIA", "DOW", "DD", "VIAB", 'KHC', 'UAA', 'HPE', 'ARNC', 'PYPL' ] try: opts, args = getopt.getopt(argv, "hf:d:s:w:p:", ["help", "f=", "d=", "s=", "p="]) for opt, arg in opts: if opt == '-h': print( sys.argv[0] + '-f <filename> -d <dirName> -s <symbol> -w <watch_list> -p <pattern>' ) sys.exit() if opt == '-f': print("getting file", arg) fileName = arg if opt == '-d': dirName = arg if opt == '-s': symbol = arg if symbol.find(",") == -1: symbols = [symbol] else: symbols = load_json_for_symbol(symbol) if opt == '-p': pattern = arg if opt == '-w': watch_list_file = arg symbols = load_watch_lists(watch_list_file) print(symbols) symbols = list(set(symbols) - set(skipped_list)) if symbols is not None: symbols = np.sort(symbols) if len(symbols) == 0: print("please specifiy symbol as -s or -w") exit(-1) today = datetime.datetime.today() date_filter = construct_day_filter(today) m = mongo_api() #if not check_persist_timing(m, collection_name, date_filter, today): # return if fileName is not None and os.path.exists(fileName): persist_option_hist_file(fileName, symbols) elif fileName is not None and not os.path.exists(fileName): print("Can't find", fileName) exit(-1) if dirName is not None: try: persist_option_dirs(dirName, symbols, pattern) except KeyError: print("Error when reading option_dirs") exit(-1) except TypeError: print("Error in argument") exit(-1)
import numpy.ma as ma from trade_api.tda_api import Td from trade_api.mongo_api import mongo_api import os as os import sys as sys import json import csv import datetime from analyze_historical import OptionStats from optionML import getResultByType import timedelta from util import debug_print, append_df, load_watch_lists from trade_api.db_api import read_one_stock_quote, getStockDesc, get_stock_price_history, recordExists from trade_api.calculate_iv import newton_vol_call, newton_vol_put, newton_vol_call_div, newton_vol_put_div m = mongo_api() def readTable(tableName, projection=None): if projection is None: projection = '*' df = m.read_df(tableName, False, projection, [], {}, {}) return df def getMaxMin(df, colName): return np.max(df[colName]), np.min(df[colName]) def aggr(df, projCol, expression): gb = df.groupby(by=projCol, as_index=True)
def getResultByType(request_type, code, symbol, parameters, use_td=False): result = None if request_type == 'quote': #ib_a = init_ib() #c_list = ib_a.qualifyContracts([symbol]) #p_list = ib_a.reqTickers(c_list) #for p in p_list: # print(p.marketPrice()) td = init_td(code) result = td.get_quotes(symbol) elif request_type == 'init_token': init_token() elif request_type == 'option_chain': result = getOptions(parameters, symbol, code) elif request_type == 'price_history': if code is not None or use_td: td = init_td(code) if parameters == '': parameters = getDefaultPriceHistoryParameters() parameters = json.loads(parameters) result = td.get_price_history(symbol, None, None, parameters) else: #print(parameters) f = Finnapi('', 2) result = f.getCandleStick(symbol, parameters) elif request_type == 'option_history': m = mongo_api() result = m.read_df('optionstats', False, \ [{'date': {'$dateFromParts': {'year': '$year', 'month': '$month', 'day': '$d_index'}}}, 'symbol'], \ ['callvol', 'calloi', 'putvol', 'putoi'], \ {'symbol': {'$in': [symbol]}}, \ {'symbol': 1, 'date': 1}) elif request_type == 'stock_profile': if code is not None or use_td: td = init_td(code) result = td.getInstrument(symbol) elif request_type == 'upgrade': f = Finnapi('', 2) result = f.getUpDownGrade(symbol) result.gradeTime = Td.convert_timestamp_to_time(result.gradeTime, 's') elif request_type == 'price-target': f = Finnapi('', 2) result = f.getPriceTarget(symbol) elif request_type == 'recommendations': f = Finnapi('', 2) result = f.getRecommendations(symbol) elif request_type == 'patterns': f = Finnapi('', 2) if parameters == '': parameters = {'resolution': 'D'} result = f.getPatterns(symbol, parameters) elif request_type == 'techind': f = Finnapi('', 2) if parameters == '': todayTimestamp = date.today() d1, d2 = gettimestampForPeriod(todayTimestamp, 'D', -10) parameters = { 'resolution': 'D', 'indicator': 'sma', 'timeperiod': 3, 'from': d2, 'to': d1 } else: if parameters != '': parameters = json.loads(parameters) if parameters["resolution"] == 'D' or parameters[ "resolution"] == 'M': todayTimestamp = date.today() count = parameters["count"] d1, d2 = gettimestampForPeriod(todayTimestamp, parameters["resolution"], -count) ind_period = parameters["timeperiod"] indicator = parameters["indicator"] parameters = { 'resolution': 'D', 'indicator': indicator, 'timeperiod': ind_period, 'from': d2, 'to': d1 } result = f.getTechInd(symbol, parameters) elif request_type == 'rev-est' or request_type == 'eps-est' or request_type == 'earnings': f = Finnapi('', 2) result = f.getEarnings(symbol, parameters, request_type) return result
def recordExists(collection_name, filter): m = mongo_api() bexist = (m.count(collection_name, filter) > 0) return bexist
def getStockDesc(symbols, descName): m = mongo_api() df = m.read_df('stock_quotes', True, ['symbol', descName], [], {"symbol": {'$in': symbols}}, {"symbol":1}) assert(df.shape[0] == len(symbols)) return df[descName]