merged_stock_earn[j], xname='e_over_p_diff', yname=j, saveName='e_over_p_diff', hlines=[], title='e_over_p_diff') return merged_stock_earn # collecting spy sqlcursor = SQL_CURSOR() ts = ALPHA_TIMESERIES() j = 0 # reading in the spy data spy, j = ConfigTable('SPY', sqlcursor, ts, readType, hoursdelay=2) AddInfo(spy, spy, debug=debug) # processing new earnings connectionCal = SQL_CURSOR('earningsCalendar.db') connectionCalv2 = SQL_CURSOR('earningsCalendarForTraining.db') fd = ALPHA_FundamentalData() my_3month_calendar = GetUpcomingEarnings(fd, ReDownload) print(my_3month_calendar) it = 0 preLoaded = [] if not ReDownload: #tables = connectionCalv2.cursor().execute("select name from sqlite_master where type = 'table';") #for t in tables: # print(t) try:
def ProcessTicker(ticker, earningsExp, sqlcursor, spy, j, connectionCal): print(ticker) if debug: print(earningsExp) tstock_info, j = ConfigTable(ticker, sqlcursor, ts, readType, j, hoursdelay=23) if len(tstock_info) == 0: return [] try: AddInfo(tstock_info, spy, debug=debug) except (ValueError, KeyError): print('Error getting infor for %s' % ticker) return [] prev_earnings = None overview = None try: overview = pd.read_sql( 'SELECT * FROM overview WHERE Symbol="%s"' % (ticker), connectionCal) prev_earnings = pd.read_sql( 'SELECT * FROM quarterlyEarnings WHERE ticker="%s"' % (ticker), connectionCal) except: print('no previous info for %s' % ticker) return [] if debug: print(prev_earnings) prev_earnings['earningDiff'] = prev_earnings[ 'reportedEPS'] - prev_earnings['estimatedEPS'] # shift some of the inputs by one day to have the day before inputs #print(list(tstock_info.columns)) for a in [ 'open', 'high', 'low', 'close', 'adj_close', 'volume', 'dividendamt', 'splitcoef', 'pos_volume', 'neg_volume', 'sma10', 'sma20', 'sma50', 'sma100', 'sma200', 'rstd10', 'rsi10', 'cmf', 'BolLower', 'BolCenter', 'BolUpper', 'KeltLower', 'KeltCenter', 'KeltUpper', 'copp', 'daily_return_stddev14', 'beta', 'alpha', 'rsquare', 'sharpe', 'cci', 'stochK', 'stochD', 'obv', 'force', 'macd', 'macdsignal', 'bop', 'HT_DCPERIOD', 'HT_DCPHASE', 'HT_TRENDMODE', 'HT_SINE', 'HT_SINElead', 'HT_PHASORphase', 'HT_PHASORquad', 'adx', 'willr', 'ultosc', 'aroonUp', 'aroonDown', 'aroon', 'senkou_spna_A', 'senkou_spna_B', 'chikou_span', 'SAR', 'vwap14', 'vwap10', 'vwap20', 'chosc', 'market', 'corr14', 'fiveday_prior_vix', 'oneday_prior_vix', 'twoday_prior_vix', 'thrday_prior_vix' ]: tstock_info[a + '_daybefore'] = tstock_info[a].shift(1) #print(tstock_info[['rsi10_daybefore','rsi10','willr_daybefore','willr']].tail()) # cleaning the previous earnings data types prev_earnings['reportedDate'] = pd.to_datetime( prev_earnings['reportedDate'], errors='coerce') prev_earnings['fiscalDateEnding'] = pd.to_datetime( prev_earnings['fiscalDateEnding'], errors='coerce') if debug: print(tstock_info[[ 'daily_return', 'adj_close', 'thrday_future_return', 'oneday_future_return' ]]) # Collecting support levels prev_earnings['tech_levels'] = '' for earn_date in prev_earnings.fiscalDateEnding.values: # coverting this datetime64 to datetime earn_dateA = (earn_date - np.datetime64('1970-01-01T00:00:00Z')) / np.timedelta64( 1, 's') earn_dateA = datetime.datetime.utcfromtimestamp(earn_dateA) prior_year_tstock_info = GetTimeSlot(tstock_info, startDate=earn_dateA) tech_levels = techindicators.supportLevels(prior_year_tstock_info, drawhlines=False) if len(tech_levels) > 0: tech_levels = [str(level[1]) for level in tech_levels] prev_earnings.loc[prev_earnings.fiscalDateEnding == earn_date, ['tech_levels']] = ','.join(tech_levels) # merging or joining on the report date merged_stock_earn = pd.merge(prev_earnings, tstock_info, how="left", left_on='reportedDate', right_index=True) merged_stock_earn['high_from_open'] = merged_stock_earn[ 'high'] - merged_stock_earn['open'] #merged_stock_earn['e_over_p_diff'] = merged_stock_earn['reportedEPS'].diff()/merged_stock_earn['adj_close'].diff() merged_stock_earn['e_over_p_diff'] = (merged_stock_earn['reportedEPS'] / merged_stock_earn['adj_close']).diff( periods=-1) merged_stock_earn['e_over_p_test'] = (merged_stock_earn['reportedEPS'] / merged_stock_earn['adj_close']) if debug: print(merged_stock_earn[[ 'fiscalDateEnding', 'adj_close', 'e_over_p_test', 'e_over_p_diff', 'open' ]]) # Drawing output if doPlot: for j in [ 'daily_return', 'oneday_future_return', 'thrday_future_return', 'high_from_open' ]: MakePlot(merged_stock_earn['earningDiff'], merged_stock_earn[j], xname='EarningsDiff', yname=j, saveName='earningDiff', hlines=[], title='earningDiff') MakePlot(merged_stock_earn['e_over_p_diff'], merged_stock_earn[j], xname='e_over_p_diff', yname=j, saveName='e_over_p_diff', hlines=[], title='e_over_p_diff') return merged_stock_earn
if [a[0]] not in runList: runList += [[a[0]]] for a in allList: #if len(runList)>1000: # break if [a[0]] not in runList: runList += [[a[0]]] print('Running: ', len(runList)) #for s in b.etfs: for s in runList: if s[0].count('^'): continue print(s[0]) estock_info, j = ConfigTable(s[0], sqlcursor, ts, readType, j) if len(estock_info) == 0 or len(estock_info) < 181: continue #print(estock_info['adj_close'][-30:]) sdf = pd.DataFrame( [[ s[0], (estock_info['close'].values[-1] - estock_info['open'].values[-1]) / estock_info['open'].values[-1], #estock_info['adj_close'].pct_change(1), estock_info['adj_close'].pct_change(2).values[-1], estock_info['adj_close'].pct_change(3).values[-1], estock_info['adj_close'].pct_change(4).values[-1], estock_info['adj_close'].pct_change(5).values[-1], estock_info['adj_close'].pct_change(15).values[-1],
ts = ALPHA_TIMESERIES() ticker = 'SPY' #ticker='TLT' #ticker='QQQ' #ticker='GLD' #ticker='HAL' #ticker='GUSH' #ticker='AVCT' #ticker='RZV' spy = runTicker(api, ticker) stock_info = None spy = None sqlcursor = SQL_CURSOR() readType = 'full' spy, j = ConfigTable(ticker, sqlcursor, ts, readType) print('spy') print(spy) # add info if len(spy) == 0: print('ERROR - empy info %s' % ticker) spy['daily_return'] = spy['adj_close'].pct_change(periods=1) + 1 spy['openclosepct'] = (spy.close - spy.open) / spy.open + 1 spy['closeopenpct'] = (spy.open - spy.shift(1).close) / spy.shift(1).close + 1 spy['afterhourspct'] = (spy.shift(-1).open - spy.close) / spy.close + 1 spy['year'] = spy.index.year spy['day'] = spy.index.day spy['month'] = spy.index.month spy['dayofyear'] = spy.index.dayofyear
def GenerateSignal(ticker, out_file_name='out_bull_instructions.csv', price_targets=[]): connectionCal = SQL_CURSOR('earningsCalendarv2.db') fd = ALPHA_FundamentalData() sqlcursor = SQL_CURSOR() ts = ALPHA_TIMESERIES() api = ALPACA_REST() stockInfoQuarter, stockInfoAnnual, company_overview = CollectEarnings( ticker, connectionCal) # annual balance sheet balance_sheet_annual = fd.get_balance_sheet_annual(ticker)[0] balance_sheet_annual['fiscalDateEnding'] = pd.to_datetime( balance_sheet_annual['fiscalDateEnding'], errors='coerce') for d in balance_sheet_annual.columns: if d not in ['fiscalDateEnding', 'totalAssets']: balance_sheet_annual[d] = pd.to_numeric(balance_sheet_annual[d], errors='coerce') # quarterly income statement income_statement_quarterly = fd.get_income_statement_quarterly(ticker)[0] for d in income_statement_quarterly.columns: if d not in ['fiscalDateEnding', 'reportedCurrency']: income_statement_quarterly[d] = pd.to_numeric( income_statement_quarterly[d], errors='coerce') for d in ['fiscalDateEnding']: income_statement_quarterly[d] = pd.to_datetime( income_statement_quarterly[d], errors='coerce') if debug: print(income_statement_quarterly) print(income_statement_quarterly.dtypes) tstock_info, j = ConfigTable(ticker, sqlcursor, ts, 'full') spy, j = ConfigTable('SPY', sqlcursor, ts, 'compact') #print(spy.columns) est = pytz.timezone('US/Eastern') today = datetime.now(tz=est) + maindatetime.timedelta(minutes=-40) #today = datetime.utcnow() + maindatetime.timedelta(minutes=-30) d1 = today.strftime("%Y-%m-%dT%H:%M:%S-04:00") five_days = ( today + maindatetime.timedelta(days=-7)).strftime("%Y-%m-%dT%H:%M:%S-04:00") minute_prices = [] ntry = 0 while ntry < 3: try: minute_prices = runTicker(api, ticker, timeframe=TimeFrame.Minute, start=five_days, end=d1) break except (requests.exceptions.ConnectionError): ntry += 1 # may want to restrict to NYSE open times try: spy = AddInfo(spy, spy) tstock_info = AddInfo(tstock_info, spy, AddSupport=True) except (ValueError, KeyError): print('Error processing adding info %s' % ticker) recent_quotes = getQuotes(api, ticker) if debug: print(tstock_info[[ 'adj_close', 'sma20', 'sma20cen', 'vwap10cen', 'vwap10' ]][50:-10]) earn = Earnings(ticker, income_statement_quarterly, company_overview, balance_sheet_annual, stockInfoQuarter, stockInfoAnnual, tstock_info, minute_prices, recent_quotes) #earn.BuildPDF() earn.WriteCSV(out_file_name, price_targets)
from ReadData import ALPACA_REST,runTicker,ConfigTable,ALPHA_TIMESERIES,GetTimeSlot,SQL_CURSOR import sys import sqlite3 ts = ALPHA_TIMESERIES() sqlcursor = SQL_CURSOR() sc = sqlcursor.cursor() doClean=False doReload=False ticker='DUG' daily_prices,j = ConfigTable(ticker, sqlcursor,ts,'full',hoursdelay=18) daily_prices_365d = GetTimeSlot(daily_prices, days=365) split_dates = daily_prices_365d[daily_prices_365d.splitcoef!=1.0] if len(split_dates)>0: print(split_dates) #print(daily_prices.to_string()) #sc.execute('DROP TABLE DUG') #list_of_tables = table_names = sc.execute("SELECT name from sqlite_master WHERE type ='table' AND name NOT LIKE 'sqlite_%';").fetchall() print("tables: %s" %len(table_names)) #splitID = sc.execute('SELECT COUNT(splitcoef) from DUG WHERE splitcoef!=1.0 AND Date>2021-01-23').fetchall()[0][0] splitID = sc.execute("SELECT COUNT(splitcoef) from SPY WHERE splitcoef!=1.0 AND Date>'2021-01-23'").fetchall()[0][0] #splitID = sc.execute("SELECT * from DUG WHERE splitcoef!=1.0 AND Date>'2021-01-23'").fetchall()[0][0] splitIDl = sc.execute("SELECT * from DUG WHERE splitcoef!=1.0 AND Date>'2021-01-23'").fetchall()[0] print(splitID) print(splitIDl) # create a list to reload
ARIMAauto(minute_prices['close'], extra_vars=[], model_var='close', n_periods=300, fit_range_max=len(minute_prices['close']) - 100, seasonal_m=1) # 300 ARIMAauto(minute_prices['close'], extra_vars=[], model_var='close', n_periods=300, fit_range_max=len(minute_prices['close']) - 1, seasonal_m=1) # 300 ACF(minute_prices['close'], 50, timescale='none', arima_order=(6, 1, 1)) ts = ALPHA_TIMESERIES() sqlcursor = SQL_CURSOR() daily_prices, j = ConfigTable(ticker, sqlcursor, ts, 'full') daily_prices_180d = GetTimeSlot(daily_prices, days=180) daily_prices_365d = GetTimeSlot(daily_prices, days=365) #plt.plot(daily_prices_180d['adj_close']) #plt.show() daily_prices_180d['daily_return'] = daily_prices_180d['adj_close'].pct_change( periods=1) ARIMAauto(daily_prices_180d['adj_close'], extra_vars=[], model_var='adj_close', n_periods=10, fit_range_max=-1, seasonal_m=12) ACF(daily_prices_365d['adj_close'], 50, sarima_order=(1, 1, 1, 18), trend='t') ACF(daily_prices_365d['adj_close'], 50, arima_order=(24, 1, 1), trend='t') ACF(daily_prices_180d['adj_close'], 50, arima_order=(24, 1, 1), trend='t')
# print(company_overview.dtypes) # quarterly income statement income_statement_quarterly = fd.get_income_statement_quarterly(ticker)[0] for d in income_statement_quarterly.columns: if d not in ['fiscalDateEnding', 'reportedCurrency']: income_statement_quarterly[d] = pd.to_numeric( income_statement_quarterly[d], errors='coerce') for d in ['fiscalDateEnding']: income_statement_quarterly[d] = pd.to_datetime( income_statement_quarterly[d], errors='coerce') if debug: print(income_statement_quarterly) print(income_statement_quarterly.dtypes) tstock_info, j = ConfigTable(ticker, sqlcursor, ts, 'compact') spy, j = ConfigTable('SPY', sqlcursor, ts, 'compact') est = pytz.timezone('US/Eastern') today = datetime.now(tz=est) + maindatetime.timedelta(minutes=-40) #today = datetime.utcnow() + maindatetime.timedelta(minutes=-30) d1 = today.strftime("%Y-%m-%dT%H:%M:%S-04:00") five_days = ( today + maindatetime.timedelta(days=-7)).strftime("%Y-%m-%dT%H:%M:%S-04:00") minute_prices = runTicker(api, ticker, timeframe=TimeFrame.Minute, start=five_days, end=d1)
plt.axhline(y=h[0],color=h[1],linestyle=h[2]) #xmin=h[1], xmax=h[2], if doSupport: techindicators.supportLevels(my_stock_info) if draw: plt.show() if doPDFs: plt.savefig(outdir+'%s.pdf' %(saveName)) plt.savefig(outdir+'%s.png' %(saveName)) if not draw: plt.close() plt.close() ticker='X' j=0 readType='full' sqlcursor = SQL_CURSOR() ts = ALPHA_TIMESERIES() stock_info,j=ConfigTable(ticker, sqlcursor,ts,readType, j) spy,j = ConfigTable('SPY', sqlcursor,ts,readType,hoursdelay=2) AddInfo(spy,spy,debug=debug) AddInfo(stock_info,spy,debug=debug) stock_info['sma20d'] = stock_info['adj_close'] - stock_info['sma20'] stock_infoc = stock_info #GetTimeSlot(stock_info,days=70) stock_infoc['daily_return'] = stock_infoc['adj_close'].pct_change() stock_infoc['for_daily_return'] = stock_infoc['adj_close'].shift(-1).pct_change() stock_infoc['openClose'] = stock_infoc['close'] - stock_infoc['open'] stock_infoc['openClose_next'] = stock_infoc['openClose'].shift(-1)
X_test = stock_info[COLS] # use only COLS vector_y_pred = model.predict(X_test) stock_info['pred'] = np.argmax(vector_y_pred, axis=1) if debug: print(stock_info[[ 'adj_close', 'open', 'pred', 'sma50r', 'sma20r', 'sma200r', 'downSL', 'upSL' ]]) return stock_info ts = ALPHA_TIMESERIES() connectionCal = SQL_CURSOR('earningsCalendar.db') sqlcursor = SQL_CURSOR() ticker = 'X' j = 0 # reading in the spy data spy, j = ConfigTable('SPY', sqlcursor, ts, 'full', hoursdelay=2) AddInfo(spy, spy, debug=debug) GetNNSelection(ticker, ts, connectionCal, sqlcursor, spy, debug=False, j=j, training_dir='models/', training_name='stockEarningsModelTestv2noEPS')
df = [] try: df = pd.read_csv(inFileName) except (FileNotFoundError) as e: print("Testing multiple exceptions. {}".format(e.args[-1])) df = [] if debug: print(df) todayFilter = (today + datetime.timedelta(days=-1 * filter_shift_days)) d1 = todayFilter.strftime("%Y-%m-%dT%H:%M:%S-05:00") thirty_days = ( todayFilter + datetime.timedelta(days=-30)).strftime("%Y-%m-%dT%H:%M:%S-05:00") one_day = (todayFilter + datetime.timedelta(days=-1)).strftime("%Y-%m-%dT%H:%M:%S-05:00") spy, j = ConfigTable('SPY', sqlcursor, ts, 'full', hoursdelay=18) if filter_shift_days > 0: spy = GetTimeSlot(spy, days=6 * 365, startDate=todayFilter) spy_daily_prices_60d = GetTimeSlot(spy, days=60 + filter_shift_days) spy_daily_prices_365d = GetTimeSlot(spy, days=365 + filter_shift_days) spy_daily_prices_5y = GetTimeSlot(spy, days=5 * 365 + filter_shift_days) # create a dataframe to store all of the info curr_results = pd.DataFrame(columns=[ 'ticker', 'quote', 'trade', 'bar_close', 'bar_high', 'bar_low', 'bar_open' ]) df_store_data = pd.DataFrame(columns=[ 'ticker', 'time_span', 'fit_expectations', 'stddev', 'fit_diff_significance', 'current_price' ])
for etf in base.etfs: all_tickers += [etf[0]] describe_tickers += [etf[4].strip(',')] ticker_describe_map[etf[0]] = etf[4].strip(',') # check new stocks proc_all_tickers = all_tickers if debug: print('Processing: %s' % len(proc_all_tickers)) iticker = 0 for ticker in proc_all_tickers: iticker += 1 # if not loaded, then let's compute stuff if len(df_store_data[(df_store_data['ticker'] == ticker)]) == 0: daily_prices, j = ConfigTable(ticker, sqlcursor, ts, 'full', hoursdelay=18) if filter_shift_days > 0: daily_prices = GetTimeSlot(daily_prices, days=6 * 365, startDate=todayFilter) daily_prices_60d = GetTimeSlot(daily_prices, days=60 + filter_shift_days) daily_prices_180d = GetTimeSlot(daily_prices, days=180 + filter_shift_days) daily_prices_365d = GetTimeSlot(daily_prices, days=365 + filter_shift_days) daily_prices_3y = GetTimeSlot(daily_prices, days=3 * 365 + filter_shift_days) daily_prices_5y = GetTimeSlot(daily_prices,
def generateMeanRevFigure(apiA, sqlA, tsA, tickerA, doRelativeToSpy=False): """generateMeanRevFigure: apiA - alpaca api sqlA - mysql cursor tsA - time series api tickerA - str - ticker doRelativeToSpy - bool - compute the ratio to SPY for performance """ figs = [] today = datetime.datetime.now(tz=est) d1 = today.strftime("%Y-%m-%dT%H:%M:%S-05:00") d1_set = today.strftime("%Y-%m-%d") #d1_set = "2022-01-19" twelve_hours = ( today + datetime.timedelta(hours=-12)).strftime("%Y-%m-%dT%H:%M:%S-05:00") minute_prices = runTicker(apiA, tickerA, timeframe=TimeFrame.Minute, start=twelve_hours, end=d1) daily_prices, j = ConfigTable(tickerA, sqlA, tsA, 'full', hoursdelay=18) # add todays numbers if available if len(minute_prices) > 0: df_today = pd.DataFrame([[ minute_prices['open'][0], minute_prices['high'].max(), minute_prices['low'].min(), minute_prices['close'][-1], minute_prices['close'][-1], minute_prices['volume'].sum(), 0.0, 1.0 ]], columns=[ 'open', 'high', 'low', 'close', 'adj_close', 'volume', 'dividendamt', 'splitcoef' ], index=[d1_set ]) #index=[minute_prices.index[-1]]) df_today.index = pd.to_datetime(df_today.index) if len(daily_prices ) == 0 or daily_prices.index[-1] < df_today.index[0]: daily_prices = pd.concat([daily_prices, df_today]) daily_prices = daily_prices.sort_index() #st.table(daily_prices.tail()) filter_shift_days = 0 if filter_shift_days > 0: daily_prices = GetTimeSlot(daily_prices, days=6 * 365, startDate=todayFilter) daily_prices_60d = GetTimeSlot(daily_prices, days=60 + filter_shift_days) daily_prices_180d = GetTimeSlot(daily_prices, days=180 + filter_shift_days) daily_prices_365d = GetTimeSlot(daily_prices, days=365 + filter_shift_days) daily_prices_3y = GetTimeSlot(daily_prices, days=3 * 365 + filter_shift_days) daily_prices_5y = GetTimeSlot(daily_prices, days=5 * 365 + filter_shift_days) daily_prices_180d['daily_return'] = daily_prices_180d[ 'adj_close'].pct_change(periods=1) figs += [ FitWithBand( daily_prices_60d.index, daily_prices_60d[['adj_close', 'high', 'low', 'open', 'close']], ticker=tickerA, outname='60d') ] figs += [ FitWithBand( daily_prices_180d.index, daily_prices_180d[['adj_close', 'high', 'low', 'open', 'close']], ticker=tickerA, outname='180d') ] figs += [ FitWithBand( daily_prices_365d.index, daily_prices_365d[['adj_close', 'high', 'low', 'open', 'close']], ticker=tickerA, outname='365d') ] figs += [ FitWithBand( daily_prices_3y.index, daily_prices_3y[['adj_close', 'high', 'low', 'open', 'close']], ticker=tickerA, outname='3y') ] figs += [ FitWithBand( daily_prices_5y.index, daily_prices_5y[['adj_close', 'high', 'low', 'open', 'close']], ticker=tickerA, outname='5y') ] # Compute relative to spy if doRelativeToSpy: spy, j = ConfigTable('SPY', sqlA, tsA, 'full', hoursdelay=18) if filter_shift_days > 0: spy = GetTimeSlot(spy, days=6 * 365, startDate=todayFilter) spy_daily_prices_60d = GetTimeSlot(spy, days=60 + filter_shift_days) spy_daily_prices_365d = GetTimeSlot(spy, days=365 + filter_shift_days) spy_daily_prices_5y = GetTimeSlot(spy, days=5 * 365 + filter_shift_days) figs += [ FitWithBand(daily_prices_365d.index, daily_prices_365d[[ 'adj_close', 'high', 'low', 'open', 'close' ]], ticker=tickerA, outname='365dspycomparison', spy_comparison=spy_daily_prices_365d[[ 'adj_close', 'high', 'low', 'open', 'close' ]]) ] figs += [ FitWithBand( daily_prices_60d.index, daily_prices_60d[['adj_close', 'high', 'low', 'open', 'close']], ticker=tickerA, outname='60dspycomparison', spy_comparison=spy_daily_prices_60d[[ 'adj_close', 'high', 'low', 'open', 'close' ]]) ] figs += [ FitWithBand( daily_prices_5y.index, daily_prices_5y[['adj_close', 'high', 'low', 'open', 'close']], ticker=tickerA, outname='5yspycomparison', spy_comparison=spy_daily_prices_5y[[ 'adj_close', 'high', 'low', 'open', 'close' ]]) ] return figs