def get_dh(ticker, nfcs=4, nss=4, pgDb=None, sector=None): """ get eps and stock data and then create dg,dh,dfcs where dg: earnings data from temp_eps table dh: corresponding eps, differnet in PE reverse ratio, and quarterly returns dfcs: [hwX] forecasts of [ma] period """ #- get EPS data sqx = "SELECT *,\"epsDif\" as ma,rptdate::int/100 as yyyymm FROM temp_eps WHERE ticker={!r} ORDER BY rptdate".format( ticker) dg = sqlQuery(sqx, pgDb) #dg = dg.rename({"pbdate":"asof"},axis='columns') #- get closing price data sDu = pull_stock_data(ticker, days=1000, src='iex')[['close', 'pbdate']] sDu = sDu.rename({"close": "price"}, axis='columns') cur_prc = sDu['price'][-1] dx = freq_d2m(sDu, method='last', fq='M') dx['pbdate'] = dx['pbdate'].astype('int') dx.loc[:, 'yyyymm'] = (dx['pbdate'][:] / 100).astype('int') #dx['rtm'] = dx['price'].diff() dx['rtm'] = dx['price'].pct_change( ) * 100.0 # monthly return and rescale to % dx['pr1'] = dx['price'].shift() """ dx['rt1'] = dx['rt'].shift(-1) dx['rt2'] = dx['rt'].shift(-2) dx['pb1'] = dx['pbdate'].shift(-1) dx['pb2'] = dx['pbdate'].shift(-2) """ #- merge data #dh = pd.merge(dx[['rt1','pb1','rt2','pb2','rt','yyyymm','pbdate','price']],dg[['ma','yyyymm','asof','ticker']],on=['yyyymm']) dh = pd.merge(dx[['rtm', 'yyyymm', 'pbdate', 'price', 'pr1']], dg[['ma', 'yyyymm', 'asof', 'ticker']], on=['yyyymm']) dh.loc[:, 'ma'] = dh['ma'] / dh['pr1'] * 100.0 # use PE-ratio difference dh['rt'] = dh['price'].pct_change() * 100.0 # quarterly return ntr = 5 # nth powter of taylor series vv = dh['ma'].dropna().astype(float) if len(vv) < 1: return [], [], [] for j in range(nfcs): vv.append(taylor_appx(vv, ntr)) hwX = vv[-nfcs:] hwN = len(hwX) hwS = json.dumps(list(hwX)) dfcs = pd.DataFrame( { 'hwX': [hwS], 'ticker': [ticker], 'prc_cur': [cur_prc], 'sector': [sector] }, columns=['sector', 'ticker', 'hwX', 'prc_cur']) return dfcs, dh, dg
def run_fcs(ticker, debugTF=False, funcName='rForecast', **optx): # get data datax = pull_stock_data(ticker) asof = int(datax['pbdate'].iloc[-1]) # idxtm=map(lambda x:datetime.datetime.strptime(str(x),"%Y%m%d"),datax['pbdate']) # datax.set_index(pd.DatetimeIndex(idxtm),inplace=True) if debugTF is True: print datax.tail() # get r-code pandas2ri.activate() rstring = 'source("./_alan_ohlc_fcs.r")' r(rstring) # convert to r-data #df=pandas2ri.py2ri(datax[['pbdate','close']]) df = pandas2ri.py2ri(datax['close'][:]) # run r-function opts = { 'nfcs': 30, 'dwmTF': True, 'autoArima': False, 'difTF': True, 'funcname': 'rAR', 'logTF': True, 'plevel': 0.7, 'freq': 'W' } opts.update(optx) optR = subDict(opts, [ 'nfcs', 'plevel', 'funcname', 'autoArima', 'logTF', 'difTF', 'freq', 'fcsLst', 'dwmTF' ]) if debugTF: print >> sys.stderr, "==Input Args:{}".format(optR) print >> sys.stderr, "==asof {},df:\n{}".format( asof, datax['close'][-5:]) if funcName in robj.globalenv: funcArg = robj.globalenv[funcName] ret = funcArg(df, asof, debugTF=debugTF, **optR) if opts['dwmTF'] is True: dwm = pandas2ri.ri2py(ret[1]) dwm['ticker'] = ticker else: dwm = pd.DataFrame() dd = pandas2ri.ri2py(ret[0]) dd['ticker'] = ticker return (dd, dwm, datax)
def get_csvdata(args, sep='|', src=None, days=730, start=None, end=None, columns=None, hdrLst=None): """ Get data in datafram with selected [columns] """ if isinstance(args, pd.DataFrame): df = args if columns is not None and df.size > 0: df = df[list(set(df.columns) & set(columns.split(',')))] if hdrLst is not None: xLst, yLst = hdrLst.split('=') xyD = dict(zip(xLst.split(','), yLst.split(','))) df.rename(columns=xyD, inplace=True) return df if len(args) < 1: return None filename = args[0] if filename == '-': df = pd.read_csv(sys.stdin, sep=sep) elif src is not None: from _alan_calc import pull_stock_data df = pull_stock_data(filename, days=days, src=src, start=start, end=end) else: df = pd.read_csv(filename, sep=sep) if df.size < 1: print >> sys.stderr, "**ERROR: Data not found!" return {} if columns is not None: df = df[list(set(df.columns) & set(columns.split(',')))] df.dropna(inplace=True) if hdrLst is not None: xLst, yLst = hdrLst.split('=') xyD = dict(zip(xLst.split(','), yLst.split(','))) df.rename(columns=xyD, inplace=True) return df
def get_csvdata(args,sep='|',src=None,days=730,start=None,end=None,colLst=None,hdrLst=None,**optx): """ Get data in datafram with selected [colLst] """ if isinstance(args,pd.DataFrame): df = args if colLst is not None and df.size > 0: df = df[ list(set(df.columns) & set(colLst.split(','))) ] if hdrLst is not None: xLst,yLst = hdrLst.split('=') xyD = dict(zip(xLst.split(','),yLst.split(','))) df.rename(columns=xyD,inplace=True) return df if len(args)<1: return None filename=args[0] if filename=='-': df=pd.read_csv(sys.stdin,sep=sep) elif src.upper()=='FILENAME': df=pd.read_csv(filename,sep=sep) elif len(args)>1: from yh_hist_batch import batch_yh_hist df = batch_yh_hist(args,days=days,src=src,start=start,end=end,**optx) elif src is not None: from _alan_calc import pull_stock_data df = pull_stock_data(filename,days=days,src=src,start=start,end=end,**optx) else: df = pd.read_csv(filename,sep=sep) if df.size < 1: sys.stderr.write("**ERROR:{} @{}()".format('Data Not Found!','get_csvdata')) return {} if colLst is not None: df = df[ list(set(df.columns) & set(colLst.split(','))) ] df.dropna(inplace=True) if hdrLst is not None: xLst,yLst = hdrLst.split('=') xyD = dict(zip(xLst.split(','),yLst.split(','))) df.rename(columns=xyD,inplace=True) return df
def run_ohlc_fcs(ticker, opts=None, debugTF=False, pgDB=None, **kwargs): """ forecast 'nfcs' periods based on raw data 'datax' return (dd,dwm,datax) where dd: forecast values dwm: forecast values of next day, week, month (optional) datax: data used for forecast calculation Note, dwm can be optional if dwmTF is False """ if opts is None: (opts, _) = opt_ohlc_fcs([]) if len(kwargs) > 0: opts.update(kwargs) if debugTF: pqint(opts, file=sys.stderr) days = getKeyVal(opts, 'days', 730) freq = getKeyVal(opts, 'freq', 'D') # get data if isinstance(ticker, pd.DataFrame): datax = ticker ticker = '' else: # get data optx = subDict(opts, ['src', 'days', 'start', 'end']) datax = pull_stock_data(ticker, pgDB=pgDB, **optx) if 'ticker' in datax: ticker = datax['ticker'].iloc[0] if datax is None or len(datax) < 1: return (None, None, None) #idxtm=map(lambda x:datetime.datetime.strptime(str(x),"%Y%m%d"),datax['pbdate']) #datax.set_index(pd.DatetimeIndex(idxtm),inplace=True) if debugTF is True: pqint(opts, file=sys.stderr) pqint(datax.tail(), file=sys.stderr) nobs = days if 'epochs' in datax: asof = int( datetime.datetime.fromtimestamp( int(datax['epochs'].iloc[-1]) / 1000).strftime('%Y%m%d')) fcsLst = np.array([5, 10, 30]) else: asof = int(datax['pbdate'].iloc[-1]) fcsLst = np.array([1, 5, 23]) vprc = datax['close'][-nobs:] # get r-code pandas2ri.activate() fpath = os.path.dirname(__file__) if len(fpath) < 1: fpath = '.' rstring = 'source("{}/{}")'.format(fpath, "_alan_ohlc_fcs.r") if debugTF: pqint(rstring, file=sys.stderr) r(rstring) # convert to r-data df = pandas2ri.py2ri(vprc) # run r-function [rGARCH|rAR] optx = subDict(opts, [ 'nfcs', 'plevel', 'funcname', 'autoArima', 'logTF', 'difTF', 'freq', 'fcsLst', 'dwmTF' ]) if debugTF: pqint("==Input Args:{}".format(optx), file=sys.stderr) pqint("==df\n:{}".format(vprc.tail()), file=sys.stderr) ret = robj.globalenv['rForecast'](df, asof, debugTF=debugTF, **optx) #ret=robj.globalenv['rForecast'](df,asof,plevel=plevel,funcname=funcname,autoArima=autoArima,debugTF=debugTF,logTF=logTF,difTF=difTF,freq=freq,fcsLst=fcsLst) if opts['dwmTF'] is True: dwm = pandas2ri.ri2py(ret[1]) dwm['ticker'] = ticker else: dwm = pd.DataFrame() dd = pandas2ri.ri2py(ret[0]) dd['ticker'] = ticker dd['freq'] = freq return (dd, dwm, datax)
def run_alan_plot(tkLst, opts=None, optx=None): if opts is None: opts, _ = opt_alan_plot([]) if optx is not None: opts.update(optx) for ky, va in opts.items(): exec("{}=va".format(ky)) colorUD = ['red', 'green'] if lang == 'cn' else ['green', 'red'] if pngDIR is None or os.path.isdir(pngDIR) is False: pngDIR = "/home/web/bb_site/html/images" if debugTF is True: pqint("===options:{}".format(opts), file=sys.stderr) for stock in tkLst: try: if chartType == 'minute': optx = {'gap': '1m', 'ranged': '1d', 'outTF': False} datax = pull_stock_data(stock, **optx) pqint(datax.tail(), file=sys.stderr) else: datax = pull_stock_data(stock, start, end) except Exception as e: pqint("***Data ERROR:", str(e), file=sys.stderr) continue if chartType == 'minute' and 'epochs' in datax: datax['pbdatetime'] = datax.epochs.apply(epoch_parser) dtfmt = '%H:%M %b %d' else: datax['pbdatetime'] = datax.pbdate.apply(ymd_parser) dtfmt = '%m-%d-%y' try: block = False if bscTF is True else True datax.dropna(inplace=True) pqint(datax.tail(), file=sys.stderr) if chartType != 'minute': datax.reset_index(inplace=True) datax['mpldatetime'] = datax.pbdatetime.apply(mdates.date2num) #- calc macd & rsi datax = run_tech(datax, pcol='close', winLst=[ma1, ma2], debugTF=debugTF, nanTF=True) fig, axes = plot_candlestickCombo(datax, stock, ma1, ma2, savePng=savePng, block=block, pngDIR=pngDIR, chartType=chartType, trendTF=trendTF, npar=npar, debugTF=debugTF, colorUD=colorUD, title=title) if savePng is False: plt.show(axes) # DEPRECATED, use run_tech() + plot_candlestickCombo() #graphData(datax,stock,ma1,ma2,savePng=savePng,block=block,pngDIR=pngDIR,chartType=chartType) if bscTF is True: run_ohlc(datax) bsc_plot(datax, stock, days=7, savePng=savePng, block=True, pngDIR=pngDIR, dtfmt=dtfmt) except Exception as e: pqint(str(e), file=sys.stderr) return datax