def tmpl2mp4(ts='',category='ItD',title='intraday_briefing',datax={},debugTF=False,_pn_={},**optx):
'''
Create comment, mp3comment based on 'title' and svg and
save them into into MDB table: 'mkt_briefing_details' (for 'intraday_briefing' and 'daily_briefing')
or 'daily_single_stock' (for title='daily_single_stock')
'''
if title[-9:]=='_briefing':
tablename="mkt_briefing_details"
rptTxt = dat2rptTxt(ts=ts,_pn_=_pn_,debugTF=debugTF,**optx)
zpk = ['category','templateName','rpt_time','lang']
sys.stderr.write("==SAVE {} to MDB table:{}\n".format(title,tablename))
elif title=='daily_single_stock':
dbname='ara'
tablename=title
deltaTolerance= optx.pop('deltaTolerance',43200)
ticker=getKeyVal(optx,'ticker','')
o,m,e=find_mdb(dict(ticker=ticker),dbname=dbname,tablename=tablename,sortLst=['pbdt'])
if len(o)>0 and 'pbdt' in o[0]:
pbdtMod=o[0]['pbdt']
deltaPassed=int(pd.Timedelta(pd.datetime.now()-pbdtMod).total_seconds())
if deltaPassed<=deltaTolerance:
sys.stderr.write(" --pbdtMod:{},deltaPassed:{},deltaTolerance:{}\n".format(pbdtMod,deltaPassed,deltaTolerance))
sys.stderr.write(" --Directly use comment from table: {}\n".format(tablename))
return o[0]
rptTxt = dat2rptTxt(ts=ts,_pn_=_pn_,debugTF=debugTF,**optx)
_pn_['headTitle']=_pn_['stock_performance']['comment'];
ticker=_pn_['ticker']=_pn_['stock_performance']['ticker'];
_pn_['category']=category
_pn_['templateName']=title
_pn_['pbdt']=_pn_['stock_performance']['pbdt'];
_pn_['lang']=getKeyVal(optx,'lang','cn')
zpk=['ticker']
sys.stderr.write("==SAVE {}:{} to MDB table:{}\n".format(title,ticker,tablename))
else:
return {}
_pn_.update(title=title)
saveBriefing(_pn_,tablename=tablename,zpk=zpk)
if debugTF:
sys.stderr.write("=====ts:\n{}\n".format(ts))
sys.stderr.write("=====optx:\n{}\n".format(optx))
sys.stderr.write("=====rptTxt:\n{}\n".format(rptTxt))
sys.stderr.write("=====_pn_:\n{}\n".format(_pn_))
sys.stderr.write("=====keys:\n{}\n".format(_pn_.keys()))
if 'tmplLst' in _pn_:
tmplLst = _pn_['tmplLst']
sys.stderr.write("=====tmplLst: {}\n".format(tmplLst))
for tname in tmplLst:
dx = _pn_[tname]
sys.stderr.write("===Chartpath: {}:{}\n".format(tname,dx['chartpath']))
else:
sys.stderr.write("*ERROR:tmplLst not found in {}\n".format(_pn_.keys()))
return {}
txtOnly=optx.pop('txtOnly',False)
if txtOnly is True:
sys.stderr.write("=====RUN Text Only: {}\n".format(txtOnly))
return _pn_
sys.stderr.write("=====RUN pn2mp4: {}\n".format(tmplLst))
dpn = pn2mp4(_pn_=_pn_,zpk=zpk,**optx)
return dpn
def topic_theme_majorplayer(**opts):
'''
use 'themename' as tablename
'''
from _alan_str import find_mdb
fund = getKeyVal(opts, 'fund', 'renaissance-technologies-llc')
clientM = getKeyVal(opts, 'clientM', None)
dbname = getKeyVal(opts, 'dbname', 'ara')
themename = getKeyVal(opts, 'themename', 'topic_theme_majorplayer')
dfTF = getKeyVal(opts, 'dfTF', True)
jobj = dict(fund=fund)
df, mDB, errmsg = find_mdb(jobj,
tablename=themename,
dbname=dbname,
dfTF=dfTF)
colX = [
'ticker', 'close', 'marketCap', 'changePercent', 'CurrentShares',
'percentPos', 'SharesChangePercent', 'fiftyTwoWeekRange', 'pbdate',
'shortName', 'fund', 'funddate', 'pbdt'
]
colDc = {
'close': 'closePrice',
'pbdate': 'closeDate',
'shortName': 'company',
'SharesChangePercent': 'SharesChg%',
'changePercent': 'Chg%',
'percentPos': 'Position%'
}
df = subDict(df, colX)
df = renameDict(df, colDc)
return df
def create_theme_media(dbname="ara",
tablename="topic_theme_media",
debugTF=False,
**optx):
''' create MDB::'topic_theme_media' table based on 'madmoney_hist' info
arguments:
'end' default to 2-weeks of today in YYYYMMDD
'callLst' default to ["4","5"] types
'''
zpk = getKeyVal(optx, 'zpk', {'ticker', 'pbdate', 'start', 'end'})
start = getKeyVal(optx, 'start', 0)
end = getKeyVal(optx, 'end', None)
callLst = getKeyVal(optx, 'callLst', ["4", "5"])
if end is None:
end = int(dt2ymd(next_date(weeks=-2)))
xqTmp = "SELECT pbdate,adjusted as price FROM prc_hist where name='{}' and pbdate>{} and pbdate <={} ORDER BY pbdate"
jobj = {"Call": {"$in": callLst}, "pbdate": {"$lte": end, "$gte": start}}
df, mdb, emsg = find_mdb(jobj,
sortLst=["pbdate"],
dbname="ara",
tablename="madmoney_hist",
dfTF=True)
dd = []
for j in df.index:
dx = df.iloc[j].to_dict()
dx['pbdate'] = int(dx['pbdate'])
ticker = dx['ticker']
pbdate = period1 = dx['pbdate']
period2 = int(dt2ymd(next_date(period1, months=1)))
xqr = xqTmp.format(ticker, period1, period2)
dp = sqlQuery(xqr)
vprc = dp['price'].values
if len(vprc) < 1:
continue
vdate = [int(x) for x in dp['pbdate'].values]
sPrice, ePrice = vprc[0], vprc[-1]
period1, period2 = vdate[0], vdate[-1]
rrt = vprc[-1] / vprc[0] * 100. - 100
day_diff = ymd_diff(period1, period2)
dx.update(rrt=rrt, day_diff=day_diff, start=period1, end=period2)
dx.update(sPrice=sPrice, ePrice=ePrice)
sys.stderr.write(
"{j:03d}|{ticker:5s}|{Call}|{rrt:8.2f}|{day_diff:3d}|{start:8d}|{end:8d}|{pbdate:8d}|{Company}|\n"
.format(j=j, **dx))
dd.append(dx)
dy, _, emsg = insert_mdb([dx],
tablename=tablename,
dbname=dbname,
zpk=zpk)
if start > 12345678:
jobj = {"day_diff": {"$lt": 20}, "pbdate": {"$lte": start}}
mdb[dbname][tablename].delete_many(jobj)
df = pd.DataFrame(dd)
return df
def main_tst(description="Stock Update MP4 Creator",usage="Usage: %prog [option] daily_briefing|intraday_briefing [SYMBOL] [True|False]"):
opts, args = parse_opt(sys.argv, description=description,usage=usage)
ticker=getKeyVal(opts,'ticker','AAPL')
txtOnly=getKeyVal(opts,'txtOnly',False)
_pn_={}
tmplname = 'daily_briefing' if len(args)<1 else args[0]
ticker = ticker if len(args)<2 else args[1]
txtOnly = True if len(args)>2 and args[2][:1].lower() in ['1','t'] else txtOnly
sys.stderr.write("===RUN:{}:{}:{}\n".format(tmplname,ticker,txtOnly))
dpn = tmpl_wrapper(tmplname,ticker=ticker,debugTF=True,txtOnly=txtOnly)
return dpn
def search_quote(tkLst,fdLst,**opts):
tkLst=get_sector_etfname(tkLst,**opts)
sys.stderr.write("---tkLst: {} @ search_quote\n".format(tkLst))
instrument = getKeyVal(opts,'instrument','stock')
outTF = getKeyVal(opts,'outTF',True)
hostname,dbname,tablename,lang = getKeyVal(opts,['hostname','dbname','tablename','lang'],['localhost','ara',None,None])
colx='ticker' if instrument=='stock' else 'series'
data=[]
opts.pop('ticker',None)
for ticker in tkLst:
try:
# get quotes from MDB::"yh_quote_curr" for yahoo source indices setup in the PGDB::'mapping_series_label'
if instrument=='stock' or re.search(r'[=^.]',ticker):
mktLst =['^GSPC','^DJI','^IXIC','^SOX']
if ticker.upper() in mktLst:
tablename="market_indicator_quote"
elif re.search(r'[=^.]',ticker):
tablename="yh_spark_hist"
else:
tablename="iex_spark_hist"
jobj={"ticker":ticker}
ret = list(mgDB[tablename].find(jobj,{"_id":0},sort=[("epochs",-1)]).limit(1))
#ret,_,_=find_mdb(jobj,tablename=tablename,dbname="ara")
ret = subDict(ret,['ticker','close','change','pchg','xclose','epochs','pbdate','pbdt'])
ret = renameDict(ret,{'pchg':'changePercent','xclose':'prevClose'})
else: # get quotes for all fields from pgDB
ret=geteach_quote(ticker,fdLst='*',tablename=tablename,lang=lang,dbname=dbname,hostname=hostname,colx=colx)
if ret is not None and len(ret)>0:
#data=data.append(ret,ignore_index=True)
data.extend(ret)
else:
continue
except Exception as e:
pqint( "**ERROR:{} @ {}".format(str(e),search_quote) ,file=sys.stderr)
continue
if len(data)<1:
return None
if not outTF:
return data
data=pd.DataFrame(data)
if fdLst is None:
pass
elif len(fdLst)>2 and fdLst.lower()=='all':
pass
else:
colx=['ticker','epochs','open','high','low','close','volume','xclose','change','pchg','pbdt','hhmm','pbdate','changePercent','prevClose','marketCap']
data=subDF(data,colx)
# data = data_output(data,output)
return data
def plot_templates(*args, **opts):
funcname = getKeyVal(opts, 'funcname', 'plot_ts')
style = getKeyVal(opts, 'style', 'classic')
plt.style.use(style)
if funcname in globals():
funcArg = globals()[funcname]
else:
funcArg = plot_ts
opts.pop('funcname', None)
try:
ret = funcArg(args[0], **opts)
except Exception as e:
sys.stderr.write("**ERROR:{} @{}\n".format(str(e), funcname))
ret = str(e), None, None
return ret
def batch_yh_hist(tkLst=[], opts=None, **optx):
#- Set input parameters
if opts is None or len(opts) < 1:
opts, _ = opt_yh_hist([])
if optx is not None:
opts.update(optx)
kys = ['gap', 'ranged', 'tsTF', 'pchgTF', 'debugTF']
debugTF = getKeyVal(opts, 'debugTF', False)
indexTF = getKeyVal(opts, 'indexTF', True)
output = getKeyVal(opts, 'output', None)
sep = getKeyVal(opts, 'sep', '|')
#for ky,va in opts.items():
# exec('{}=va'.format(ky))
hdrTF = True
if 'funcArg' in opts and opts['funcArg'] in globals():
funcArg = globals()[opts['funcArg']]
else:
funcArg = yh_hist
if len(tkLst) > 0 and tkLst[0] == '-':
tkLst = sys.stdin.read().split("\n")
dm = pd.DataFrame()
for j, ticker in enumerate(tkLst):
hdrTF = True if j < 1 else False
try:
df = funcArg(ticker, hdrTF=hdrTF, **subDict(opts, kys))
if len(df) < 1:
continue
if output is None or len(output) < 1:
dm = pd.concat([dm, df])
except Exception as e:
pqint("**ERROR: {}.{}\n{}".format(j, ticker, str(e)),
file=sys.stderr)
continue
if output is not None and 'ticker' not in df:
df['ticker'] = ticker
if output == 'csv':
sep = sep.encode().decode(
'unicode_escape'
) if sys.version_info.major == 3 else sep.decode("string_escape")
sys.stdout.write(df.to_csv(sep=sep, index=indexTF, header=hdrTF))
elif output == 'html':
sys.stdout.write(df.to_html(index=indexTF))
elif output == 'json':
sys.stdout.write(df.to_json(orient='records'))
hdrTF = False
return dm
def geteach_minute_db(ticker,fdLst,**opts):
''' pull minute data from database
'''
subtopic=getKeyVal(opts,'subtopic','')
src = opts.pop('src','iex')
src= "yh" if '^' in ticker or '=' in ticker else src
tablename="{}_spark_hist".format(src)
sys.stderr.write("---Get minute data {} from MDB::{}\n".format(ticker,tablename))
try:
jobj={"ticker":ticker}
# Get latest pbdate
xd = mgDB[tablename].find_one({"$query":jobj,"$orderby":{"epochs":-1}},{"pbdate":1,"_id":0})
jobj.update(xd)
if subtopic in ['sparkline','spark']:
d = mgDB[tablename].find(jobj,{"_id":0},sort=[("epochs",-1)]).limit(20)
else:
d = mgDB[tablename].find(jobj,{"_id":0})
if d is None:
return {}
data=list(d)
if len(data)<1:
sys.stderr.write(" --No data found from {}\n".format(tablename))
return {}
datax = pd.DataFrame(data)
#datax,_,_ = find_mdb({"ticker":ticker},clientM=mgDB,tablename=tablename,sortLst={"epochs"},dfTF=True)
except Exception as e:
sys.stderr.write("**ERROR:{} @ {}\n".format(str(e),'geteach_minute_db'))
datax={}
return datax
def search_comment(tkLst,fdLst,**opts):
topicLst='hourly|news|report|theme|peers|industry|MFRM'.split('|')
topic=getKeyVal(opts,'topic','MFRM')
if topic not in topicLst:
return None
argName="{}_comment".format(topic)
if topic in topicLst and argName in globals():
pqint("==RUNNING {}() Inputs:{}".format(argName,opts),file=sys.stderr)
try:
data=globals()[argName](tkLst,fdLst,**opts)
except Exception as e:
pqint("**ERROR:{} to run {}".format(str(e),argName),file=sys.stderr)
return data
output=opts.pop('output',None)
data=pd.DataFrame()
optx=subDict(opts,['tablename','lang','dbname','hostname','topic','subtopic','factor'])
if tkLst[0]=='*':
data=geteach_comment('*',fdLst,**optx)
# data = data_output(data,output)
return data
for ticker in tkLst:
ret=geteach_comment(ticker,fdLst,**optx)
if ret is not None and len(ret)>0:
data=data.append(ret,ignore_index=True)
else:
continue
# data = data_output(data,output)
if len(data)<1:
return None
return data
def yh_hist(ticker=None,
gap='1m',
ranged='1d',
hdrTF=True,
tsTF=True,
debugTF=False,
**optx):
pchgTF = getKeyVal(optx, 'pchgTF', False)
dx, jTmp = yh_data(ticker=ticker,
gap=gap,
ranged=ranged,
hdrTF=hdrTF,
tsTF=tsTF,
debugTF=debugTF,
**optx)
# change to prc_temp columns setup
clx = dx.columns
if gap[-1] != 'm': # for daily data
if 'ticker' in clx: # for daily data
dx.rename(columns={'ticker': 'name'}, inplace=True)
else:
dx['name'] = ticker
clx = dx.columns
colv = [
'open', 'high', 'low', 'close', 'volume', 'adjusted', 'pbdate',
'name'
]
else:
colv = ['open', 'high', 'low', 'close', 'volume', 'epochs', 'ticker']
if pchgTF:
colv += ['xclose', 'change', 'pchg']
colx = [x for x in colv if x in clx]
sys.stderr.write("{}\n{}\n{}\n".format(colx, colv, clx))
sys.stderr.write("{}\n".format(dx[colx].tail()))
return dx[colx]
def find_ipoReport(dbname='ara',
tablename='ipoReport',
debugTF=False,
sort='marketCapMM',
**optx):
output = getKeyVal(optx, 'output', 'html')
if sort not in [
'Range52Week', 'change', 'changePercent', 'currDate', 'currPrice',
'daysSinceIPO', 'fiftyDayAverage', 'ipoDate', 'ipoPrice',
'marketCapMM', 'shortName', 'ticker', 'trailingPE'
]:
sort = 'marketCapMM'
df, clientM, _ = find_mdb(dbname=dbname,
tablename=tablename,
dfTF=True,
sortLst=[sort])
pd.options.display.float_format = '{:,.2f}'.format
cfm = {'marketCapMM': "{:,.0f}".format, 'changePercent': "{:.2%}".format}
if output.lower() in ['csv']:
return (df.to_csv(sep="|", index=False))
elif output.lower() in ['tsv']:
return (df.to_csv(sep="\t", index=False))
elif output.lower() in ['html']:
return (df.to_html(formatters=cfm, index=False))
else:
return (df)
def plot_daily_conclusion(da,
db=[],
fig=None,
ax=None,
figsize=(11, 6),
backend='Agg',
title='',
chartformat='svg',
debugTF=False,
**optx):
'''
plot daily_peers by marketcap barh
'''
sys.stderr.write("===start:{}()\n".format("plot_daily_conclusion"))
ticker = getKeyVal(optx, 'ticker')
if backend is not None:
plt.switch_backend(backend)
chartpath, chartname = gen_chartpath(**optx)
sns.set(style="whitegrid")
# define plot
if fig is None:
fig, ax = plt.subplots(figsize=figsize)
ax.grid(False, axis='y')
ax.spines['right'].set_color('none')
ax.spines['top'].set_color('none')
ax.spines['bottom'].set_color('none')
ax.spines['left'].set_color('none')
bottom = optx['bottom'] if 'bottom' in optx else 0.25
#da.plot(ax=ax,kind=kind,position=position,width=width)
da = da[['comment', 'pchg']].sort_values(by='pchg',
ascending=False).iloc[0:10, :]
plt.title('指標變化百分比', fontproperties=fontProp(size=15))
da['comment'] = [xstr.strip('[,.，。]') for xstr in da['comment'].values]
ax.barh(y=da['comment'], width=da['pchg'])
ax.set_yticklabels(labels=da['comment'],
fontproperties=fontProp(size=15, weight='bold'))
# plot features
loc = optx['loc'] if 'loc' in optx else 9
ncol = optx['ncol'] if 'ncol' in optx else 2
fancybox = optx['fancybox'] if 'fancybox' in optx else True
borderaxespad = optx['borderaxespad'] if 'borderaxespad' in optx else 0.
plt.legend(loc=loc,
ncol=ncol,
fancybox=fancybox,
borderaxespad=borderaxespad)
rotation = optx['rotation'] if 'rotation' in optx else '30'
fontsize = optx['fontsize'] if 'fontsize' in optx else 12
plt.xticks(rotation=rotation, fontsize=fontsize)
plt.legend(frameon=False)
if len(title) > 0:
plt.title(title)
plt.subplots_adjust(left=0.65, bottom=bottom)
if chartname is not None and len(chartname) > 5:
plt.savefig(chartpath, format=chartformat)
elif backend is not None and backend.lower() == 'tkagg':
plt.show()
return chartpath, ax, fig
def topic_theme_ipo(**opts):
from _alan_str import find_mdb
subtab = getKeyVal(opts, 'subtab', '')
opts.pop('output', None) # not output type assigned
updTF = opts.pop('updTF', False)
jobj = json.loads(opts.pop('jobj', '{}'))
sector = opts.pop('sector', '')
if len(sector) > 5:
sLst = sector.split(',')
jobj.update(sector={"$in": sLst})
#df = get_ipoReport(updTF=updTF,**opts) # DEPRECATED
#df = create_topic_theme_ipo(updTF=updTF,**opts) # SETUP in crontab
tablename = opts.pop('tablename', None)
if tablename is None:
tablename = 'topic_theme_ipo'
df, _, _ = find_mdb(jobj, dbname='ara', tablename=tablename, dfTF=True)
colX = [
'ticker', 'ipoDate', 'marketCap', 'ipoPrice', 'Price', 'closePrice',
'ipoChg%', 'dayChg%', 'EPS', 'Range52Week', 'Company', 'sector',
'industry', 'pubDate'
]
df = subDict(df, colX)
#-onTheFly run, not used
#-run daily crontab to create 'topic_theme_ipo' table via
#-python3 -c "from ipoReport import create_topic_theme_ipo as ctt;ctt()"
if subtab.lower() == 'conflict':
df = df.query('"ipoChg%">10')
return df
def iex_spark_process(dd, dq=[], **optx):
''' Utility function
to process data in iex_spark_batch()
'''
from _alan_calc import renameDict, subDict
dx = []
for k, v in dd.items():
mtmp = v['chart']
m = [x for x in mtmp if x['close'] is not None]
if len(m) < 1:
continue
# grab xclose value from dq
xq = [x for x in dq if x['ticker'] == k]
if len(xq) > 0:
xclose = getKeyVal(xq[0], 'xclose', None)
else:
xclose = None
for j, x in enumerate(m):
m[j].update(ticker=k.replace('.', '-'))
pbdt = pd.Timestamp("{} {}".format(x['date'], x['minute']))
hhmm = x['minute'].replace(":", "")
pbdate = int(x['date'].replace("-", ""))
epochs = int(pbdt.strftime('%s000'))
m[j].update(pbdt=pbdt, pbdate=pbdate, epochs=epochs, hhmm=hhmm)
if xclose is not None and xclose > 0:
try:
pchg = x['close'] / xclose - 1
change = x['close'] - xclose
m[j].update(xclose=xclose, pchg=pchg, change=change)
except Exception as e:
sys.stderr.write("**ERROR:{},{}\n{}\n".format(k, j, m[j]))
#if debugTF:
# sys.stderr.write("====={}\n{}\n".format(k,m))
dx.extend(m)
return dx
def topic_theme_media_OLD(**opts):
from _alan_str import find_mdb
subtab = getKeyVal(opts, 'subtab', 'blacklist')
if subtab.lower() == 'whitelist':
jobj = {"rrt": {"$gt": 10}}
dtmp, mDB, errmsg = find_mdb(jobj,
tablename='topic_theme_media',
dbname='ara',
dfTF=True)
df = dtmp.sort_values(by='rrt', ascending=False).iloc[:100]
elif subtab.lower() == 'conflict':
dtmp, mDB, errmsg = find_mdb({},
tablename='topic_theme_media',
dbname='ara',
dfTF=True)
a1 = list(dtmp.query("rrt>=1.0")['ticker'].unique())
a2 = list(dtmp.query("rrt<=-1.0")['ticker'].unique())
aLst = set(a1).intersection(set(a2))
df = dtmp.loc[dtmp['ticker'].isin(aLst)].sort_values(by='ticker')
else:
jobj = {"rrt": {"$lt": -10}}
dtmp, mDB, errmsg = find_mdb(jobj,
tablename='topic_theme_media',
dbname='ara',
dfTF=True)
df = dtmp.sort_values(by='pbdate', ascending=False).iloc[:100]
colX = [
"ticker", "pbdate", "start", "end", "sPrice", "ePrice", "rrt",
"Price", "SegmentDscript", "CallDscript", "Company"
]
df = subDF(df, colX)
return df
def hourly_comment(tkLst,fdLst,tablename=None,lang=None,dbname="ara",hostname='localhost',output=None,start=None,end=None,topic=None,subtopic=None,**optx):
from _alan_str import find_mdb
if fdLst=='*':
field = {}
else:
field = set(fdLst.split(','))
limit=0
instrument = getKeyVal(optx,'instrument','stock')
rpt_hm = getKeyVal(optx,'rpt_hm',None)
hhmm = getKeyVal(optx,'hhmm',None)
pqint(tablename,optx,file=sys.stderr)
#if subtopic.lower() == 'sector':
tablename="mkt_briefing_media"
if subtopic == 'sector':
tablename="hourly_report"
if len(tkLst)<1:
jobj={"ticker":{"$in":["^GSPC","^DJI","^IXIC"]}}
sortLst=['rpt_time']
field={'ticker','comment','rpt_time','rpt_hm','rpt_status','title','label'}
limit=3
elif rpt_hm is not None or len(tkLst)>0:
tablename="mkt_briefing_details"
jobj={}
if rpt_hm is not None:
hmLst = [int(x) for x in rpt_hm.split(',')]
jobj.update(rpt_hm={"$in":hmLst})
if len(tkLst)>0 and tkLst[0]!='$' :
jobj.update(ticker={"$in":tkLst})
if len(field)<1:
field = {"ticker","label","rpt_hm","rpt_time","cprice","xprice","comment","rpt_date","pngname"}
sortLst=['rpt_time']
elif hhmm is not None:
hmLst = [int(x) for x in hhmm.split(',')]
jobj={"hhmm":{"$in":hmLst}}
sortLst=['pbdt']
else:
jobj={}
sortLst=['pbdt']
limit=1
outTF = getKeyVal(optx,'outTF',True)
df,_,_=find_mdb(jobj,tablename=tablename,dbname=dbname,field=field,sortLst=sortLst,limit=limit,dfTF=outTF)
data = df # data = data_output(df,output)
pqint( "===hourly_comment():\nFind:{},Field:{},Sort:{}\n".format(jobj,field,sortLst) ,file=sys.stderr)
pqint( " --tkLst: {},fdLst: {}".format(tkLst,fdLst) ,file=sys.stderr)
pqint( " --from tablename: {}".format(tablename) ,file=sys.stderr)
pqint( " --DF:\n{}".format(df)[:200]+"\n" ,file=sys.stderr)
return data
def plot_mktidx(tkLst=[],
fig=None,
ax=None,
figsize=(11, 6),
backend='Agg',
title='',
chartformat='svg',
debugTF=False,
**optx):
sys.stderr.write("===start:{}()\n".format("plot_mktidx"))
chartpath = getKeyVal(optx, 'chartpath', '')
optx = locals()
from csv2plotj2ts import get_csv2plot
optx.update(optx['optx'])
optx.pop('optx', None)
opts = {
'debugTF': True,
'pngname': None,
'debugTF': True,
'rsiYN': True,
'renColumns': {
'^IXIC': '納斯達克指數',
'^DJI': '道瓊指數',
'^GSPC': '標普500'
},
'colLst': 'close,pchg,epochs,ticker',
'hdrLst': None,
'npar': 15,
'days': 730,
'j2ts': None,
'start': None,
'pivot_group': 'ticker',
'gap': '1m',
'trendTF': False,
'ohlcTF': False,
'src': 'yh',
'pivot_value': 'close',
'end': None,
'title': '美股大盤走勢2019-09-17',
'lang': 'cn',
'backend': 'tkAgg',
'nbins': 6,
'tsTF': True,
'ranged': '1d',
'ohlcComboTF': False,
'sep': '|',
'interpolateYN': False,
'xaxis': 'epochs',
'pltStyle': 'classic',
'x_fmt': '%H:%M'
}
if len(tkLst) < 1:
tkLst = ['^GSPC', '^DJI', '^IXIC']
plt.switch_backend(backend)
if len(chartpath) > 5:
optx.update(pngname=chartpath)
datax, fig, ax = get_csv2plot(tkLst, opts=opts, **optx)
return (chartpath, ax, fig)
def yh_financials(ticker,
modules='incomeStatementHistory',
saveDB=False,
clientM=None,
useDB=False,
dbname='ara',
tablename='',
**optx):
''' pull financials from yahoo and save to 'tablename'
e.g.,
python3 -c "from yh_chart import yh_financials as yf;yf('AAPL');"
'''
debugTF = getKeyVal(optx, 'debugTF', False)
types = getKeyVal(optx, 'types', 'quoteSummary')
xmod = modules
jdM = []
if useDB is True:
jdTmp, dbM, err = find_mdb({"ticker": ticker},
clientM,
dbname=dbname,
tablename='yh_' + xmod)
if len(jdTmp) < 1:
return []
jdMod = jdTmp[0]
else:
try:
jdTmp = yh_quoteSummary(ticker, modules=xmod, **optx)
jdMod = jdTmp['quoteSummary']['result'][0][xmod]
jD = proc_summaryQuote(jdMod, ticker=ticker, xmod=xmod)
if jD:
jdM = [jD] if not isinstance(jD, list) else jD
except:
sys.stderr.write("**ERROR:{} not found via {}\n".format(
ticker, modules))
if saveDB is True and len(tablename) > 0 and len(jdM) > 0:
m, dbM, err = insert_mdb(jdM,
clientM=clientM,
tablename=tablename,
**optx)
sys.stderr.write("=== {} of {} saved to {}\n".format(
ticker, jdM[-1], tablename))
sys.stderr.write("{}\n".format(
pd.DataFrame(jdM)[['ticker', 'module']].to_string(index=False)))
return jdM
def mongo_upd_tst(jobj={},limit=0,sortLst={},field={},dfTF=False,debugTF=False,**optx):
hostname,port,dbname,tablename = getKeyVal(optx,
['hostname','port','dbname','tablename'], ['localhost',27017,'ara','rssNews'])
data,_,_=find_mdb(jobj,tablename=tablename,dbname=dbname,field=field,sortLst=sortLst,limit=limit,dfTF=dfTF)
if len(data)<1:
return None
if debugTF is True:
prn_dict(field=field,sortLst=sortLst,limit=limit,**optx)
return data
def geteach_financials_history(ticker,fdLst,**opts):
debugTF = getKeyVal(opts,'debugTF',False)
subtopic = getKeyVal(opts,'subtopic','')
jobj={'ticker':ticker}
if subtopic == 'eps':
tablename='earnings_yh'
jobj.update({'actualEPS':{'$ne':np.nan}})
jd=list(mgDB[tablename].find(jobj,{'_id':0},sort=[("pbdate",-1)]))
if len(jd)<1:
return []
for j,xd in enumerate(jd):
jd[j]['EPSReportDate']="{}{}{}{}-{}{}-{}{}".format(*list(str(jd[j]['pbdate'])))
datax = pd.DataFrame(jd)
elif subtopic == 'roe':
freq = getKeyVal(opts,'freq','Q')
tablename='qS_IS_{}'.format('A' if freq=='A' else 'Q')
jd=list(mgDB[tablename].find(jobj,{'_id':0}))
if len(jd)<1:
return []
datax = pd.DataFrame(jd) if len(jd)>0 else []
tablename='qS_BS_{}'.format('A' if freq=='A' else 'Q')
jd=list(mgDB[tablename].find(jobj,{'_id':0}))
if len(jd)>0:
for j,xd in enumerate(jd):
a=jd[j]['pbdate']
jd[j]['reportDate']='{:04d}-{:02d}-{:02d}'.format(int(a/10000),int(a/100)%100,a%100)
jd[j]['freq']=freq
d2 = pd.DataFrame(jd)
datax = datax.merge(d2,on=['ticker','pbdate','endDate'],how='inner')
if debugTF:
sys.stderr.write("=====geteach_financials_history() d2:{}".format(d2))
if 'netIncome' in datax.columns and 'totalStockholderEquity' in datax.columns:
datax['roe'] = datax['netIncome']/datax['totalStockholderEquity']
if debugTF:
sys.stderr.write("=====geteach_financials_history() datax:{}".format(datax))
datax=subDF(datax,['ticker','roe','pbdate','reportDate','freq','endDate','netIncome','totalStockholderEquity'])
sys.stderr.write(" ---datax:{}\n{}\n".format(ticker,datax))
else:
return []
return datax
def search_mp3(tkLst,fdLst,**opts):
tagContent='''
<audio class="audioClass" controls>
<source src="{}" type="audio/mpeg">
Your browser does not support the audio tag.
</audio>
'''
attrType=getKeyVal(opts,'attrType','src') # tag|src|content for tag_setup|src_filename|content
tagContent=getKeyVal(opts,'tagContent',tagContent)
limit=getKeyVal(opts,'limit',30)
aliasDir=getKeyVal(opts,'aliasDir','/apps/fafa/pyx/alan/US/mp3_hourly')
routeDir=getKeyVal(opts,'routeDir','/US/mp3_hourly')
filename=getKeyVal(opts,'filename','*.mp3')
fpath="{}/{}".format(aliasDir,filename)
fpLst = sorted(iglob(fpath), key=os.path.getmtime, reverse=True)
sys.stderr.write("===Search {}\n".format(attrType))
sys.stderr.write(" --searching:{}, found:{}\n".format(fpath,fpLst))
data=[]
for fname in fpLst[:limit]:
fp=fname.replace(aliasDir,routeDir)
if attrType=='tag':
data=tagContent.format(fp)
return data
elif attrType=='content':
fp=open(fname,'rb')
data=fp.read()
fp.close()
return data
else:
data=fp
return data
#data.append(fp)
#sys.stderr.write(" --Ret Data:\n{}\n".format(data))
return data
def search_list(tkLst,fdLst,**opts):
topic,start,subtopic,output = getKeyVal(opts,['topic','start','subtopic','output'],[None,None,None,None])
hostname,dbname,tablename,lang = getKeyVal(opts,['hostname','dbname','tablename','lang'],['localhost','ara',None,None])
pqint("Using topic:{},subtopic:{},field:{}".format(topic,subtopic,fdLst) ,file=sys.stderr)
if hasattr(subtopic,'__len__') is not True:
subtopic = ''
if topic in ['filter','recommend','AI','market']: # select either [getlist_filter|getlist_recommend]
if 'id' not in opts:
id = None
funcName = "getlist_{}".format(topic)
if funcName in globals():
searchListFunc = globals()[funcName]
pqint( "Applying {}: {}".format(funcName,searchListFunc) ,file=sys.stderr)
data = searchListFunc(fdLst,subtopic=subtopic,start=start,id=id)
# data = data_output(df,output)
return data
elif topic in ['ytdRtn']:
from ytdRtn_calc import ytdRtn_calc,ytdOTF
if subtopic is None or len(subtopic)<1:
subtopic='sector'
if start is None or len(start)<1:
start=20200219
data = ytdRtn_calc(start=start,group=subtopic)
return data
#tablename='ytdRtn_{}'.format(subtopic)
#data=ytdOTF(ytdRtn_calc,start=start,deltaTolerance=900,tablename=tablename,zpkChk=['group'],zpk=[subtopic],group=subtopic,debugTF=True,dbname='ara')
#sys.stderr.write(" --ytdRtn_calc data:{}\n{}\n".format(type(data),data))
#outTF=opts.pop('outTF',True)
#if outTF:
# data=pd.DataFrame(data)
#return data
elif topic in ['peers']:
data = geteach_peers(tkLst,fdLst,subtopic=subtopic,output=output)
return data
if tablename is None:
tablename='spdr_list'
data=geteach_list(tkLst,fdLst,tablename=tablename,lang=lang,dbname=dbname,hostname=hostname)
# data = data_output(data,output)
return data
def search_allocation(tkLst,fdLst,**opts):
output = getKeyVal(opts,'output',None)
from bl_portopt2 import bl_example
try:
data = bl_example(tkLst)
data['mktCapWeigh'] = data['mktCapWeigh']*100.0
data['weight'] = data['weight']*100.0
data['rrt'] = data['rrt']*100.0
pd.options.display.float_format = '{:.2f}'.format
# data = data_output(data,output)
except Exception as e:
data = None
return data
def mongo_en2cn(jobj={},limit=0,sort=[],field={},debugTF=False,src='en',dest='zh-TW',**optx):
'''
RSS news title translator
Find data via 'jobj' of 'dbname':'tablename' of 'field'
Then translate 'field' excepnt 'field'_cn from 'src' to 'dest'
Note: any field has value in 'field'_cn won't be re-translated
'''
hostname,port,dbname,tablename = getKeyVal(optx,
['hostname','port','dbname','tablename'], ['localhost',27017,'ara','rssNews'])
def upd_data(currM,incr,**myquery):
myquery.pop("_id",None)
newSet = title2cn(incr,debugTF=debugTF,src=src,dest=dest,**myquery)
mysetting = { "$set": newSet }
if len(newSet)<1:
print("===NOTHING TO UPDATE {}\n".format(myquery),file=sys.stderr)
return myquery
print("===UPDATING From:{}:\nTo:{}\nVia:{}\n".format(myquery,newSet,currM),file=sys.stderr)
xout = currM.update_many(myquery, mysetting)
if debugTF:
print(" --updated out[{}]:{}\n".format(xout.modified_count,xout.raw_result),file=sys.stderr)
return newSet
clientM=optx.pop('clientM',None)
if not clientM:
dbM=MongoClient("{}:{}".format(hostname,port))[dbname]
else:
dbM=clientM[dbname]
currM=dbM[tablename]
if debugTF:
print(dbM,currM,file=sys.stderr)
# DO NOT TRANSLATE, AD not used
jobj.update({"source.title":{"$nin":["Insider Monkey","TipRanks"]}})
dLst=list(currM.find(jobj,field,sort=sort,limit=limit))
if len(dLst)<1:
return []
incr={"count":1}
oLst=[]
for j,dd in enumerate(dLst):
try:
print("===INPUT[{}]: {} : {}\n".format(j,jobj,dd),file=sys.stderr)
ret = upd_data(currM,incr,**dd)
ret.update(jobj)
oLst.append(ret)
except Exception as e:
print("**ERROR:{} mongo_en2cn: {} @ {}".format(j,str(e),dd),file=sys.stderr)
if debugTF:
prn_dict(field=field,sort=sort,limit=limit,**optx,file=sys.stderr)
return oLst
def useWeb(jobj={'ticker': '^GSPC'},
colx='pbdt',
dbname='ara',
tablename='yh_quote_curr',
mmGap=30,
**optx):
'''return [True|Flase] action for using web or DB
based on 'tablename' 'colx' field
'''
from _alan_calc import conn2mgdb
debugTF = getKeyVal(optx, 'debugTF', False)
webTF = optx.pop('webTF', None)
if webTF is not None and isinstance(webTF, bool):
return webTF
cdt = getKeyVal(optx, 'cdt', datetime.datetime.now())
mgDB = conn2mgdb(dbname=dbname)
dc = mgDB[tablename].find_one({
"$query": jobj,
"$orderby": {
colx: -1
}
}, {
colx: 1,
"_id": 0
})
if len(dc) < 1:
return True
pdt = dc[colx]
mmPassed = pd.Timedelta(cdt - pdt).total_seconds() / 60.0
webTF = mmPassed > mmGap
if debugTF is True:
sys.stderr.write("{}|{}|{}|{}|{}\n".format("webTF", "cdt", "pdt",
"mmPassed", "mmGap"))
sys.stderr.write("{}|{}|{}|{:.0f}|{}\n".format(webTF, cdt, pdt,
mmPassed, mmGap))
return webTF
def geteach_history(ticker,fdLst,**opts):
# for topic list 'financial','daily','minute'
topic = getKeyVal(opts,'topic','daily')
argName = "geteach_{}_history".format(topic)
try:
if argName not in globals():
argName="geteach_daily_history"
sys.stderr.write(" RUNNING Ticker:{}[fdLst:{}]:topic:{} of {}()\n".format(ticker,fdLst,topic,argName) )
argFunc = globals()[argName]
df = argFunc(ticker,fdLst,**opts)
except Exception as e:
sys.stderr.write("**ERROR: {} @{}()".format(str(e),argName) )
df = ''
return df
def geteach_minute_history(ticker,fdLst,**opts):
''' pull minute data from database (default)
OR web if 'webTF' is True
'''
webTF=subtopic=getKeyVal(opts,'webTF',False)
if webTF:
opts.update(src='yh')
return geteach_minute_web(ticker,fdLst,**opts)
data=geteach_minute_db(ticker,fdLst,**opts)
if data is None or len(data)<1:
sys.stderr.write("**WARNING:{}\n".format("data not found in DB, live pulling"))
opts.update(src='yh')
return geteach_minute_web(ticker,fdLst,**opts)
return data
def search_hist(tkLst,fdLst,**opts):
output=getKeyVal(opts,'output','json')
topic=getKeyVal(opts,'topic','daily')
opts.pop('ticker',None)
data=pd.DataFrame()
dd=[]
for ticker in tkLst:
df=geteach_history(ticker,fdLst,**opts)
sys.stderr.write(" --DF: {}\n{}\n".format(ticker,type(df)))
if isinstance(df,pd.DataFrame) and len(df)>0:
data=data.append(df,ignore_index=True)
elif isinstance(df,list) and len(df)>0:
dd=dd.extend(df)
else:
continue
if len(dd)>0:
data=pd.DataFrame(dd)
sys.stderr.write(" --DATA[{}] tail:\n{}\n".format(len(data),data.tail()))
if topic not in ['daily','minute']:
return data # data_output(data,output)
renameDict(data,{"name":"ticker"})
colx=['ticker','epochs','open','high','low','close','volume','xclose','change','pchg','pbdt','hhmm','pbdate']
data=subDF(data,colx)
return data # data_output(data,output)
def page_report(**optx):
topic, subtopic, search = getKeyVal(optx, ['topic', 'subtopic', 'search'],
['report', None, 'comment'])
optx.update(search=search, topic=topic, subtopic=subtopic, output='json')
pqint(" --page_report INPUT:\n", optx, file=sys.stderr)
if subtopic == 'company':
optx.update(topic='news')
ret = page_api(**optx)
pqint(" --page_report OUTPUT:\n",
type(ret),
"{}".format(ret)[:30],
file=sys.stderr)
if isinstance(ret, list) and len(ret) > 0 and subtopic != 'company':
return ret[0]
else:
return ret
def plot_barh2chart(df,
fig=None,
ax=None,
figsize=(11, 6),
backend='Agg',
title='',
chartformat='svg',
debugTF=False,
**optx):
if 'ticker' in df and 'peRatio' in df:
df = pd.DataFrame(df.dropna().sort_values(by='peRatio').set_index(
'ticker', drop=True))
elif 'ticker' in df:
df = pd.DataFrame(df.dropna().set_index('ticker', drop=True))
ngrid = df.shape[1]
kind = getKeyVal(optx, 'kind', 'barh')
pctTF = getKeyVal(optx, 'pctTF', False)
vtitle = getKeyVal(optx, 'vtitle', ["1-Day Price Change %", "PE Ratio"])
if pctTF is True:
vfmt = getKeyVal(optx, 'vfmt', ["{:.2}%", "{:.0f}"])
else:
vfmt = getKeyVal(optx, 'vfmt', ["{:.2%}", "{:.0f}"])
if len(df) < 1:
return '', None, None
if backend is not None and len(backend) > 1:
plt.switch_backend(backend)
bottom = optx['bottom'] if 'bottom' in optx else 0.35
style = getKeyVal(optx, 'style', 'classic')
plt.style.use(style)
chartpath, chartname = gen_chartpath(**optx)
fig, axes = plt.subplots(ngrid, figsize=figsize, sharey=True)
axes = df.plot(kind=kind,
legend=False,
ax=plt.gca(),
subplots=True,
sharey=True)
for j in range(ngrid):
if vtitle[j] is not None and len(vtitle[j]) > 0:
axes[j].set_title(vtitle[j])
for i, v in enumerate(df.iloc[:, j]):
axes[j].text(v, i + .05, vfmt[j].format(v), fontweight='bold')
plt.subplots_adjust(left=.1,
bottom=bottom,
right=.90,
top=.90,
wspace=.20,
hspace=.50)
if chartname is not None and len(chartname) > 5:
plt.savefig(chartpath,
format=chartformat) #, bbox_inches='tight',dpi=1000)
elif backend is not None and backend.lower() == 'tkagg':
plt.show()
return (chartpath, axes, fig)