def parseArgs():
long_options = [
"action=", "sql=", "mongouri=", "outputfile=", "inputfile=",
"startdate=", "enddate=", "symbol="
]
args = myutils.verifyArgs(myutils.parseArgs(long_options),
required_args=['--action'])
myutils.logTrace("parseArgs: verified args=", args)
return args
def fetchCsv(httpGetUrl):
r = requests.get(httpGetUrl)
myutils.logTrace("fetchCsv: httpGetUrl:", httpGetUrl, "r.status_code",
r.status_code)
if r.status_code == 200:
return r.text.split("\n")
else:
r.raise_for_status()
def fetchYahooHistoricalDataCsv(symbol, startdate, enddate):
url = "http://ichart.finance.yahoo.com/table.csv?s={0}&a={1}&b={2}&c={3}&d={4}&e={5}&f={6}&g=d&ignore=.csv".format(
symbol, startdate.month - 1, startdate.day, startdate.year,
enddate.month - 1, enddate.day, enddate.year)
retMe = list(
map(lambda quote: buildQuoteRecord(quote, symbol),
convertCsvToJson(fetchCsv(url))))
myutils.logTrace("fetchYahooHistoricalDataCsv: len(retMe):", len(retMe),
"retMe[0]:",
(json.dumps(retMe[0]) if len(retMe) > 0 else ""))
return retMe
def fetchYahooHistoricalData(symbol, startdate, enddate):
myutils.logTrace("fetchYahooHistoricalData:", symbol, startdate, enddate)
sql = 'select * from yahoo.finance.historicaldata where startDate="{0}" and endDate="{1}" and symbol="{2}"'.format(
startdate, enddate, symbol)
r = fetchYahooData(sql)
if not haveQuoteData(r):
myutils.logTrace("fetchYahooHistoricalData: response:", json.dumps(r))
return []
else:
return r["query"]["results"]["quote"]
def analyzeGoldenCrosses(quotes):
goldenCrossQuotes = list(
filter(
lambda quote: "GoldenCross" in quote and quote["GoldenCross"] ==
True, quotes))
goldenCrossMovingUpQuotes = list(
filter(
lambda quote: quote["200-SimpleMovingAverage-1-PeriodReturn"] > 0
and quote["50-SimpleMovingAverage-1-PeriodReturn"] > 0,
goldenCrossQuotes))
myutils.logTrace("analyzeGoldenCrosses: goldenCrossQuotes:",
len(goldenCrossQuotes), json.dumps(goldenCrossQuotes))
myutils.logTrace("analyzeGoldenCrosses: goldenCrossMovingUpQuotes:",
len(goldenCrossMovingUpQuotes),
json.dumps(goldenCrossMovingUpQuotes))
myutils.logTrace(
"analyzeGoldenCrosses: goldenCrossQuotes:", "\n",
"mean(Adj Close-5-PeriodForwardReturn):",
meanField(goldenCrossQuotes, "Adj Close-5-PeriodForwardReturn"), "\n",
"mean(Adj Close-20-PeriodForwardReturn):",
meanField(goldenCrossQuotes, "Adj Close-20-PeriodForwardReturn"), "\n",
"mean(Adj Close-65-PeriodForwardReturn):",
meanField(goldenCrossQuotes, "Adj Close-65-PeriodForwardReturn"), "\n",
"mean(Adj Close-130-PeriodForwardReturn):",
meanField(goldenCrossQuotes, "Adj Close-130-PeriodForwardReturn"),
"\n", "stdev(Adj Close-130-PeriodForwardReturn):",
stdevField(goldenCrossQuotes, "Adj Close-130-PeriodForwardReturn"),
"\n", "mean(Adj Close-260-PeriodForwardReturn):",
meanField(goldenCrossQuotes, "Adj Close-260-PeriodForwardReturn"),
"\n", "stdev(Adj Close-260-PeriodForwardReturn):",
stdevField(goldenCrossQuotes, "Adj Close-260-PeriodForwardReturn"))
myutils.logTrace(
"analyzeGoldenCrosses: goldenCrossMovingUpQuotes:", "\n",
"mean(Adj Close-5-PeriodForwardReturn):",
meanField(goldenCrossMovingUpQuotes,
"Adj Close-5-PeriodForwardReturn"), "\n",
"mean(Adj Close-20-PeriodForwardReturn):",
meanField(goldenCrossMovingUpQuotes,
"Adj Close-20-PeriodForwardReturn"), "\n",
"mean(Adj Close-65-PeriodForwardReturn):",
meanField(goldenCrossMovingUpQuotes,
"Adj Close-65-PeriodForwardReturn"), "\n",
"mean(Adj Close-130-PeriodForwardReturn):",
meanField(goldenCrossMovingUpQuotes,
"Adj Close-130-PeriodForwardReturn"), "\n",
"stdev(Adj Close-130-PeriodForwardReturn):",
stdevField(goldenCrossMovingUpQuotes,
"Adj Close-130-PeriodForwardReturn"), "\n",
"mean(Adj Close-260-PeriodForwardReturn):",
meanField(goldenCrossMovingUpQuotes,
"Adj Close-260-PeriodForwardReturn"), "\n",
"stdev(Adj Close-260-PeriodForwardReturn):",
stdevField(goldenCrossMovingUpQuotes,
"Adj Close-260-PeriodForwardReturn"))
def processHistoricalData(args):
args = myutils.verifyArgs(args, required_args=['--symbol'])
db = myutils.getMongoDb(os.environ["R4_MONGO_URI"])
quotes = db.quotes.find({"Symbol": args["--symbol"]}, sort=[
("_id", 1)
]) # note: _id's are sorted by date
quotes = computeMetaMetrics(list(quotes))
for quote in quotes:
myutils.logTrace("processHistoricalData: ",
json.dumps(quote, indent=2, sort_keys=True))
analyzeGoldenCrosses(quotes)
def playWithPandas(args):
args = myutils.verifyArgs(args, required_args=['--symbol'])
db = myutils.getMongoDb(os.environ["R4_MONGO_URI"])
quotes = db.quotes.find({"Symbol": args["--symbol"]}, sort=[
("_id", 1)
]) # note: _id's are sorted by date
quotes = computeMetaMetrics(list(quotes))
df = pandas.DataFrame(quotes)
myutils.logTrace("playWithPandas: df:")
myutils.logTrace(df)
# df.describe(): basic column stats like mean and stdev
myutils.logTrace("playWithPandas: df.describe():")
myutils.logTrace(df.describe())
# get a single column, returned as a "Series"
myutils.logTrace("playWithPandas: df['Adj Close']:")
myutils.logTrace(df['Adj Close'])
# slice a few rows
myutils.logTrace("playWithPandas: df[0:10]:")
myutils.logTrace(df[0:10])
myutils.logTrace("playWithPandas: df.loc[:,['Date','Close']]")
myutils.logTrace(df.loc[:, ['Date', 'Close']])
# df.at: fast access to scalar
# df.iat: same as df.at, but only allows index notation (df.at allows row/column labels)
myutils.logTrace("playWithPandas: df.at[8,'Adj Close']")
myutils.logTrace(df.at[8, 'Adj Close'])
# iloc: select rows/cols by index
# df.iloc[3]
# df.iloc[3:7]
# df.iloc[[1,3,5],[0:2]]
# df.iloc[[True,True,False],[0:2]]
myutils.logTrace("playWithPandas: df.iloc[[1,2,4],[0,2]]")
myutils.logTrace(df.iloc[[1, 2, 4], [0, 2]])
# returns a Series of booleans
# can be used for selection
myutils.logTrace("playWithPandas: df['Date'] > '2016-01-01'")
myutils.logTrace(df['Date'] > '2016-01-01')
# select by boolean
myutils.logTrace("playWithPandas: df[ df['Date'] > '2016-01-01' ] ")
myutils.logTrace(df[df['Date'] > '2016-01-01'])
# isin()
myutils.logTrace(
"playWithPandas: df[ df['Date'].isin(['2016-01-04','2016-04-06']) ] ")
myutils.logTrace(df[df['Date'].isin(['2016-01-04', '2016-04-06'])])
# isnull(), notnull(): return boolean series
# df.mean(0): means of columns (default)
# df.mean(1): means of rows
# df.mean(0, skipna=True (default)): means of columns
# df.sum(0)
# z-scaling: df_z = (df - df.mean()) / df.std()
# http://pandas.pydata.org/pandas-docs/stable/basics.html#basics-stats
myutils.logTrace("playWithPandas: df[ df['GoldenCross'] == True ]")
goldenCrosses = df[df['GoldenCross'] == True]
myutils.logTrace(goldenCrosses)
myutils.logTrace("playWithPandas: means, stdevs: ")
myutils.logTrace(goldenCrosses.loc[:, [
'Adj Close-130-PeriodForwardReturn',
'Adj Close-260-PeriodForwardReturn'
]].mean())
myutils.logTrace(goldenCrosses.loc[:, [
'Adj Close-130-PeriodForwardReturn',
'Adj Close-260-PeriodForwardReturn'
]].std())
myutils.logTrace(goldenCrosses.loc[:, [
'Adj Close-130-PeriodForwardReturn',
'Adj Close-260-PeriodForwardReturn'
]].std(ddof=0)) # unbiased
myutils.logTrace(
"playWithPandas: df[ (df['GoldenCross'] == True) & (df['200-SimpleMovingAverage-1-PeriodReturn'] > 0) & (df['50-SimpleMovingAverage-1-PeriodReturn'] > 0)]"
)
goldenCrosses = df[(df['GoldenCross'] == True)
& (df['200-SimpleMovingAverage-1-PeriodReturn'] > 0) &
(df['50-SimpleMovingAverage-1-PeriodReturn'] > 0)]
myutils.logTrace(goldenCrosses)
myutils.logTrace("playWithPandas: means, stdevs: ")
myutils.logTrace(goldenCrosses.loc[:, [
'Adj Close-130-PeriodForwardReturn',
'Adj Close-260-PeriodForwardReturn'
]].mean())
myutils.logTrace(goldenCrosses.loc[:, [
'Adj Close-130-PeriodForwardReturn',
'Adj Close-260-PeriodForwardReturn'
]].std())
myutils.logTrace(goldenCrosses.loc[:, [
'Adj Close-130-PeriodForwardReturn',
'Adj Close-260-PeriodForwardReturn'
]].std(ddof=0)) # unbiased
myutils.logTrace(
"playWithPandas: statistics.pstdev: ",
statistics.pstdev(
goldenCrosses['Adj Close-260-PeriodForwardReturn'].tolist()))
myutils.logTrace("playWithPandas: goldenCrosses.to_dict('records')")
myutils.logTrace(json.dumps(goldenCrosses.to_dict('records'), indent=2))
def upsertQuote(db, quote):
myutils.logTrace("upsertQuote: ", json.dumps(quote))
db.quotes.update_one({"_id": quote["_id"]}, {"$set": quote}, upsert=True)
return quote
myutils.logTrace("playWithPandas: goldenCrosses.to_dict('records')")
myutils.logTrace(json.dumps(goldenCrosses.to_dict('records'), indent=2))
#
# main entry point ---------------------------------------------------------------------------
#
args = parseArgs()
#
#
#
if args["--action"] == "fetchYahooData":
args = myutils.verifyArgs(args, required_args=['--sql'])
r = fetchYahooData(args['--sql'])
myutils.logTrace("fetchYahooData:", json.dumps(r, indent=2,
sort_keys=True))
#
#
#
elif args["--action"] == "fetchYahooHistoricalDataCsv":
args = myutils.setArgDefaultValue(args, '--startdate', '1950-01-01')
args = myutils.setArgDefaultValue(args, '--enddate',
date.today().strftime("%Y-%m-%d"))
args = myutils.verifyArgs(
args, required_args=['--symbol', '--startdate', '--enddate'])
fetchYahooHistoricalDataCsv(
args['--symbol'], datetime.strptime(args['--startdate'], "%Y-%m-%d"),
datetime.strptime(args['--enddate'], "%Y-%m-%d"))