def parseArgs():
long_options = [
"action=", "sql=", "mongouri=", "outputfile=", "inputfile=",
"startdate=", "enddate=", "symbol="
]
args = myutils.verifyArgs(myutils.parseArgs(long_options),
required_args=['--action'])
myutils.logTrace("parseArgs: verified args=", args)
return args
def processHistoricalData(args):
args = myutils.verifyArgs(args, required_args=['--symbol'])
db = myutils.getMongoDb(os.environ["R4_MONGO_URI"])
quotes = db.quotes.find({"Symbol": args["--symbol"]}, sort=[
("_id", 1)
]) # note: _id's are sorted by date
quotes = computeMetaMetrics(list(quotes))
for quote in quotes:
myutils.logTrace("processHistoricalData: ",
json.dumps(quote, indent=2, sort_keys=True))
analyzeGoldenCrosses(quotes)
def downloadYahooHistoricalData(args):
db = myutils.getMongoDb(os.environ["R4_MONGO_URI"])
args = myutils.setArgDefaultValue(args, '--startdate', '1950-01-01')
args = myutils.setArgDefaultValue(args, '--enddate',
date.today().strftime("%Y-%m-%d"))
args = myutils.verifyArgs(
args, required_args=['--symbol', '--startdate', '--enddate'])
historicalQuotes = fetchYahooHistoricalDataCsv(
args['--symbol'], datetime.strptime(args['--startdate'], "%Y-%m-%d"),
datetime.strptime(args['--enddate'], "%Y-%m-%d"))
# historicalQuotes = searchAndFetchYahooHistoricalData( args['--symbol'],
# args['--startdate'],
# args['--enddate'] )
[upsertQuote(db, historicalQuote) for historicalQuote in historicalQuotes]
def playWithPandas(args):
args = myutils.verifyArgs(args, required_args=['--symbol'])
db = myutils.getMongoDb(os.environ["R4_MONGO_URI"])
quotes = db.quotes.find({"Symbol": args["--symbol"]}, sort=[
("_id", 1)
]) # note: _id's are sorted by date
quotes = computeMetaMetrics(list(quotes))
df = pandas.DataFrame(quotes)
myutils.logTrace("playWithPandas: df:")
myutils.logTrace(df)
# df.describe(): basic column stats like mean and stdev
myutils.logTrace("playWithPandas: df.describe():")
myutils.logTrace(df.describe())
# get a single column, returned as a "Series"
myutils.logTrace("playWithPandas: df['Adj Close']:")
myutils.logTrace(df['Adj Close'])
# slice a few rows
myutils.logTrace("playWithPandas: df[0:10]:")
myutils.logTrace(df[0:10])
myutils.logTrace("playWithPandas: df.loc[:,['Date','Close']]")
myutils.logTrace(df.loc[:, ['Date', 'Close']])
# df.at: fast access to scalar
# df.iat: same as df.at, but only allows index notation (df.at allows row/column labels)
myutils.logTrace("playWithPandas: df.at[8,'Adj Close']")
myutils.logTrace(df.at[8, 'Adj Close'])
# iloc: select rows/cols by index
# df.iloc[3]
# df.iloc[3:7]
# df.iloc[[1,3,5],[0:2]]
# df.iloc[[True,True,False],[0:2]]
myutils.logTrace("playWithPandas: df.iloc[[1,2,4],[0,2]]")
myutils.logTrace(df.iloc[[1, 2, 4], [0, 2]])
# returns a Series of booleans
# can be used for selection
myutils.logTrace("playWithPandas: df['Date'] > '2016-01-01'")
myutils.logTrace(df['Date'] > '2016-01-01')
# select by boolean
myutils.logTrace("playWithPandas: df[ df['Date'] > '2016-01-01' ] ")
myutils.logTrace(df[df['Date'] > '2016-01-01'])
# isin()
myutils.logTrace(
"playWithPandas: df[ df['Date'].isin(['2016-01-04','2016-04-06']) ] ")
myutils.logTrace(df[df['Date'].isin(['2016-01-04', '2016-04-06'])])
# isnull(), notnull(): return boolean series
# df.mean(0): means of columns (default)
# df.mean(1): means of rows
# df.mean(0, skipna=True (default)): means of columns
# df.sum(0)
# z-scaling: df_z = (df - df.mean()) / df.std()
# http://pandas.pydata.org/pandas-docs/stable/basics.html#basics-stats
myutils.logTrace("playWithPandas: df[ df['GoldenCross'] == True ]")
goldenCrosses = df[df['GoldenCross'] == True]
myutils.logTrace(goldenCrosses)
myutils.logTrace("playWithPandas: means, stdevs: ")
myutils.logTrace(goldenCrosses.loc[:, [
'Adj Close-130-PeriodForwardReturn',
'Adj Close-260-PeriodForwardReturn'
]].mean())
myutils.logTrace(goldenCrosses.loc[:, [
'Adj Close-130-PeriodForwardReturn',
'Adj Close-260-PeriodForwardReturn'
]].std())
myutils.logTrace(goldenCrosses.loc[:, [
'Adj Close-130-PeriodForwardReturn',
'Adj Close-260-PeriodForwardReturn'
]].std(ddof=0)) # unbiased
myutils.logTrace(
"playWithPandas: df[ (df['GoldenCross'] == True) & (df['200-SimpleMovingAverage-1-PeriodReturn'] > 0) & (df['50-SimpleMovingAverage-1-PeriodReturn'] > 0)]"
)
goldenCrosses = df[(df['GoldenCross'] == True)
& (df['200-SimpleMovingAverage-1-PeriodReturn'] > 0) &
(df['50-SimpleMovingAverage-1-PeriodReturn'] > 0)]
myutils.logTrace(goldenCrosses)
myutils.logTrace("playWithPandas: means, stdevs: ")
myutils.logTrace(goldenCrosses.loc[:, [
'Adj Close-130-PeriodForwardReturn',
'Adj Close-260-PeriodForwardReturn'
]].mean())
myutils.logTrace(goldenCrosses.loc[:, [
'Adj Close-130-PeriodForwardReturn',
'Adj Close-260-PeriodForwardReturn'
]].std())
myutils.logTrace(goldenCrosses.loc[:, [
'Adj Close-130-PeriodForwardReturn',
'Adj Close-260-PeriodForwardReturn'
]].std(ddof=0)) # unbiased
myutils.logTrace(
"playWithPandas: statistics.pstdev: ",
statistics.pstdev(
goldenCrosses['Adj Close-260-PeriodForwardReturn'].tolist()))
myutils.logTrace("playWithPandas: goldenCrosses.to_dict('records')")
myutils.logTrace(json.dumps(goldenCrosses.to_dict('records'), indent=2))
goldenCrosses['Adj Close-260-PeriodForwardReturn'].tolist()))
myutils.logTrace("playWithPandas: goldenCrosses.to_dict('records')")
myutils.logTrace(json.dumps(goldenCrosses.to_dict('records'), indent=2))
#
# main entry point ---------------------------------------------------------------------------
#
args = parseArgs()
#
#
#
if args["--action"] == "fetchYahooData":
args = myutils.verifyArgs(args, required_args=['--sql'])
r = fetchYahooData(args['--sql'])
myutils.logTrace("fetchYahooData:", json.dumps(r, indent=2,
sort_keys=True))
#
#
#
elif args["--action"] == "fetchYahooHistoricalDataCsv":
args = myutils.setArgDefaultValue(args, '--startdate', '1950-01-01')
args = myutils.setArgDefaultValue(args, '--enddate',
date.today().strftime("%Y-%m-%d"))
args = myutils.verifyArgs(
args, required_args=['--symbol', '--startdate', '--enddate'])
fetchYahooHistoricalDataCsv(