def getsavedata(stk):
firstday = '2013-01-01'
today = datetime.date.today().strftime('%Y-%m-%d')
savename=getsavename(stk,'.p')
# TODO: make path for savename
# mkdir(dirname(savename))
# load (and update if needed)
if os.path.exists(savename):
quotes=pickle.load( open(savename, "rb") )
lastquote = sorted(quotes.keys())[-1]
# update with new values
prevdate = datetime.datetime.strptime(today,'%Y-%m-%d') - datetime.timedelta(days=1)
prevdate=prevdate.strftime('%Y-%m-%d')
if lastquote != prevdate:
nextdate = datetime.datetime.strptime(lastquote,'%Y-%m-%d') + datetime.timedelta(days=1)
nextdate=nextdate.strftime('%Y-%m-%d')
pprint([prevdate, lastquote,nextdate,today])
quotes.update( ystockquote.get_historical_prices(stk,nextdate,today) )
savestock(stk,quotes)
# get all new
else:
quotes = ystockquote.get_historical_prices(stk,firstday,today)
savestock(stk,quotes)
return quotes
def display(self):
c = self.db.cursor()
now = datetime.datetime.now()
if now.weekday() == 5:
now = now - timedelta(days=1)
elif now.weekday() == 6:
now = now - timedelta(days=2)
now = now.strftime("%Y%m%d")
spynow = ystockquote.get_historical_prices('SPY', now, now)
spynow = float(spynow[1][4])
for row in c.execute("SELECT * FROM stocks ORDER BY tradeDate"):
cur = ystockquote.get_historical_prices(row[1], now, now)
cur = float(cur[1][4])
orig = row[5] * row[6]
spy_orig = row[7] * row[8]
if row[5] < 0:
continue
print "Ticker:", row[1]
print " Date:", row[3]
print " Unit Price:", row[6]
print " SPY Price:", row[8]
print " Current Price:", cur
print " Current SPY:", spynow
print " Return:", 100 * ((cur * row[5]) - orig) / orig
print " SPY Return:", 100 * ((spynow * row[7]) - spy_orig) / spy_orig
def get_percent_change(symbol):
startdate = str(nicedatestring3.getnicedatetime(one_month_ago))
list_of_percent_changes = []
pdata = ystockquote.get_historical_prices(symbol, startdate, enddate)
del pdata[0]
pdata.reverse()
#print pdata
# This gets the percent changes for all dates
for n in range(len(pdata) - 1):
day_before = float(pdata[n][4])
day = float(pdata[(n + 1)][4])
percent_change = ((day - day_before) / day) * 100
#print day, pdata[(n + 1)][0], day_before, pdata[n][0], percent_change
list_of_percent_changes.append(percent_change)
#print list_of_percent_changes
VolCalcSD.stdv(list_of_percent_changes)
threshold = VolCalcSD.stdv(list_of_percent_changes) * 2
print "threshold", threshold
#This determines the price and date when the breakthrough happened
for p in range(len(pdata) - 1):
day_before = float(pdata[p][4])
day = float(pdata[(p + 1)][4])
percent_change = ((day - day_before) / day) * 100
if percent_change > threshold:
print day, pdata[p +
1][0], day_before, pdata[(p)][0], percent_change
date_of_threshold_penetration = pdata[p + 1][0]
print
print
breakthrough = percent_change
print "breakthrough", date_of_threshold_penetration, breakthrough
break
list_of_percent_changes.append(percent_change)
#Now I have to figure out how to get the percent change per day since penetration
startdate = date_of_threshold_penetration.replace(
'-', '') #This takes out the dashes from the date it was penetrated
pdata = ystockquote.get_historical_prices(symbol, startdate, enddate)
del pdata[0]
pdata.reverse()
print pdata[0][4]
print
print symbol
for n in range(1, len(pdata)):
closing_price = float(pdata[n][4])
price_that_penetration_occurred = float(pdata[0][4])
print pdata[n][0], closing_price, (
(closing_price - price_that_penetration_occurred) /
price_that_penetration_occurred) * 100
def collect_history(self):
if self.end_date:
df = pd.DataFrame(ystockquote.get_historical_prices(
symbol=self.symbol, start_date=self.start_date, end_date=self.end_date)).T
else:
df = pd.DataFrame(ystockquote.get_historical_prices(
symbol=self.symbol, start_date=self.start_date)).T
df = df.apply(lambda x: pd.to_numeric(x), axis=1)
self.history = df
return df
def getsavedata(stk,firstday='2013-01-01',lastday='today',forceupdate=False):
if lastday == 'today':
lastday = datetime.date.today().strftime('%Y-%m-%d')
savename=getsavename(stk,'.p')
#did we dl anything? do we need to wait
dl=0
# TODO: make path for savename
# mkdir(dirname(savename))
# load (and update if needed)
if os.path.exists(savename):
quotes=pickle.load( open(savename, "rb") )
lastquote = sorted(quotes.keys())[-1]
# what is the last possible day we could have values for
# this is only meaningful of lastday is "today"
prevdate = datetime.datetime.strptime(lastday,'%Y-%m-%d') - datetime.timedelta(days=1)
prevdate=prevdate.strftime('%Y-%m-%d')
# if we dont have yestrdays quotes (and we arn't forcing a different date range)
if lastquote != prevdate and not forceupdate:
nextdate = datetime.datetime.strptime(lastquote,'%Y-%m-%d') + datetime.timedelta(days=1)
nextdate=nextdate.strftime('%Y-%m-%d')
# set the first day of data to retrieve to the
# next day (first missing day) in the data we have
firstday = nextdate
forceupdate=True
if forceupdate:
pprint([prevdate, lastquote,firstday,lastday])
quotes.update( ystockquote.get_historical_prices(stk,firstday,lastday) )
savestock(stk,quotes)
dl=1
# get all new
else:
quotes = ystockquote.get_historical_prices(stk,firstday,lastday)
savestock(stk,quotes)
dl=1
if dl: time.sleep(10)
# did we miss anything?
populateMissing(stk,quotes)
return quotes
def getQuotes(self, companyLabel):
# TODO: if NO CHANGE:
# return nothing
print 'Fetching Quotes', companyLabel
results = ystockquote.get_historical_prices(companyLabel, '20100101', '20130301')
div = ystockquote.get_dividend_yield(companyLabel)
print 'Fetched Quotes'
if str(div) == 'N/A':
div = 0
else:
div = float(div) * .01
del results[0] # column names
quotes = []
for i,entry in enumerate(results):
# second column is opening price
if i+1 >= len(results) or self.isFirstWorkDay(entry[0], results[i+1][0]):
gain = float(entry[1+2]) * (1+div)
quotes.append(gain)
return quotes
def get_percent_change(self, start_date, end_date):
"""Pulls data from Yahoo's API and calculates the percent change from the start data to the end date."""
# q = 'select * from yahoo.finance.symbol where symbol in ("'
# q += self.symbol + '")'
# Format Query for YQL
q = 'select * from yahoo.finance.historicaldata where symbol = "%s" and startDate = "%s" and endDate = "%s"' % (self.symbol, start_date, end_date)
query = urllib.quote_plus(q)
# Format URL for YQL
url = "http://query.yahooapis.com/v1/public/yql?q="
url += query + "&env=http%3A%2F%2Fdatatables.org%2Falltables.env"
# Launch Yahoo Request
r = BeautifulSoup(requests.get(url).text)
symbols = r.find_all("symbol")
# print r.prettify()
# If YQL Api is not down, simply calculate percent change
if(len(symbols) > 0):
p2 = float(symbols[0].close.string)
p1 = float(symbols[1].close.string)
self.percent_change = (p2 - p1) / (.5 * (p1 + p2)) * 100
# Otherwise call the ystocksymbol gem
else:
self.data = ystockquote.get_historical_prices(self.symbol, convert_date(start_date), convert_date(end_date))
days = len(self.data) - 1
# print self.data
p2 = float(self.data[1][4])
p1 = float(self.data[days][4])
self.percent_change = (p2 - p1) / (.5 * (p1 + p2)) * 100
def fetchQuotes(sym, start=FROM_DATE, end=CURRENT_DATE):
his = None
data = None
try:
# print start, end
data = ystockquote.get_historical_prices(sym, start, end)
except Exception:
print "Please check the dates. Data might not be available. 404 returned"
# 404 due to data yet not available
if data:
his = DataFrame(collections.OrderedDict(sorted(data.items()))).T
his = his.convert_objects(convert_numeric=True)
his.index = pd.to_datetime(his.index)
his.insert(0, 'symbol', sym, allow_duplicates=True)
# insert the date as dataframe too
his.insert(1, 'date', his.index)
# his.columns = getColumns('stock_quote_historical') # Removing as db dependency is removed
his.columns = getColumnsNoSql('stock_quote_historical')
daily = ystockquote.get_all(sym)
# print daily
# persist(his, daily, sym, end)
return his, daily
def DownLoadStocks():
#StockNameList = 'C:\Users\Makaye\Desktop\Investment\Code\yahoo_stock_symbols_4.csv'
StockNameWithExchange = 'C:\Users\Makaye\Desktop\Investment\Code\TickersNamesExhanges_4.csv'
fid = open(StockNameWithExchange,'r')
Data = fid.readlines()
StockTicker = [day.split(',')[0] for day in Data]
#StockName = [day.split(',')[1].split('\n')[0] for day in Data]
StockExchange = [day.split(',')[2].split('\n')[0] for day in Data]
fid.close()
# Create the directories if needed
BaseDir = 'C:\Users\Makaye\Desktop\Investment\Stocks'
start_date = '20100101'
end_date = '20111111'
for i in range(0,len(StockTicker)):
CurExh = StockExchange[i]
CurTick = StockTicker[i]
CurDir= os.path.join(BaseDir,CurExh)
if not os.path.exists(CurDir):
os.makedirs(CurDir)
# download the data for each exchange
OutDir = os.path.join(BaseDir,CurExh,CurTick+".csv")
if not os.path.exists(OutDir):
try:
print "DownLoading: "+CurExh+": "+CurTick+", "+str(i)+"/"+str(len(StockTicker))
fid = open(OutDir,'w')
Y=ys.get_historical_prices(StockTicker[i],start_date,end_date)
for j in Y:
temp=",".join(["%s"% el for el in j])+'\n'
fid.write(temp)
fid.close()
except:
print "Problem with: "+CurExh+": "+CurTick
def main():
# ticker_file = "input/tickers.txt"
# bucket_name = "edgarsentimentbucket"
# conn = S3Connection(argv[1], argv[2])
# path_bucket = conn.get_bucket(bucket_name)
# k = Key(path_bucket)
# k.key = ticker_file
# pathfile = k.get_contents_as_string()
# try:
# lines = pathfile.split('\n')
# except AttributeError:
# lines = pathfile
try:
print "started"
# for linie in open(ticker_file, "r"):
for linie in sys.stdin:
try:
print linie
ticker_arr = linie.split('\t')
curr_ticker = ticker_arr[1]
curr_cik = ticker_arr[2]
curr_date = ticker_arr[3]
if not '-' in curr_date:
curr_date = curr_date[0:4] + '-' + curr_date[4:6] + '-' + curr_date[6:8]
curr_date = curr_date.strip()
curr_date_obj = crteDateObj(curr_date)
yest_date_obj = curr_date_obj - timedelta(days=1)
yest_date = crteDateStr(yest_date_obj)
try:
price_dict = ystockquote.get_historical_prices(curr_ticker, yest_date, curr_date)
curr_close = price_dict[curr_date]['Close']
curr_adj_close = price_dict[curr_date]['Adj Close']
curr_open = price_dict[curr_date]['Open']
yest_close = price_dict[yest_date]['Close']
yest_adj_close = price_dict[yest_date]['Adj Close']
yest_open = price_dict[yest_date]['Close']
except:
curr_close = "NA"
curr_adj_close = "NA"
curr_open = "NA"
yest_close = "NA"
yest_adj_close = "NA"
yest_open = "NA"
try:
all_dict = ystockquote.get_all(curr_ticker)
curr_mkt_cap = all_dict['market_cap']
except:
curr_mkt_cap = "NA"
print curr_ticker + '\t' + curr_cik + '\t' + curr_date + '\t' + curr_open + '\t' + \
curr_close + '\t' + curr_adj_close + '\t' + yest_open + '\t' + yest_close + '\t' + \
yest_adj_close + '\t' + curr_mkt_cap
except:
print "bad"
except:
print "didn't start"
def get_historical_prices(self): data = ystockquote.get_historical_prices(self.symbol, self.start_date, self.end_date) data_headers = data[0] data.pop(0) self.data = data[::-1] self.data_headers = data_headers
def get_stock_year_price(symbol, year_begin, year_end):
'''
Get the prices of Stock {symbol} from year_begin (int) to year_end (int)
Returns a dictionary like:
{"Symbol": symbol, "Prices": {"2015-1-1":..., "2015-2-1":...}}
'''
d = datetime.datetime(year_begin, 1, 1)
end_date = datetime.datetime(year_end, 12, 31)
ret = {"Symbol": symbol, "Prices": {}}
# iterating over month
while d < end_date:
next_mon = d + relativedelta(months=1)
# fetch this month's all trade days' data
try:
m = ystockquote.get_historical_prices(
symbol, d.strftime("%Y-%m-%d"), next_mon.strftime("%Y-%m-%d"))
except urllib2.HTTPError:
d = next_mon
continue
for day, data in m.iteritems():
ret["Prices"][day] = data
d = next_mon
return ret
def yahoo(symbol, start_date=None, stop_date=None):
"""
Loads the prices from the start date for the given symbol
Only new quotes are downloaded.
"""
if not stop_date:
stop_date = date.today().strftime("%Y%m%d")
if not start_date:
query = db.Query(Quote)
query.order('-date')
query.filter('symbol = ', symbol)
latest_quote = query.get()
if latest_quote:
start_date = latest_quote.date
else:
start_date = date.today() - timedelta(days=120)
if start_date == date.today():
return
start_date = start_date.strftime("%Y%m%d")
prices = ystockquote.get_historical_prices(symbol, start_date, stop_date)
headers = prices[0]
try:
close = Quote.get_idx(headers, 'Close')
date_ = Quote.get_idx(headers, 'Date')
open = Quote.get_idx(headers, 'Open')
high = Quote.get_idx(headers, 'High')
low = Quote.get_idx(headers, 'Low')
except Exception, e:
logging.warning('Could not download %s:%s', symbol, e)
return None
def calcOutcome(self, call):
#print 'getting outcome'
try:
nextWeekRes = ystockquote.get_historical_prices(call['ticker'],call['callDate'].strftime("%Y%m%d"),call['weekACall'].strftime("%Y%m%d"))
except Exception,e:
nextWeekRes = None
print 'Exception in next week on ', call['ticker'], 'call date: ', call['callDate']
def main():
error_file = "no_quote.txt"
ticker_file = "tickers.txt"
quotes_file = "quotes.txt"
error_file = open(error_file, 'w')
ticker_file = open(ticker_file, 'r')
quotes_file = open(quotes_file, 'w')
for ticker in ticker_file:
try:
ticker_arr = ticker.split('\t')
curr_ticker = ticker_arr[0]
curr_cik = ticker_arr[1]
curr_date = ticker_arr[2]
if not '-' in curr_date:
curr_date = curr_date[0:4] + '-' + curr_date[4:6] + '-' + curr_date[6:8]
# crteDateObj(curr_date)
price_dict = ystockquote.get_historical_prices(curr_ticker, curr_date, curr_date)
if curr_date in price_dict and 'Close' in price_dict[curr_date] and 'Open' in price_dict[curr_date]:
curr_close = price_dict[curr_date]['Close']
curr_open = price_dict[curr_date]['Open']
quotes_file.write(curr_ticker + '\t' + curr_cik + '\t' + curr_date + '\t' + curr_open + '\t' + curr_close)
except:
error_file.write(curr_ticker + ', ' + curr_cik + ', ' + curr_date)
error_file.close()
ticker_file.close()
quotes_file.close()
def get_eod_data(symbol, start_date, end_date):
'''
Task used to obtain end of day data
'''
try:
price_data = ystockquote.get_historical_prices(symbol, start_date,
end_date)
logger.info('Obtained data for {0} from {1} to {2}'.format(
symbol, start_date, end_date))
return {'symbol': symbol, 'data': price_data, 'status': 'success'}
except urllib.error.HTTPError as exc:
logger.error({
'status': 'failed',
'symbol': symbol,
'msg': str(exc),
'exception': 'HTTPError'
})
return {'status': 'failed', 'symbol': symbol, 'msg': str(exc)}
except urllib.error.URLError as exc:
logger.error({
'status': 'failed',
'symbol': symbol,
'msg': str(exc),
'exception': 'URLError'
})
return {'status': 'failed', 'symbol': symbol, 'msg': str(exc)}
except Exception as exc:
logger.error({
'status': 'failed',
'symbol': symbol,
'msg': str(exc),
'exception': 'Last'
})
return {'status': 'failed', 'symbol': symbol, 'msg': str(exc)}
def yahooExtract(ticker, country):
# This function uses the built-in python library 'ystockquote', which inquires
# historical prices and return them as dictionaries. We then determine the
# average prices within each year, storing these averages into a csv file in a
# separate folder.
rawdata = ystockquote.get_historical_prices(ticker, '1995-01-01', '2011-12-31')
prices = {}
for date in rawdata.keys():
if date[0:4] in prices.keys():
prices[date[0:4]].append(float(rawdata[date]['Close']))
else:
prices[date[0:4]] = []
prices[date[0:4]].append(float(rawdata[date]['Close']))
data = []
for year in sorted(prices.keys()):
data.append([year, float(sum(prices[year]))/len(prices[year])])
with open('done/' + country + '.csv', 'w') as file1:
file1write = csv.writer(file1, delimiter = ',')
file1write.writerows(data)
file1.close()
def update_price():
# pull today
now = datetime.now()
today = ("%s-%02d-%02d" % (now.year, now.month, now.day))
quotes = ystockquote.get_historical_prices('SPY', '2015-01-15',
today) #pull data
#writing json
with open('www/live-data/price.json', 'w') as file:
json.dump(quotes, file)
#get current price
current_price = ystockquote.get_price('SPY')
current_date = now.strftime('%Y-%m-%d')
current_time = now.strftime('%H:%M:%S')
current_value = str(current_date) + "," + str(current_time) + "," + str(
current_price)
header = "date,time,price"
with open('www/live-data/current.csv', 'wb') as f:
fwriter = csv.writer(f, delimiter=" ")
fwriter.writerow([header])
fwriter.writerow([current_value])
if app_config.DEPLOYMENT_TARGET:
flat.deploy_folder('www/live-data',
'live-data',
max_age=app_config.DEFAULT_MAX_AGE)
def import_ofx(self, ofx_file):
ofx = OfxParser.parse(file(ofx_file))
idx = {}
for s in ofx.security_list:
idx[s.uniqueid] = s.ticker
c = self.db.cursor()
for t in ofx.account.statement.transactions:
c.execute("SELECT id FROM stocks WHERE id = ?", [t.id])
row = c.fetchone()
if row:
print "Skipping duplicate transaction:", t.id
continue
spydate = t.tradeDate
# Fidelity transactions can "close" on a weekend?!?
if spydate.weekday() == 5:
spydate = spydate - timedelta(days=1)
elif spydate.weekday() == 6:
spydate = spydate - timedelta(days=2)
spy = ystockquote.get_historical_prices('SPY',
spydate.strftime("%Y%m%d"), spydate.strftime("%Y%m%d"))
spy_price = float(spy[1][4])
spy_units = (float(t.units) * float(t.unit_price)) / spy_price
c.execute("INSERT INTO stocks VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
(t.id, idx[t.security], t.security, t.tradeDate, t.settleDate,
float(t.units), float(t.unit_price), spy_units, spy_price))
self.db.commit()
def prices(symbol):
"""
Loads the prices from the start date for the given symbol
Only new quotes are downloaded.
"""
to = date.today().strftime("%Y%m%d")
c = db.cursor()
c.execute("SELECT DATE_ADD(max(date), INTERVAL 1 DAY) FROM quote where symbol = %s",
(symbol))
(_from, ) = c.fetchone()
if _from == date.today():
print "Skipping %s" % symbol
return
print "Downloading %s" % symbol
if _from is None:
_from = start_date
else:
_from = _from.strftime("%Y%m%d")
prices = ystockquote.get_historical_prices(symbol, _from, to)
headers = prices[0]
try:
close = get_idx(headers, 'Close')
date_ = get_idx(headers, 'Date')
open = get_idx(headers, 'Open')
high = get_idx(headers, 'High')
low = get_idx(headers, 'Low')
quotes = prices[1:]
for l in quotes:
#print "%s %s" % (l[date_], l[close])
insert(symbol, l[date_], l[close], l[high], l[low], l[open])
print "Inserted %s new quotes for %s" % (len(quotes), symbol)
except:
print "Could not download %s" % symbol
def fetchQuotes(sym, start=FROM_DATE, end=CURRENT_DATE):
his = None
data = None
try:
# print start, end
data = ystockquote.get_historical_prices(sym, start, end)
except Exception:
print "Please check the dates. Data might not be available. 404 returned"
# 404 due to data yet not available
if data:
his = DataFrame(collections.OrderedDict(sorted(data.items()))).T
his = his.convert_objects(convert_numeric=True)
his.index = pd.to_datetime(his.index)
his.insert(0, 'symbol', sym, allow_duplicates=True)
# insert the date as dataframe too
his.insert(1, 'date', his.index)
# his.columns = getColumns('stock_quote_historical') # Removing as db dependency is removed
his.columns = getColumnsNoSql('stock_quote_historical')
daily = ystockquote.get_all(sym)
# print daily
# persist(his, daily, sym, end)
return his, daily
def main():
error_file = "no_quote.txt"
ticker_file = "tickers.txt"
quotes_file = "quotes.txt"
error_file = open(error_file, 'w')
ticker_file = open(ticker_file, 'r')
quotes_file = open(quotes_file, 'w')
for ticker in ticker_file:
try:
ticker_arr = ticker.split('\t')
curr_ticker = ticker_arr[0]
curr_cik = ticker_arr[1]
curr_date = ticker_arr[2]
if not '-' in curr_date:
curr_date = curr_date[0:4] + '-' + curr_date[
4:6] + '-' + curr_date[6:8]
# crteDateObj(curr_date)
price_dict = ystockquote.get_historical_prices(
curr_ticker, curr_date, curr_date)
if curr_date in price_dict and 'Close' in price_dict[
curr_date] and 'Open' in price_dict[curr_date]:
curr_close = price_dict[curr_date]['Close']
curr_open = price_dict[curr_date]['Open']
quotes_file.write(curr_ticker + '\t' + curr_cik + '\t' +
curr_date + '\t' + curr_open + '\t' +
curr_close)
except:
error_file.write(curr_ticker + ', ' + curr_cik + ', ' + curr_date)
error_file.close()
ticker_file.close()
quotes_file.close()
def volatlility_number(symbol, startdate, enddate):
pdata = ystockquote.get_historical_prices(symbol, startdate, enddate)
closing_price_list = []
list_of_logs = []
for p in pdata:
closing_price_list.append(p[4])
closing_price_list.remove('Close')
nice = [float(i) for i in closing_price_list]
#print "this is how nice", nice
for n in range(1, (len(pdata) - 1)):
day = float(pdata[n][4])
day_before = float(pdata[(n + 1)][4])
#print day
#print day_before
H = (day / day_before)
G = math.log(H)
list_of_logs.append(G)
#print G
#print ""
standard_deviation = stdv(list_of_logs)
volatility_number = (standard_deviation * math.sqrt(252))
return volatility_number
def downStock(self, symbol, startDate, endDate):
#if 600 append .ss 300 and 000 .sz
tmpSymbol = self.getYahooSymbol(symbol)
try:
prices = ystockquote.get_historical_prices(tmpSymbol, startDate,
endDate)
data = []
for date in prices.keys():
open = (float(prices[date]['Open']))
high = (float(prices[date]['High']))
low = (float(prices[date]['Low']))
close = (float(prices[date]['Close']))
volume = (float(prices[date]['Volume']))
adj_close = float(prices[date]['Adj Close'])
sql = 'insert into tbl_historical(symbol,open,high,low,close,volume,date) VALUES (%s,%s,%s,%s,%s,%s,%s)'
params = (symbol, open, high, low, close, volume, date)
data.append(params)
return data
except HTTPError as e:
logging.warning('yahoo 没有这只股票 %s', e)
if (404 == e.getcode()):
'标致yahoo没有改股票的数据'
self.stockDao.updateSource(symbol, 'wind')
except DataError as ex:
logging.warning(ex)
def storeStockHistory(conn, code, fromDate, toDate):
storeList = []
try:
data = ystockquote.get_historical_prices(code, fromDate, toDate)
except:
print ( code + " URL error")
return
if len(data) == 0:
print ( code + " data none")
return
#pprint(data)
for stockDate in data:
print (code + ":" + stockDate)
storeDic = data[stockDate]
storeDic["code"] = code
storeDic["s_date"] = stockDate
storeDic["adj_close"] = data[stockDate]["Adj Close"]
storeDic["s_close"] = data[stockDate]["Close"]
storeDic["s_open"] = data[stockDate]["Open"]
storeDic["s_high"] = data[stockDate]["High"]
storeDic["s_low"] = data[stockDate]["Low"]
storeDic["s_volume"] = data[stockDate]["Open"]
storeList.append(storeDic)
cur = conn.cursor()
cur.executemany("INSERT INTO stock.price(code, s_date, adj_close, s_close, s_open, s_high, s_low, s_volume) VALUES (%(code)s, %(s_date)s, %(adj_close)s, %(s_close)s, %(s_open)s, %(s_high)s, %(s_low)s, %(s_volume)s)", storeList)
conn.commit()
cur.close()
def get_high_prices(self, date): date = datetime.date(2015, 10, 21) day_before = date - timedelta(1) day_after = date + timedelta(1) price = ystockquote.get_historical_prices(self.symbol, str(day_before), str(day_after)) self.day_before_price = price[str(day_before)]['High'] self.day_after_price = price[str(day_after)]['High']
def stockmine(ticker, dayweek="d", splitsize=60): # splitsize <= 0 will not split, "d" for day, "w" for weeks
count = 0 # Keeps count of the number of days currently stored before writing
writeCount = 0 # how many file writes have been made
dayStat = []
monthStat = []
if ticker != None:
header = True
today = date.today()
stock = ystockquote.get_historical_prices(
str(ticker), "20000101", today.strftime("%Y%m%d"), dayweek
) # retrieve financial data from Yahoo
for data in stock: # go through stock data individually
if header == False:
dayStat.append([data[0], data[5]]) # appends date and volume
count += 1
if count >= splitsize and splitsize > 0: # when splitsize days have been reached
monthStat.append(
dayStat
) # list containing sublists, where each sublist contains the day, which contains sublist of date/vol
dayStat = None
dayStat = []
count = 0
writeCount += 1
else:
header = False # skips first line
if splitsize <= 0:
monthStat = dayStat
return monthStat
def import_historic_quotes(self, years=15):
result = []
today = datetime.date.today()
first_day = datetime.date.today() - datetime.timedelta(days=int(years * 365))
today_str = today.strftime('%Y-%m-%d')
first_day_str = first_day.strftime('%Y-%m-%d')
logger.info('Start import of historic quotes')
for sec in Security.objects.all():
logger.debug('Security ' + str(sec))
no_quote = False
no_yahoo_id = False
if sec.yahoo_id != '' and not sec.yahoo_id.startswith('unknown'):
try:
quote = ystockquote.get_historical_prices(sec.yahoo_id, first_day_str, today_str)
except urllib.error.HTTPError:
no_quote = True
else:
logger.debug('Found quotes')
for key in quote:
logger.debug('Security ' + str(sec))
self.add(sec, key, quote[key]['Close'])
else:
no_yahoo_id = True
result.append({'stock_id': sec.id,
'yahoo_id': sec.yahoo_id,
'name': sec.name,
'no_quote': no_quote,
'no_yahoo_id': no_yahoo_id})
return result
def getBench(f_tdelta,bench):
sDate = datetime.today().date() + timedelta(days=-f_tdelta)
sDate = sDate.strftime("%Y-%m-%d")
eDate = datetime.today().date()
eDate = eDate.strftime("%Y-%m-%d")
tdate = []
temp = []
print "Getting prices for " + str(bench)
temp.append(ystockquote.get_historical_prices(bench, sDate, eDate))
for y in temp[0]:
tdate.append(y)
tdate = sorted(tdate, reverse=True)
ptemp = [[0 for x in xrange(1)] for x in xrange(len(tdate))]
for datee in range(len(tdate)):
for assetnum in range(1):
try:
q = temp[assetnum][tdate[datee]]['Close']
ptemp[datee][assetnum] = q
except KeyError:
print "wiww"
ptemp[datee][assetnum] = ptemp[datee-1][assetnum]
return ptemp
def __init__(self, name, start_date):
self.name = name.upper()
try:
self.stockDate = dt.date(int(start_date[0:4]), int(start_date[5:7]), int(start_date[8:10]))
except ValueError:
print "Error: bad date"
except IndexError:
print "Error: dates must be iso format"
got_price = 0
attempts = 0
while not got_price and attempts < 5:
try:
self.price = float(
ystock.get_historical_prices(self.name, self.stockDate.isoformat(), self.stockDate.isoformat())[1][
4
]
)
except:
self.stockDate = self.stockDate + self.one_day
attempts += 1
else:
got_price = 1
if self.price == 0:
print "Unable to find stock name"
def ADX(self, stock, date1, date2):
data = ys.get_historical_prices(stock, date1, date2)
if date1 in data.keys() and date2 in data.keys():
upMove = float(data[date2]['High']) - float(data[date1]['High'])
downMove = float(data[date1]['Low']) - float(data[date2]['Low'])
if (upMove > downMove) and (upMove > 0):
PDM = upMove
else:
PDM = 0
if (downMove > upMove) and (downMove > 0):
NDM = downMove
else:
NDM = 0
hilo = float(data[date2]['High']) - float(data[date1]['High'])
hiclo = abs(float(data[date2]['High']) - float(data[date1]['Close']))
hipreclo = abs(float(data[date2]['Low']) - float(data[date1]['Close']))
TR = max([hilo, hiclo, hipreclo])
length = len(self.MMA[stock]['PDM']) + 1
if length == 1:
self.MMA[stock]['PDM'].append(PDM)
self.MMA[stock]['NDM'].append(NDM)
self.MMA[stock]['ATR'].append(TR)
elif length > 1 and length < 14:
self.MMA[stock]['PDM'].append(((length-1)*self.MMA[stock]['PDM'][-1] + PDM)/length)
self.MMA[stock]['NDM'].append(((length-1)*self.MMA[stock]['NDM'][-1] + NDM)/length)
self.MMA[stock]['ATR'].append(((length-1)*self.MMA[stock]['ATR'][-1] + TR)/length)
else:
self.MMA[stock]['PDM'].append(((length-1)*self.MMA[stock]['PDM'][-1] + PDM)/length)
self.MMA[stock]['NDM'].append(((length-1)*self.MMA[stock]['NDM'][-1] + NDM)/length)
self.MMA[stock]['ATR'].append(((length-1)*self.MMA[stock]['ATR'][-1] + TR)/length)
del self.MMA[stock]['PDM'][0]
del self.MMA[stock]['NDM'][0]
del self.MMA[stock]['ATR'][0]
if self.MMA[stock]['ATR'][-1] != 0:
self.MMA[stock]['PDI'].append(100*self.MMA[stock]['PDM'][-1]/self.MMA[stock]['ATR'][-1])
self.MMA[stock]['NDI'].append(100*self.MMA[stock]['NDM'][-1]/self.MMA[stock]['ATR'][-1])
else:
self.MMA[stock]['PDI'].append(100*self.MMA[stock]['PDM'][-1])
self.MMA[stock]['NDI'].append(100*self.MMA[stock]['NDM'][-1])
if self.MMA[stock]['PDI'][-1]+self.MMA[stock]['NDI'][-1] != 0:
diff = abs((self.MMA[stock]['PDI'][-1]-self.MMA[stock]['NDI'][-1])/(self.MMA[stock]['PDI'][-1]+self.MMA[stock]['NDI'][-1]))
else:
diff = abs((self.MMA[stock]['PDI'][-1]-self.MMA[stock]['NDI'][-1]))
if length == 1:
self.MMA[stock]['DIFF'].append(diff)
elif length > 1 and length < 14:
self.MMA[stock]['DIFF'].append(((length-1)*self.MMA[stock]['DIFF'][-1]+diff)/length)
else:
self.MMA[stock]['DIFF'].append(((length-1)*self.MMA[stock]['DIFF'][-1]+diff)/length)
del self.MMA[stock]['DIFF'][0]
self.MMA[stock]['ADX'].append(100*self.MMA[stock]['DIFF'][-1])
def get(self):
title = "Rotation Portfolio App"
for first_days_of_month_prices in tradedates.first_days_of_months:
x = ystockquote.get_historical_prices('GOOGl', first_days_of_month_prices, first_days_of_month_prices)
self.render("base.html", title=title, x=x)
def upload_historical_prices(company,backDate=backDate): ticker = company.ticker try: prices = ystockquote.get_historical_prices(ticker,str(backDate), str(yesterday)) #print prices except Exception, e: #print "Failed to get historical prices for %s" % company.name return
def get_data(self, comp, start_date, end_date):
feature_dict = {}
results = ystockquote.get_historical_prices(comp, start_date, end_date)
feature_dict[FEATURES] = results.pop(0)
for quotes in results:
key = quotes.pop(0)
feature_dict[key] = quotes
return feature_dict
def getdata(stock, start, end):
data = []
pre_data = ystock.get_historical_prices(stock, start, end)
for i in pre_data[1:]:
data.append(i[-1])
return data
def get_data(self, comp, start_date, end_date):
feature_dict = {}
results = ystockquote.get_historical_prices(comp ,start_date, end_date)
feature_dict[FEATURES] = results.pop(0)
for quotes in results:
key = quotes.pop(0)
feature_dict[key] = quotes
return feature_dict
def getStockHistory(self, start, end):
history = ystockquote.get_historical_prices(self.name, start, end)
dates = []
data = []
for i in sorted(history):
dates.append(dt.datetime.strptime(i, '%Y-%m-%d').date())
data.append(float(history[i]['Close'])) # returns price at close of day
return dates, data
def get_hist_data():
l = ystockquote.get_historical_prices(ric, '20150101', '20150916')
#print l
xd = [float(x[4]) for x in l[1:]]
#yd = [datetime.strptime(y[0], '%Y-%m-%d') for y in l[1:]]
yd = [y for y in range(len(l[1:]))]
xd.reverse()
return xd, yd
def get_prices(tickers, start_yyyymmdd, end_yyyymmdd):
symbols = dict()
for symbol in tickers:
all_prices = get_historical_prices(symbol, start_yyyymmdd, end_yyyymmdd)
price_dates = sorted(all_prices.keys(), key=lambda k: order_dates(k))
close_prices = [float(all_prices[d]["Adj Close"]) for d in price_dates]
symbols[symbol] = close_prices
return symbols
def get_latest_remote_date(symbol):
# assumes that the most recent remote date will be in the last week
try:
hist_data = ysq.get_historical_prices(symbol, last_week, today)
hist_data.pop(0)
return hist_data[0][0]
except:
return ""
def get_latest_remote_date(symbol):
# assumes that the most recent remote date will be in the last week
try:
hist_data = ysq.get_historical_prices(symbol, last_week, today)
hist_data.pop(0)
return hist_data[0][0]
except:
return ""
def getdata(stock, start, end):
data = []
pre_data = ystock.get_historical_prices(stock, start, end)
for i in pre_data[1:]:
data.append(i[-1])
return data
def historic_closing_prices():
''' pulls historic closing price data from the Yahoo finance API '''
ticker = request.args.get('ticker')
start = (datetime.datetime.now() + datetime.timedelta(days=-365)).strftime('%Y-%m-%d')
end = datetime.datetime.now().strftime('%Y-%m-%d')
prices = ystockquote.get_historical_prices(ticker, start, end)
table = closing_price_table(prices)
return callback_wrapper(json.dumps(table))
def getBestCrossover(self):
from pprint import pprint
#self.tickerSymbol = self.tickerTextBox.get()
if self.stockVar.get() == 4:
self.tickerSymbol = self.tickerTextBox.get()
print(self.pressed)
if self.pressed == 0: #Box not checked
self.startDate = self.date1TextBox.get()
self.endDate = self.date2TextBox.get()
#print(startDate, endDate)
#print((ystockquote.get_historical_prices(tickerSymbol, startDate, endDate).keys()))
#print(str(ystockquote.get_historical_prices(tickerSymbol, startDate, endDate)))
entireDictionaryOfData = ystockquote.get_historical_prices(
self.tickerSymbol, self.startDate, self.endDate)
#entireDictionaryOfData = ystockquote.get_historical_prices('CAS', '2013-11-01', '2013-11-05')
#pprint(entireDictionaryOfData.keys())
listOfDates = self.getListOfDatesFromHistoricalData(
entireDictionaryOfData)
#print(listOfDates)
listOfCloses = self.getListOfClosesFromHistoricalData(
entireDictionaryOfData, listOfDates)
#print(entireStringOfData.find("'Close'"))
#print(ystockquote.get_trade_date('GOOG'))
#theBestCrossover = PSO.runPSO(listOfCloses)
if self.algorithmVar.get() == 0:
theBestCrossover = HillClimbing.runHillClimbing(listOfCloses)
if self.algorithmVar.get() == 1:
theBestCrossover = NelderMeadNew.runNelderMead(listOfCloses)
if self.algorithmVar.get() == 2:
theBestCrossover = GeneticAlgorithms.runGA(listOfCloses)
if self.algorithmVar.get() == 3:
theBestCrossover = DifferentialEvolution.runDE(listOfCloses)
if self.algorithmVar.get() == 4:
theBestCrossover = PSO.runPSO(listOfCloses)
if self.algorithmVar.get() == 5:
theBestCrossover = SA.runSA(listOfCloses)
#theBestCrossover = PSO.figureOutBestConstants(listOfCloses)
#theBestCrossover = NelderMeadNew.runNelderMead(listOfCloses)
shortLength = theBestCrossover[0]
longLength = theBestCrossover[1]
if shortLength > longLength:
shortLength, longLength = longLength, shortLength
#return ('Short Length = '', Long Length = ').format(shortLength, longLength
#print(listOfDates)
self.bestCrossoverLabel['text'] = 'The Best Crossover is: '
self.answerLabel['text'] = 'Short Length = ' + str(
shortLength) + ', Long Length = ' + str(longLength)
self.makeGraph(listOfCloses, listOfDates, self.tickerSymbol,
shortLength, longLength)
def test_get_historical_prices_1mo(self):
symbol = 'GOOG'
start_date = '2013-01-02'
end_date = '2013-04-01'
prices = ystockquote.get_historical_prices(symbol, start_date,
end_date, '1mo')
self.assertIsInstance(prices, dict)
self.assertEqual(len(prices), 4)
def get_stock_returns(ticker):
""" Given args ticker=<symbol>, looks up the Yahoo!
historical data for the company. Calculate the daily
returns for it."""
return_object = {}
if ticker is None or ticker == '':
return_object['error'] = 'No symbol given.'
return return_object
# look at last quarter of data
now = datetime.datetime.now()
before = now - datetime.timedelta(days=90)
now_string = now.strftime('%Y-%m-%d')
before_string = before.strftime('%Y-%m-%d')
try:
# grab quotes from Yahoo! finance
historical = ystockquote.get_historical_prices(
ticker,
before_string,
now_string)
# use adjusted close for calculating
# daiy returns
daily_close = [
float(stats['Adj Close'])
for date, stats
in historical.items()]
if len(daily_close) < 3:
return_object['error'] = 'Too few data points.'
return return_object
returns = []
# TODO: Moving average?
# Or add a smoothening factor to discard the day-to-day
# noise in prices
for i in range(len(daily_close)-1):
now = daily_close[i]
tomorrow = daily_close[i+1]
returns.append((tomorrow-now)/now)
# want them sorted from latest to earliest
# because we later zip them up with similar lists
# and some stocks may not have earlier information
returns.reverse()
return_object['results'] = returns
return return_object
# Yahoo! doesn't respond, ticker not given/valid
except:
return_object['error'] = 'Not a valid ticker.'
return return_object
def getEvents(security):
events = []
data = ystockquote.get_historical_prices(security, '2011-06-01', '2013-01-01')
for date in data.keys():
timestamp = date
value = float(data[date]['Adj Close'])
event = MarketUpdate(security, timestamp, value)
events.append(event)
return events
def getStockHistoryClose(self, start, end):
history = ystockquote.get_historical_prices(self.symbol, start, end)
dates = []
data = []
for i in sorted(history):
dates.append(dt.datetime.strptime(i, '%Y-%m-%d').date())
data.append(float(history[i]['Close']))
# returns price at close of day
return dates, data
def GetData(name=False,update=False):
""" Pulls down data for quote from startDate to
endDate and removes unneeded data (as per Prof.
Evans's video 'Download SPY data'). Data is stored in
(quote).csv in the same directory as this file"""
#Input Checkers
dateChecker=re.compile("([1-2][0-9][0-9][0-9])-([0-1][0-9])-([0-3][0-9])")
data = False
while data == False:
startDate="a"
endDate="a"
#Inputs
if name:
quote = name
else:
quote = raw_input("Ticker: ")
while len(startDate)!=10 and dateChecker.search(startDate) == None:
startDate = raw_input("Start Date (yyyy-mm-dd): ")
while len(endDate)!=10 and dateChecker.search(endDate) == None:
endDate = raw_input("End Date (yyyy-mm-dd): ")
#Establish Connection and download data
try:
pricedict = ystockquote.get_historical_prices(quote,startDate,endDate)
data = True
except Exception:
print "Page not found. Check inputs and connection and try again."
#Obtain list of days and sort from earliest to latest
dayList=[]
for day in pricedict:
dayList.append(day.split('-'))
sortedDays = sorted(dayList)
#Populate rows of CSV file
csvData=[['Date','Volume','Adj Close']]
for n in range(len(sortedDays)):
sortedDays[n] = sortedDays[n][0]+'-'+sortedDays[n][1]+'-'+sortedDays[n][2]
csvData.append([sortedDays[n],pricedict[sortedDays[n]]['Volume'],pricedict[sortedDays[n]]['Adj Close']])
filename = quote + ".csv"
if update:
saveQ = raw_input("Type 'Yes' to save as " + filename + " or 'No' to choose your own file name\n Note: Typing Yes will overwrite the previous file!: ")
else:
saveQ = raw_input("Type 'Yes' to save as " + filename + " or 'No' to \nchoose your own file name: ")
if saveQ == 'No' or saveQ == 'no':
filename = raw_input("Please choose file name (must end in .csv): ")
with open(filename,'w') as outputfile:
csvwriter = csv.writer(outputfile,lineterminator='\n')
for n in range(len(csvData)):
csvwriter.writerow(csvData[n])
print "Data written to ", filename
print
def __init__(self, ticker, start_day=None, length=730):
self.ticker = ticker
if (start_day == None):
today = datetime.today()
start_day = today - timedelta(days=length)
start_str = str(start_day.strftime('%Y-%m-%d'))
end_str = str(today.strftime('%Y-%m-%d'))
else:
start_str = start_day
#start = time.strptime(start_day, "%Y-%m-%d")
start = datetime.strptime(start_str, "%Y-%m-%d")
end_day = start + timedelta(days=length)
end_str = str(end_day.strftime('%Y-%m-%d'))
i = y.get_historical_prices(ticker, start_str, end_str)
#Lag Pandas DataFrame
self.df = pd.DataFrame(i)
#Snu Dataframe
self.df = self.df.transpose()
#endre datatype til float
self.df = self.df.astype(float)
self.df = self.df.rename(
columns={
'Close': 'close',
'High': 'high',
'Open': 'open',
'Low': 'low',
'Volume': 'volume'
})
stoch = abstract.STOCH(self.df, 14, 1, 3)
macd = abstract.MACD(self.df)
atr = abstract.ATR(self.df)
obv = abstract.OBV(self.df)
rsi = abstract.RSI(self.df)
self.df['atr'] = pd.DataFrame(atr)
self.df['obv'] = pd.DataFrame(obv)
self.df['rsi'] = pd.DataFrame(rsi)
#kombinerer to dataframes
self.df = pd.merge(self.df,
pd.DataFrame(macd),
left_index=True,
right_index=True,
how='outer')
self.df = pd.merge(self.df,
stoch,
left_index=True,
right_index=True,
how='outer')
def get_price_history(ticker, start_date, end_date):
"""
example usage:
get_price_history('appl','2014-11-01','2014-11-30')
returns a pandas dataframe
"""
data = ystockquote.get_historical_prices(ticker, start_date, end_date)
df = pd.DataFrame(collections.OrderedDict(sorted(data.items()))).T
df = df.convert_objects(convert_numeric=True)
return df
def run(self):
while True:
spec = self.in_queue.get()
data = ys.get_historical_prices(spec['symbol'], spec['start'],
spec['end'])
spec.update({'data': data})
# Emit a new dictionary equivalent to spec but with a new key 'data'
# containing the price data.
self.out_queue.put(spec)
self.in_queue.task_done()
def stock_price(CompanyName):
stockrunning = True
while stockrunning == True:
try:
inp = CompanyName[1:]
startdate = str(input("Please input a start date for the price check in the form YYYY-MM-DD: "))
enddate = str(input("Please input an end date for the price check in the form YYYY-MM-DD: "))
a=ystockquote.get_historical_prices(inp, startdate, enddate) # prints out stock price between dates.
return(a)
stockrunning=False
except:
print("An exception has occured please try again.")
def get_stock(company):
start_date = request.args.get("startdate")
end_date = request.args.get("enddate")
stock_data = ystockquote.get_historical_prices(company, start_date, end_date)
stock_return_data = []
for date, score in stock_data.iteritems():
stock_return_data.append([get_timestamp(date), float(score["Close"])])
stock_return_data = sorted(stock_return_data)
return json.dumps(stock_return_data)
def get_weekly_data(self):
now = datetime.now()
prev = now.replace(day=now.day - 8)
week_hist = ystockquote.get_historical_prices(
'P', prev.strftime('%Y-%m-%d'), now.strftime('%Y-%m-%d'))
week_data = []
for date, data in week_hist.iteritems():
week_data.append((date, data['Close']))
return week_data
def test_get_historical_prices(self):
prices = ystockquote.get_historical_prices(self.symbol, '2013-01-02',
'2013-01-15')
headers = [
'Date', 'Open', 'High', 'Low', 'Close', 'Volume', 'Adj Close'
]
self.assertEqual(len(prices), 11)
self.assertEqual(prices[0], headers)
self.assertEqual(prices[1][0], '2013-01-15')
self.assertEqual(prices[-1][0], '2013-01-02')
self.assertGreater(float(prices[1][6]), 0.0)
self.assertGreater(float(prices[-1][6]), 0.0)
def main(argv):
ticker_file = "input/tickers.txt"
bucket_name = "edgarsentimentbucket"
conn = S3Connection(argv[1], argv[2])
path_bucket = conn.get_bucket(bucket_name)
k = Key(path_bucket)
k.key = ticker_file
pathfile = k.get_contents_as_string()
try:
lines = pathfile.split('\n')
except AttributeError:
lines = pathfile
for linie in lines:
try:
ticker_arr = linie.split('\t')
curr_ticker = ticker_arr[0]
curr_cik = ticker_arr[1]
curr_date = ticker_arr[2]
if not '-' in curr_date:
curr_date = curr_date[0:4] + '-' + curr_date[
4:6] + '-' + curr_date[6:8]
curr_date_obj = crteDateObj(curr_date)
yest_date_obj = curr_date_obj - timedelta(days=1)
yest_date = crteDateStr(yest_date_obj)
try:
price_dict = ystockquote.get_historical_prices(
curr_ticker, yest_date, curr_date)
curr_close = price_dict[curr_date]['Close']
curr_adj_close = price_dict[curr_date]['Adj Close']
curr_open = price_dict[curr_date]['Open']
yest_close = price_dict[yest_date]['Close']
yest_adj_close = price_dict[yest_date]['Adj Close']
yest_open = price_dict[yest_date]['Close']
except:
curr_close = "NA"
curr_adj_close = "NA"
curr_open = "NA"
yest_close = "NA"
yest_adj_close = "NA"
yest_open = "NA"
try:
all_dict = ystockquote.get_all(curr_ticker)
curr_mkt_cap = all_dict['market_cap']
except:
curr_mkt_cap = "NA"
print curr_ticker + '\t' + curr_cik + '\t' + curr_date + '\t' + curr_open + '\t' + \
curr_close + '\t' + curr_adj_close + '\t' + yest_open + '\t' + yest_close + '\t' + \
yest_adj_close + '\t' + curr_mkt_cap
except:
pass
def getyahoovals(ticker,
startdate=datetime.datetime(1980, 1, 1),
enddate=datetime.datetime(2040, 12, 31)):
strp = lambda x: x.strftime('%Y%m%d')
vals = ystockquote.get_historical_prices(ticker, strp(startdate),
strp(enddate))
dates = [
datetime.datetime.strptime(val[0], '%Y-%m-%d')
for val in vals[1:][::-1]
]
quotes = [float(val[-1]) for val in vals[1:][::-1]]
return streams.BasicStream(dates, scipy.array(quotes))