def fetch_data(self):
print("fetching")
stock = self.stock
start_date = datetime.strftime(self.date - timedelta(days=500), '%Y-%m-%d')
end_date = datetime.strftime(self.date, '%Y-%m-%d')
print("downloading stock data...")
try:
share_getter = Share(stock)
if stock[-1] == 'L': comparitor = Share('^FTSE')
else: comparitor = Share(share_getter.get_stock_exchange())
except:
print("network not available")
comparitor = None
df, dfb = pd.DataFrame(), pd.DataFrame()
try:
df = df.from_dict(share_getter.get_historical(start_date, end_date))
dfb = dfb.from_dict(comparitor.get_historical(start_date, end_date))
print("download complete: fetched", len(df.index) +
len(dfb.index), "records")
df = df.drop(['Symbol'], axis=1)
df['comparitor'] = dfb['Adj_Close']
df['Adj_Close'] = pd.to_numeric(df['Adj_Close'], errors='ignore')
df['Volume'] = pd.to_numeric(df['Volume'], errors='ignore')
df['comparitor'] = pd.to_numeric(df['comparitor'], errors='ignore')
return df
except Exception as e:
print("error in fetch_data", e)
return df
def process_stock_data(date_1, date_2, stock_name):
nasdaq = Share('^IXIC')
yahoo = Share(stock_name)
nasdaq.refresh()
yahoo.refresh()
index_data = nasdaq.get_historical(date_1, date_2)
data = yahoo.get_historical(date_1, date_2)
# 0 Volume
# 1 Symbol
# 2 Adj Close
# 3 High
# 4 Low
# 5 Date
# 6 Close
# 7 Open
close_prices = []
#Set up as two dimensional data
for day in data:
close_price = []
close_price.append(float(day.values()[7]))
close_prices.insert(0, close_price)
return close_prices
def _fetch_data(self, base_date, end_date):
"""
helper function which retrieves data from the yahoo finance api
"""
print("downloading stock data...")
try:
share_getter = Share(self.stock)
if self.stock[-1] == 'L': comparitor = Share('^FTSE')
else: comparitor = Share(share_getter.get_stock_exchange())
except:
print("network not available")
comparitor = None
df, dfb = pd.DataFrame(), pd.DataFrame()
try:
df = df.from_dict(share_getter.get_historical(base_date, end_date))
dfb = dfb.from_dict(comparitor.get_historical(base_date, end_date))
print("download complete: fetched",
len(df.index) + len(dfb.index), "records")
df = df.drop(['Symbol'], axis=1)
df['comparitor'] = dfb['Adj_Close']
df['Adj_Close'] = pd.to_numeric(df['Adj_Close'], errors='ignore')
df['comparitor'] = pd.to_numeric(df['comparitor'], errors='ignore')
df['comparitor'] = df['Adj_Close'] / df['comparitor']
df['roll_av'] = df['Adj_Close'].rolling(center=False,
window=7).mean()
return df
except Exception as e:
print("error in fetch_data", e)
return df
def get_data(symbol='^GSPC'):
symbol1 = symbol
if symbol1[0] == '^':
symbol1 = symbol[1:]
fileName = 'c:/data/%s.csv' % symbol1
keys = ['Date', 'Close'] # only save these fields
gspc = Share(symbol)
if os.path.isfile(fileName):
result = read_data(fileName)
oneday = timedelta(1)
yesterday = datetime.today().date() - oneday
latest = datetime.strptime(result[0]['Date'], '%Y-%m-%d').date()+oneday
if latest < yesterday:
print('Extending from %s to %s ...' % (str(latest), str(yesterday)))
result = gspc.get_historical(str(latest), str(yesterday)) + result
write_data(fileName, result, keys)
else:
print('No need to update')
else:
result = gspc.get_historical('1976-02-01', '2016-02-21')
write_data(fileName, result, keys)
return result
def getDifferencePercentage(self):
closing = []
shareName = Share(self.company)
self.getTimeStamp()
startDate=''
endDate=''
hist=''
for t in self.times:
todayTimeStamp = t
yesterdayTimeStamp = t-86400
startDate = str(datetime.datetime.fromtimestamp(todayTimeStamp).strftime('%Y-%m-%d'))
yesterdayDate=str(datetime.datetime.fromtimestamp(yesterdayTimeStamp).strftime('%Y-%m-%d'))
todayHist = shareName.get_historical(startDate, startDate)
yesterdayHist = shareName.get_historical(yesterdayDate,yesterdayDate)
while(len(todayHist)==0):
todayTimeStamp = todayTimeStamp+86400
startDate = str(datetime.datetime.fromtimestamp(todayTimeStamp).strftime('%Y-%m-%d'))
todayHist = shareName.get_historical(startDate, startDate)
while(len(yesterdayHist)==0):
yesterdayTimeStamp= yesterdayTimeStamp-86400
yesterdayDate=str(datetime.datetime.fromtimestamp(yesterdayTimeStamp).strftime('%Y-%m-%d'))
yesterdayHist = shareName.get_historical(yesterdayDate,yesterdayDate)
closingPriceToday = float(todayHist[0]['Close'])
closingPriceYesterday = float(yesterdayHist[0]['Close'])
difference = (float(closingPriceYesterday) - float(closingPriceToday))*100.0/float(closingPriceYesterday)
diff2 = float(format(difference, '.3f'))
closing.append(diff2)
self.differencePercentage = closing
return closing
def compare_stock(stock_1, stock_2, start_date, end_date):
share_1 = Share( stock_1 )
hist_1 = share_1.get_historical( start_date, end_date )
share_2 = Share( stock_2 )
hist_2 = share_2.get_historical( start_date, end_date )
total_date = 0
correct_prediction = 0
len_1 = len( hist_1 )
len_2 = len( hist_2 )
length = min(len_1, len_2)
i = 0
j = 0
# for i in range (0, length):
while ( i < len_1 and j < len_2 ):
x = hist_1[i]
y = hist_2 [i];
open = float( x['Open'])
close = float( x['Close'])
x_change = (close - open) * 100 / open
open = float( y['Open'] )
close = float( y['Close'] )
y_change = (close - open) * 100 / open
if x['Date'] != y['Date']:
print "Error! DATE doesn't match"
i += 1
j += 1
continue
total_date += 1
if y_change * x_change > 0:
# print "Wow, They have Same performance"
correct_prediction += 1
else:
print x['Date'] + ": " + stock_1 + " "+ str( x_change ) + " vs. " + stock_2 + " " + str( y_change )
# print "Dam, they differs!"
i += 1
j += 1
print "Total date is: " + str(total_date) + ", and correct prediction has " + str(correct_prediction )
if total_date > 0:
print "Correction rate is: " + str( correct_prediction * 1.0 / total_date )
else:
print "ERROR: Nothing compared!"
def refresh_data(self, s_date=None, e_date=None):
if s_date and e_date:
try:
share=Share(self.name)
self.begin=s_date
self.end=e_date
share.get_historical(s_date, e_date)
self.log.refresh("{} data collected".format(self.name))
except:
self.log.error("platform is offline or not connecting")
def readLast5DaysStockPrice():
with open('stock_features.csv', 'w') as csvfile:
fieldnames = ['Stock', 'Adj_Close_1','High_1', 'Low_1'
,'Adj_Close_2','High_2', 'Low_2','VolChg2_1','IndexChg2_1'
,'Adj_Close_3','High_3', 'Low_3','VolChg3_2','IndexChg3_2'
,'Adj_Close_4','High_4', 'Low_4','VolChg4_3','IndexChg4_3'
,'Adj_Close_5','High_5', 'Low_5','VolChg5_4','IndexChg5_4']
writer = csv.DictWriter(csvfile, lineterminator='\n', fieldnames=fieldnames)
writer.writeheader()
dow_code = 'BCB/UDJIAD1'
dji_index = []
j = 0
while (j < 10):
ticker = Share(stock_dict.get(j))
i = 0
stock_price = []
current_date = date.today()
fmt = '%Y-%m-%d'
while( i < 5):
if (ticker.get_historical(current_date.strftime(fmt), current_date.strftime(fmt)) == []):
current_date = current_date - timedelta(days=1)
else:
stock_price += ticker.get_historical(current_date.strftime(fmt), current_date.strftime(fmt))
if(j == 0):
if(i == 0 and Quandl.get(dow_code, trim_start=current_date, trim_end=current_date, authtoken="T246AaoCUiwSyz1C4Vfe").values.tolist() == []):
dji_index.append(get_latest_dji())
else:
dji_index.append(Quandl.get(dow_code, trim_start=current_date, trim_end=current_date, authtoken="T246AaoCUiwSyz1C4Vfe").values.tolist()[0][0])
current_date = current_date - timedelta(days=1)
i = i + 1
AbVolChg2_1 = int(stock_price[1].get('Volume')) - int(stock_price[0].get('Volume'))
VolChg2_1 = log(AbVolChg2_1,2) if AbVolChg2_1 > 0 else -1*log(fabs(AbVolChg2_1),2)
AbVolChg3_2 = int(stock_price[2].get('Volume')) - int(stock_price[1].get('Volume'))
VolChg3_2 = log(AbVolChg3_2,2) if AbVolChg3_2 > 0 else -1*log(fabs(AbVolChg3_2),2)
AbVolChg4_3 = int(stock_price[3].get('Volume')) - int(stock_price[2].get('Volume'))
VolChg4_3 = log(AbVolChg4_3,2) if AbVolChg4_3 > 0 else -1*log(fabs(AbVolChg4_3),2)
AbVolChg5_4 = int(stock_price[4].get('Volume')) - int(stock_price[3].get('Volume'))
VolChg5_4 = log(AbVolChg5_4,2) if AbVolChg5_4 > 0 else -1*log(fabs(AbVolChg5_4),2)
writer.writerow({'Stock': stock_dict.get(j)
,'Adj_Close_1' : stock_price[0].get('Adj_Close'),'High_1' : stock_price[0].get('High'),'Low_1' : stock_price[0].get('Low')
,'Adj_Close_2' : stock_price[1].get('Adj_Close'),'High_2' : stock_price[1].get('High'),'Low_2' : stock_price[1].get('Low')
,'VolChg2_1': VolChg2_1,'IndexChg2_1': (float(dji_index[1]) - float(dji_index[0]))
,'Adj_Close_3' : stock_price[2].get('Adj_Close'),'High_3' : stock_price[2].get('High'),'Low_3' : stock_price[2].get('Low')
,'VolChg3_2': VolChg3_2,'IndexChg3_2': (dji_index[2] - dji_index[1])
,'Adj_Close_4' : stock_price[3].get('Adj_Close'),'High_4' : stock_price[3].get('High'),'Low_4' : stock_price[3].get('Low')
,'VolChg4_3': VolChg4_3,'IndexChg4_3': (dji_index[3] - dji_index[2])
,'Adj_Close_5' : stock_price[4].get('Adj_Close'),'High_5' : stock_price[4].get('High'),'Low_5' : stock_price[4].get('Low')
,'VolChg5_4': VolChg5_4,'IndexChg5_4': (dji_index[4] - dji_index[3])
})
j = j+1
def getCurrPrice(tick,date): stock=Share(tick) try: stock.get_historical(date,date) except: return None stock_detail=stock.get_historical(date,date) #print stock_detail if len(stock_detail) is 0: return None else: return float(stock_detail[0]['Adj_Close'])
def prepare_stock_graph(symbol, start, end):
stock = Share(symbol)
stringprices = list(pd.DataFrame(stock.get_historical(start, end))["Adj_Close"])
stringdates = list(pd.DataFrame(stock.get_historical(start, end))["Date"])
prices = [float(p) for p in stringprices]
dates = []
for d in stringdates:
year, month, day = d.split("-")
d = datetime.date(int(year), int(month), int(day))
dates.append(d)
return prices, dates
def getData(startDate, endDate, period):
tickerSymbols = []
sp500 = finsymbols.get_sp500_symbols()
nasdaq = finsymbols.get_nasdaq_symbols()
amex = finsymbols.get_amex_symbols()
nyse = finsymbols.get_nyse_symbols()
for companies in amex:
tickerSymbols.append(companies.get("symbol"))
print tickerSymbols
return
# start a period before the startdate to obtain the data
print startDate, " ", endDate
startDate = startDate - timedelta(days=period)
print startDate
data = {}
for company in tickerSymbols:
print company
c = Share(company)
histData = c.get_historical(str(startDate), str(endDate))
histData = histData[::-1] # reverse data so its easier to use
data[company] = histData
addTools(data, period)
with open("Data/Amex_data.json", "w") as fp:
json.dump(data, fp, indent=2)
return data
def get_stock_price_and_volume(symbol, start_date, end_date, dates): share = Share(symbol) hist_data = share.get_historical(start_date, end_date) hist_data.reverse() volume = [] price = [] i = 0 for d in dates: if i < len(hist_data): if (hist_data[i]['Date'] == d): # Weekday price.append(hist_data[i]['Close']) volume.append(hist_data[i]['Volume']) i += 1 else: # Weekend price.append(0) volume.append(0) else: # Get the current price and volume instead from historical data price.append(share.get_price()) volume.append(share.get_volume()) if len(dates) != len(volume) and len(dates) != len(price): print 'Dates and volume and/or price lists are not of same lenght!' return [price, volume]
def main():
""" Main example """
logging.info("--- SETTING UP IDENTIFIERS ---")
asset_manager_id = random.randint(1, 2**31 - 1)
calendar = Calendar()
# This can fail if yesterday was a holiday - need to add a holiday calendar
business_date = calendar.addbusdays(date.today(), -2)
logging.info("Business Date: %s", business_date)
business_date_str = business_date.isoformat()
symbols = ['TWTR', 'AAPL', 'RBS.L', 'Z77.SI', '0008.HK']
logging.info("--- PULL MARKET DATA FROM YAHOO FINANCE ---")
eod_prices = []
for symbol in symbols:
share = Share(symbol=symbol)
logging.info("Stock Name: %s", share.get_name())
close = (share.get_historical(
start_date=business_date_str,
end_date=business_date_str)[0].get('Close'))
eod_price = EODPrice(asset_manager_id=asset_manager_id,
asset_id=symbol,
business_date=business_date,
price=Decimal(close))
logging.info("EOD Price: %s", eod_price.price)
eod_prices.append(eod_price)
logging.info("--- PERSIST PRICES TO AMAAS ---")
# Some of these attributes can be derived from the eod_prices - cleanup
market_data_interface.persist_eod_prices(asset_manager_id=asset_manager_id,
business_date=business_date,
eod_prices=eod_prices,
update_existing_prices=True)
def visualize_stock(symbol, start, end):
stock = Share(symbol)
stockdata = stock.get_historical(str(start), str(end + timedelta(days=7)))
closing_prices = {}
for item in stockdata:
closing_prices[item['Date']] = float(item['Close'])
for x in range((end - start).days):
curr = start + timedelta(days=x)
if str(curr) not in closing_prices.keys():
replacement = curr + timedelta(days=1)
while str(replacement) not in closing_prices.keys(
): # loops until a replacement price is found
replacement = replacement + timedelta(days=1)
closing_prices[str(curr)] = closing_prices[str(replacement)]
stocklist = []
prev = closing_prices[str(start)]
for x in range((end - start).days):
curr = start + timedelta(days=x)
curr_price = closing_prices[str(curr)]
change = ((curr_price - prev) / prev) * 1000
prev = closing_prices[str(curr)]
stocklist.append(change)
pyplot.plot(stocklist, 'g')
def getStockData(theTicker):
global startDate
print("Getting Data for ... " + theTicker)
stock = Share(theTicker)
print(startDate)
data = stock.get_historical(startDate, DateNow)
for d in data:
tmp = []
volume = int(d['Volume'])
adjclose = float(d['Adj_Close'])
high = float(d['High'])
low = float(d['Low'])
close = float(d['Close'])
date = d['Date']
open = float(d['Open'])
# newDate = datetime.strptime(date, "%Y-%m-%d")
tmp.append(date)
tmp.append(open)
tmp.append(high)
tmp.append(low)
tmp.append(close)
tmp.append(adjclose)
tmp.append(volume)
givenStock.append(tmp)
return givenStock
def getReturns(stocks='MSFT,AAPL,NFLX,JPM,UVXY,RSX,TBT',
period_days=100,
end='2016-12-09'):
stocks = stocks.split(",")
index = 'SPY'
stocks.append(index)
if end is None:
end = datetime.today().strftime("%Y-%m-%d")
start = (datetime.today() - timedelta(period_days)).strftime("%Y-%m-%d")
i = 0
w = pd.DataFrame()
t = []
for s in stocks:
z = Share(s)
px = pd.DataFrame(z.get_historical(start, end))[['Close', 'Date']]
px['Close'] = px['Close'].astype(float)
px.index = px['Date']
del px['Date']
px.columns = [s]
t.append(px)
w = pd.concat(t, axis=1, join='inner')
w = w.sort_index().pct_change(
) #returns => w.cov() covariance matrix of returns
#calculate betas
betas = []
for s in stocks:
if s != index:
col = np.column_stack((w[s], w[index]))
b = np.cov(col) / np.var(w[index])
betas.append(b)
stocks.remove(index)
del w[index]
returns = w
return returns, stocks, np.round(betas, 4)
def getStock(id):
stock = Share(str(id)+'.TW')
today = datetime.date.today()
data = stock.get_historical('2017-04-20', str(today))
# data = stock.get_historical('2017-04-20', '2017-04-20')
# data = stock.get_historical(str(today), str(today))
return data
def calculateValue(self, date, prices):
value = 0.
date = date.strftime('%Y-%m-%d')
for sym,qty in self.holdings.items():
if sym == 'cash':
value += qty
else:
if sym not in prices:
stock = Share(sym)
# get prices until today
string_today = datetime.date.today().strftime('%Y-%m-%d')
print(string_today)
price_list = stock.get_historical(date, string_today)
# pprint.PrettyPrinter(depth = 6).pprint(price_list)
# Prices is a dictionary of dictionaries; each inner dictionary consists of (date, value) pairs for a ticker
prices[sym] = {item['Date']: float(item['Close']) for item in price_list}
# find price for the date of interest
if date in prices[sym]:
close_price = prices[sym][date]
value += close_price * qty
else:
print(date, sym)
return (None, None)
return (value, prices)
def get_stock_data(stocks_param, initial_date, end_date):
"""
This function should be called in an async way.
Since this is just a prove of concept is fine,
But this does 1 blocking http request per stock
We send as a parameter. Pretty bad leaving it syncronous.
"""
temp = []
add_header = False
response = []
if "," in stocks_param:
stocks = stocks_param.split(",")
else:
stocks = [stocks_param]
header = ["Date"]
for stock in stocks:
temp_stock = Share(stock)
temp.append(reversed(temp_stock.get_historical(initial_date,
end_date)))
header.append(str(stock))
for item in zip(*temp):
if add_header != True:
response.append(header)
add_header = True
date = item[0]["Date"]
prices = []
for resp in item:
prices.append(float(resp["Adj_Close"]))
response.append([date] + prices)
return response
def main():
start_date = '2007-01-01'
end_date = '2015-08-31'
plotly_name = 'username'
plotly_api_key = 'api-key'
# get_historical is part of the yahoo-finance module
nflx = Share('nflx')
nflx_prices = nflx.get_historical(start_date, end_date)
# how you can just extract the dates only
nflx_dates = [x['Date'] for x in nflx_prices]
# nflx_prices is currently sorted by dates because thats how the module gave them
sorted_by_price = sorted(nflx_prices, key=lambda x: x['Adj_Close'], reverse=True)
# say you wanted multiple stock prices
ticker_symbols = ['hrl', 'tsn', 'gis', 'k']
foods = {}
if do_plot:
plot_stock(plotly_name, plotly_api_key, nflx_prices, 'nflx')
for symbol in ticker_symbols:
foods[symbol] = Share(symbol).get_historical(start_date, end_date)
foods[symbol].sort(key=lambda x: x['Date'])
dates = [x['Date'] for x in foods[symbol]]
prices = [x['Adj_Close'] for x in foods[symbol]]
if do_plot:
add_trace(dates, prices, symbol)
def fetch_data(dt_from, dt_to, sym):
min_dt_from = dt.datetime(2014, 1, 1)
assert(dt_from >= min_dt_from)
r = TStockHistory.find_one(
{'Symbol': sym}, projection={'Date': 1}, sort=[('Date', pymongo.DESCENDING)])
if not r:
fetch_dt_from = min_dt_from
elif r['Date'] < dt_to:
fetch_dt_from = r['Date'] + dt.timedelta(days=1)
if fetch_dt_from > dt_to:
fetch_dt_from = None
else:
fetch_dt_from = None
if fetch_dt_from:
f = '%d-%d-%d' % (
fetch_dt_from.year, fetch_dt_from.month, fetch_dt_from.day)
t = '%d-%d-%d' % (dt_to.year, dt_to.month, dt_to.day)
print('fetch %s from network...' % (sym))
share = Share(sym)
docs = share.get_historical(f, t)
if docs:
for r in docs:
for k in ['Adj_Close', 'High', 'Low', 'Close', 'Volume']:
r[k] = np.float(r[k])
r['Date'] = dateutil.parser.parse(r['Date'])
TStockHistory.insert_many(docs)
data = TStockHistory.find(
{'Symbol': sym, 'Date': {'$gte': dt_from, '$lte': dt_to}})
rs = filter(lambda x: 'placeholder' not in x, [u for u in data])
return rs
def generateData(self):
global StartingDate
global DataSize
i = datetime.datetime.now()
EndingDate = '%s-%s-%s' % (i.year,i.month,i.day)
stock = Share(Stock_name)
data = stock.get_historical(StartingDate,EndingDate)
file = open('stock_data', 'w')
closes = [c['Close'] for c in data]
opens = [o['Open'] for o in data]
oArray = []
cArray = []
for c in closes:
cArray.append(c)
for o in opens:
oArray.append(o)
for x in range(len(data)-2):
# %Difference, Next Day %Difference, Money Made Holding for a Day
file.write(str((float(cArray[x])-float(oArray[x+1]))/100) + ' ' + str((float(cArray[x+1]) - float(oArray[x+2]))/100) + ' ' + str((float(oArray[x]) - float(oArray[x+1]))) + '\n')
self.dayChange.append((float(cArray[x])-float(oArray[x+1]))/100)
self.nextDayChange.append((float(cArray[x+1]) - float(oArray[x+2]))/100)
self.profit.append(float(oArray[x]) - float(oArray[x+1]))
#Makes sure the population size is
DataSize = len(self.dayChange)
file.close()
def calculateValue(self, date):
value = 0.
date = date.strftime('%Y-%m-%d')
for sym, qty in self.holdings.items():
if sym == 'cash':
value += qty
else:
if sym not in Portfolio.prices:
stock = Share(sym)
# get prices until today
string_today = datetime.date.today().strftime('%Y-%m-%d')
# print(string_today, date, sym)
price_list = stock.get_historical(date, string_today)
# pprint.PrettyPrinter(depth = 6).pprint(price_list)
Portfolio.prices[sym] = {
item['Date']: float(item['Close'])
for item in price_list
}
# find price for the date of interest
if date in Portfolio.prices[sym]:
close_price = Portfolio.prices[sym][date]
value += close_price * qty
else:
# I still don't know why this would ever happen. What's going on?
print(date, sym)
return None
return value
def get_data(stock,start,end):
data = Share(stock)
try:
data = pd.DataFrame(data.get_historical(start_date=start,end_date=end))
except Exception as e:
f = open('log.txt',mode='a')
f.write(stock+'\n')
f.write(str(e)+'\n')
return pd.DataFrame()
try:
data.index = data.Date
except Exception as e:
f = open('log.txt', mode='a')
f.write(stock+'\n')
f.write(str(e)+'\n')
return pd.DataFrame()
data = data.drop(['Date','Symbol'],axis=1)
data = data.sort_index()
for i in data.columns:
data[i] = data[i].astype(np.float)
#data['Adj_Open'] = 0
#data['Adj_High'] = 0
#data['Adj_Low'] = 0
#for i in range(len(data)):
# k = data['Adj_Close'][i] / data['Close'][i]
# data.loc[i:i+1,'Adj_Open'] = k*data['Open'][i]
# data.loc[i:i + 1, 'Adj_High'] = k * data['High'][i]
# data.loc[i:i + 1, 'Adj_Low'] = k * data['Low'][i]
data['Symbol'] = stock
return data
def get_market_data():
'''
Downloads returns data from Yahoo finance
'''
tickers = cc.get_ticker_list()
ticker_prices = {}
hist_prices = pd.DataFrame()
for ticker in tickers:
try:
ticker_info = Share(ticker)
ticker_data = ticker_info.get_historical('2006-01-01', '2016-11-20')
for day in ticker_data:
ticker_prices[day['Date']] = day['Adj_Close']
df = pd.DataFrame.from_dict(ticker_prices, orient = 'index')
df.columns = [ticker]
if hist_prices.empty:
hist_prices = df
else:
hist_prices = hist_prices.merge(df, how = 'outer', left_index = True, right_index = True)
except:
pass
hist_prices.index = hist_prices.index.to_datetime()
hist_prices = hist_prices.sort()
hist_prices = hist_prices.convert_objects(convert_numeric = True)
hist_returns = hist_prices.pct_change(1)
hist_returns.to_csv('data/returns_data.csv')
return
def getStockData(theTicker, startDate):
stock = Share(theTicker)
print("Getting Data for ... " + theTicker)
now = datetime.now()
DateNow = str(now.year) + "-" + str(now.month) + "-" + str(now.day)
data = stock.get_historical(startDate, DateNow)
stockData = []
for d in data:
tmp = []
volume = int(d['Volume'])
adjclose = float(d['Adj_Close'])
high = float(d['High'])
low = float(d['Low'])
close = float(d['Close'])
date = d['Date']
open = float(d['Open'])
tmp.append(date)
tmp.append(open)
tmp.append(high)
tmp.append(low)
tmp.append(close)
tmp.append(adjclose)
tmp.append(volume)
stockData.append(tmp)
return stockData
def getStockData(theTicker):
global startDate
print("Getting Data for ... " + theTicker)
stock = Share(theTicker)
print(startDate)
data = stock.get_historical(startDate, DateNow)
for d in data:
tmp = []
volume = int(d['Volume'])
adjclose = float(d['Adj_Close'])
high = float(d['High'])
low = float(d['Low'])
close = float(d['Close'])
date = d['Date']
open = float(d['Open'])
# newDate = datetime.strptime(date, "%Y-%m-%d")
tmp.append(date)
tmp.append(open)
tmp.append(high)
tmp.append(low)
tmp.append(close)
tmp.append(adjclose)
tmp.append(volume)
givenStock.append(tmp)
return givenStock
def stock_Prices():
# exit if the output already existed
# if os.path.isfile('./input/stockPrices.json'):
# sys.exit("Prices data already existed!")
priceSet = {}
fin = open('./input/tickerList.csv')
for num, line in enumerate(fin):
line = line.strip().split(',')
ticker, name, exchange, MarketCap = line
if 1:
print(num, ticker)
yahoo = Share(ticker)
time.sleep(np.random.poisson(3))
prices = yahoo.get_historical('2005-01-01', '2020-01-01')
priceDt = {}
for i in range(len(prices)):
date = ''.join(prices[i]['Date'].split('-'))
priceDt[date] = round(log(float(prices[i]['Close']) / float(prices[i]['Open'])), 6)
priceSet[ticker] = priceDt
#except:
#continue
with open('./input/stockPrices.json', 'w') as outfile:
json.dump(priceSet, outfile, indent=4)
def trailing_stop(ticker, start, end, percentage=0.15):
ticker = Share(ticker)
while True:
try:
historical = sorted(ticker.get_historical(start, end), key=itemgetter("Date"))
except ValueError as e:
raise e
except YQLQueryError as e:
continue
break
if len(historical) <= 1:
return None
initial_price = float(historical[0]["Adj_Close"])
curr_price = initial_price
stop_price = initial_price * (1 - percentage)
historical = historical[1:]
while len(historical) > 0 and stop_price < curr_price:
curr_price = float(historical[0]["Adj_Close"])
if curr_price > stop_price / (1 - percentage):
stop_price = curr_price * (1 - percentage)
historical = historical[1:]
out = (curr_price - initial_price) / initial_price
return out
def generateData(self):
global StartingDate
global DataSize
i = datetime.datetime.now()
EndingDate = '%s-%s-%s' % (i.year, i.month, i.day)
stock = Share(Stock_name)
data = stock.get_historical(StartingDate, EndingDate)
file = open('stock_data', 'w')
closes = [c['Close'] for c in data]
opens = [o['Open'] for o in data]
oArray = []
cArray = []
for c in closes:
cArray.append(c)
for o in opens:
oArray.append(o)
for x in range(len(data) - 2):
# %Difference, Next Day %Difference, Money Made Holding for a Day
file.write(
str((float(cArray[x]) - float(oArray[x + 1])) / 100) + ' ' +
str((float(cArray[x + 1]) - float(oArray[x + 2])) / 100) +
' ' + str((float(oArray[x]) - float(oArray[x + 1]))) + '\n')
self.dayChange.append(
(float(cArray[x]) - float(oArray[x + 1])) / 100)
self.nextDayChange.append(
(float(cArray[x + 1]) - float(oArray[x + 2])) / 100)
self.profit.append(float(oArray[x]) - float(oArray[x + 1]))
#Makes sure the population size is
DataSize = len(self.dayChange)
file.close()
def __init__(self, start_date, end_date, ticker):
'''clean data and store class attributes'''
stock = Share(ticker)
df = pd.DataFrame(stock.get_historical(
start_date, end_date))[['Date', 'Adj_Close']]
df['Adj_Close'] = df['Adj_Close'].astype(float)
df['Date'] = pd.to_datetime(df['Date'])
df = df.sort_index(by='Date', ascending=True)
df.index = range(len(df))
df.columns = ['Date', 'Price']
self.df = df
self.ts = df.set_index('Date')
self.date = df['Date']
self.val = df['Price']
self.start = self.ts.index[0]
self.end = self.ts.index[-1]
# following Series are used for finding the optimal parameter d by comparing the acf's and
# pacf's for the timeseries at different diff stage.
self.diff1_val = pd.Series(np.diff(self.val))
self.diff1_val_na = pd.Series(
np.concatenate(([np.nan], self.diff1_val.values)))
self.diff2_val = pd.Series(np.diff(self.diff1_val_na))
self.diff2_val_na = pd.Series(
np.concatenate(([np.nan], self.diff2_val.values)))
def prepare_stock_graph(symbol, start, end):
stock = Share(symbol)
stringprices = list(
pd.DataFrame(stock.get_historical(start, end))['Adj_Close'])
stringdates = list(pd.DataFrame(stock.get_historical(start, end))['Date'])
prices = [float(p) for p in stringprices]
dates = []
for d in stringdates:
year, month, day = d.split('-')
d = datetime.date(int(year), int(month), int(day))
dates.append(d)
return prices, dates
def get_historical(self, num_of_days):
if self._id.startswith('6'):
stk_id = self._id + '.ss'
else:
stk_id = self._id + '.sz'
stk = Share(stk_id)
start = datetime.strftime(datetime.today() - timedelta(days=num_of_days+1),
'%Y-%m-%d')
end = datetime.strftime(datetime.today() - timedelta(days=1),
'%Y-%m-%d')
hist = stk.get_historical(start, end)
high = []
low = []
opening = []
close = []
vol = []
date = []
for d in hist:
if d.has_key('High'):
high.append(float(d['High']))
low.append(float(d['Low']))
opening.append(float(d['Open']))
close.append(float(d['Close']))
vol.append(float(d['Volume']))
date.append(d['Date'])
return high, low, opening, close, vol, date
def download(symbol, start_date, end_date):
stock = Share(symbol)
# ^GSPC is the Yahoo finance symbol to refer S&P 500 index
# we gather historical quotes from 2001-01-01 up to today
hist_quotes = stock.get_historical(start_date, end_date)
l_date = []
l_open = []
l_high = []
l_low = []
l_close = []
l_volume = []
# reverse the list
hist_quotes.reverse()
for quotes in hist_quotes:
l_date.append(quotes['Date'])
l_open.append(float(quotes['Open']))
l_high.append(float(quotes['High']))
l_low.append(float(quotes['Low']))
l_close.append(float(quotes['Adj_Close']))
l_volume.append(int(quotes['Volume']))
sf = SFrame({
'datetime': l_date,
'open': l_open,
'high': l_high,
'low': l_low,
'close': l_close,
'volume': l_volume
})
# datetime is a string, so convert into datetime object
sf['datetime'] = sf['datetime'].apply(
lambda x: datetime.strptime(x, '%Y-%m-%d'))
return sf
def retrieveQuoteFromYahoo(symbol,start,end):
share = Share(symbol)
quoteList = share.get_historical(start,end)
quoteDict = {}
for quote in quoteList:
quoteDict[quote['Date']] = float(quote['Adj_Close'])
return quoteDict
def search(request):
stock = request.GET.get('put')
flagvar = 0
try:
symbol = Share(stock)
stock_name = symbol.get_name()
stock_data = symbol.get_historical('2016-01-01',
date.today().strftime("%Y-%m-%d"))
stock_df = pd.DataFrame(stock_data)
abbr = stock_df['Symbol'][0]
temp = pd.DataFrame({
'Close_Price': [],
'Low': [],
'High': [],
'Date': []
})
temp['Date'] = stock_df['Date']
temp['High'] = stock_df['High']
temp['Low'] = stock_df['Low']
temp['Close_Price'] = stock_df['Adj_Close']
path = os.getcwd()
filepath = path + os.sep + 'static' + os.sep + 'js' + os.sep + 'search'
os.chdir(filepath)
temp.to_csv(filepath + "/result.csv", index_label=False, index=False)
os.chdir(path)
flagvar = 1
except:
flagvar = 0
abbr = ''
stock_name = ''
context = {'flag': flagvar, 'abbr': abbr, 'stock_name': stock_name}
return render(request, 'search2.html', context)
def get_stock_df(ticker,start_date,end_date):
"""Get stock dataframe"""
share = Share(ticker)
share_hist = share.get_historical(start_date,end_date)
len_share_hist = len(share_hist)
dates = ['']*len_share_hist
open = [0.]*len_share_hist
close = [0.]*len_share_hist
high = [0.]*len_share_hist
low = [0.]*len_share_hist
volume = [0.]*len_share_hist
adj_close = [0.]*len_share_hist
for i in range(len_share_hist):
dates[i] = share_hist[i][DATE_STR]
open[i] = float(share_hist[i][OPEN_STR])
close[i] = float(share_hist[i][CLOSE_STR])
adj_close[i] = float(share_hist[i][ADJ_CLOSE_STR])
high[i] = float(share_hist[i][HIGH_STR])
low[i] = float(share_hist[i][LOW_STR])
volume[i] = float(share_hist[i][VOLUME_STR])
df = pd.DataFrame(open, index = pd.to_datetime(dates), columns=[OPEN_STR])
df[CLOSE_STR] = close
df[ADJ_CLOSE_STR] = adj_close
df[HIGH_STR] = high
df[LOW_STR] = low
df[VOLUME_STR] = volume
df.index.name = DATE_STR
return df.sort_index()
def get_stock_history(request, code):
if request.method == 'GET':
# print(Currency.objects.filter(url_code__iexact=code).query)
currency = Currency.objects.get(url_code__iexact=code)
date1 = request.GET.get('from', '2016-04-01')
date2 = request.GET.get('to', '2020-05-01')
if currency.exchange == 0:
date1 = datetime.strptime(date1, '%Y-%m-%d')
date2 = datetime.strptime(date2, '%Y-%m-%d')
params = currency.code.split('-')
history = [{
'price': get_currencies_date(*params, date),
'date': date.strftime('%Y-%m-%d')
} for date in get_date_list(date1, date2)]
return Response(history)
elif currency.exchange > 1:
yahoo = Share(currency.get_stock_identifier())
history = yahoo.get_historical(date1, date2)
history = map(lambda x: {
'price': x['Close'],
'date': x['Date'],
}, history)
return Response(history,
headers={'Access-Control-Allow-Origin': '*'})
elif currency.code == 'BTC':
return Response([], headers={'Access-Control-Allow-Origin': '*'})
# return {'code':self.code, 'price':Bitfinex().get_current_price()}
return Response([])
def make_csv(self,name,now_time):
if os.path.exists(self.DIR_NAME+name+".csv"):
print "Already have a %s.csv"%name
return
print "making %s.csv....." %name
from yahoo_finance import Share
share=Share(name)
info=share.get_info()
if "start" not in info:
print "Cant make a %s.csv"%name
return -1
start=info["start"]
#wrong date
if re.match("\d{4}-\d{2}-\d{2}",start)== None:
print "invalid date cant make a file"
return -1
obj=share.get_historical(start,now_time)
filename=name+".csv"
fieldnames=("Date","High","Low","Volume" )
headers = dict( (n,n) for n in fieldnames )
f=open(self.DIR_NAME+filename,"w")
writer=csv.DictWriter(f,fieldnames)
writer.writerow(headers)
for o in obj:
row=""
writer.writerow({"Date":o["Date"],"High":o["High"],"Low":o["Low"],"Volume":o["Volume"]})
f.close()
def getYahooStock(ticker, date1, date2):
companyData = Share(ticker)
dataList = companyData.get_historical(date1, date2)
endData = dataList[0]
startData = dataList[len(dataList) - 1]
print ticker, float(startData['Open']), float(endData['Open'])
return ticker, float(startData['Open']), float(endData['Open'])
def insert_data(symbol, prefix=""):
# open the database connection
conn = sqlite3.connect(prefix + 'stocks.db')
cursor = conn.cursor()
insert_query = \
'INSERT INTO historical VALUES ("{0}", "{1}", {2}, {3}, {4}, {5}, {6})'
print 'getting data for', symbol
date_output_format = '%Y-%m-%d'
date_output = datetime.now().strftime(date_output_format)
# call the api for historical data
stock = Share(symbol)
data = stock.get_historical('2014-01-01', date_output)
# iterate the data
path = 'data/hist_' + symbol + '.csv'
for day_data in data:
try:
# insert into the database
cursor.execute(
insert_query.format(symbol, day_data['Date'], day_data['Open'],
day_data['High'], day_data['Low'],
day_data['Close'], day_data['Volume']))
except Exception as e:
print e
print 'insert failed for', symbol, day_data['Date']
# commit and close the database
conn.commit()
conn.close()
def retrieveQuoteFromYahoo(symbol, start, end):
share = Share(symbol)
quoteList = share.get_historical(start, end)
quoteDict = {}
for quote in quoteList:
quoteDict[quote['Date']] = float(quote['Adj_Close'])
return quoteDict
def get_ma(stock):
prices_10 = []
prices_20 = []
stock = Share(stock)
if (stock.get_price() < 5):
return 0, 0, 0
dma_10 = now - timedelta(days=30)
date10 = str(dma_10.year) + "-" + str(dma_10.month) + "-" + str(dma_10.day)
time.sleep(0.5)
try:
data = stock.get_historical(date10, theDate)
prices_50 = stock.get_50day_moving_avg()
count = 0
for theData in data:
if count < 10:
count = count + 1
prices_10.append(float(theData['Adj_Close']))
else:
continue
count = 0
for theData in data:
if count < 20:
count = count + 1
prices_20.append(float(theData['Adj_Close']))
else:
continue
except:
return 0, 0, 0
print prices_50
return prices_10, prices_20, prices_50
def calculateValue(self, date, prices):
value = 0.
date = date.strftime('%Y-%m-%d')
for sym, qty in self.holdings.items():
if sym == 'cash':
value += qty
else:
if sym not in prices:
stock = Share(sym)
# get prices until today
string_today = datetime.date.today().strftime('%Y-%m-%d')
price_list = stock.get_historical(date, string_today)
# pprint.PrettyPrinter(depth = 6).pprint(price_list)
# Prices is a dictionary of dictionaries; each inner dictionary consists of (date, value) pairs for a ticker
prices[sym] = {
item['Date']: float(item['Adj_Close'])
for item in price_list
}
# find price for the date of interest
if date in prices[sym]:
close_price = prices[sym][date]
value += close_price * qty
else:
print(date)
print(sym)
return (None, None)
return (value, prices)
def yhGetHistory(myTicker, start, stop):
"""
Helper function used by getMeData function
myTicker: string ticker symbol
start: string, format = "YYYY-M-D"
stop: string, format = "YYYY-M-D"
return: list of dictionaries
"""
histList = []
myStock = Share(myTicker)
myRecord = myStock.get_historical(start, stop)
for i in range(len(myRecord)):
tempDate = myRecord[i]["Date"]
tempOpen = float(myRecord[i]["Open"])
tempHigh = float(myRecord[i]["High"])
tempLow = float(myRecord[i]["Low"])
tempClose = float(myRecord[i]["Close"])
tempVolume = int(myRecord[i]["Volume"])
tempAdjClose = float(myRecord[i]["Adj_Close"])
histList.append({
"Date": tempDate,
"Open": round(tempOpen, 2),
"High": round(tempHigh, 2),
"Low": round(tempLow, 2),
"Close": round(tempClose, 2),
"Volume": tempVolume,
"Adj Close": round(tempAdjClose, 2)
})
return histList
def getStockData(theTicker, startDate):
stock = Share(theTicker)
print("Getting Data for ... " + theTicker)
now = datetime.now()
DateNow = str(now.year) + "-" + str(now.month) + "-" + str(now.day)
data = stock.get_historical(startDate, DateNow)
stockData = []
for d in data:
tmp = []
volume = int(d['Volume'])
adjclose = float(d['Adj_Close'])
high = float(d['High'])
low = float(d['Low'])
close = float(d['Close'])
date = d['Date']
open = float(d['Open'])
tmp.append(date)
tmp.append(open)
tmp.append(high)
tmp.append(low)
tmp.append(close)
tmp.append(adjclose)
tmp.append(volume)
stockData.append(tmp)
return stockData
def graph_it(sym, k=4, start_date='2016-04-04', end_date='2016-04-08'): # Specificy a stock symbol, number of latent features k, a start and an end date
df = nmf_to_df(sym, k)
yahoo = Share(sym)
yahoo_days = yahoo.get_historical(start_date, end_date)
print "Completed Yahoo historical API search..."
y_money = [ day['Close'] for day in yahoo_days ] # the closing prices for each day from start to end
date_list, day_list = get_date_list(start_date, end_date)
color_list = ['b','g','r','c','m','y','k','w'] # usable colors for matplotlib
print "Creating plots..."
fig, ax1 = plt.subplots() #figsize=(30,15)
ax1.plot(day_list, y_money)
ax1.set_ylabel('Close', color=color_list[0])
for tl in ax1.get_yticklabels():
tl.set_color(color_list[0])
for sp in range(k):
y_sent = []
lat_feat = 'lat' + str(sp)
for day in date_list:
print df[lat_feat]
y_sent.append(df[lat_feat].ix[day])
ax2 = ax1.twinx()
ax2.plot(day_list, y_sent, color=color_list[sp+1])
ax2.set_ylabel('Sent'+str(sp), color=color_list[sp+1])
for tl in ax2.get_yticklabels():
tl.set_color(color_list[sp+1])
plt.show()
def main():
spy = Share('SPY') #get object about SPY
iwv = Share('IWV') #get object about IWV
end = '2017-02-04' #set the end date
start = '2007-02-04' #set the start date
spy_hist = spy.get_historical(start, end) #get historical data about spy
iwv_hist = iwv.get_historical(start, end) #get historical data about iwv
spy_adj = [float(dic['Adj_Close'])
for dic in spy_hist] #store adjust close prices
iwv_adj = [float(dic['Adj_Close'])
for dic in iwv_hist] #store adjust close prices
spy_return = getLogReturn(spy_adj)
iwv_return = getLogReturn(iwv_adj)
xt = diffLists(spy_return, iwv_return)
result = modelAR("AR30.csv", xt, 30)
print(result)
def get_data(stock, start, end):
data = Share(stock)
try:
data = pd.DataFrame(data.get_historical(start_date=start,
end_date=end))
except Exception as e:
f = open('log.txt', mode='a')
f.write(stock + '\n')
f.write(str(e) + '\n')
return pd.DataFrame()
try:
data.index = data.Date
except Exception as e:
f = open('log.txt', mode='a')
f.write(stock + '\n')
f.write(str(e) + '\n')
return pd.DataFrame()
data = data.drop(['Date', 'Symbol'], axis=1)
data = data.sort_index()
for i in data.columns:
data[i] = data[i].astype(np.float)
#data['Adj_Open'] = 0
#data['Adj_High'] = 0
#data['Adj_Low'] = 0
#for i in range(len(data)):
# k = data['Adj_Close'][i] / data['Close'][i]
# data.loc[i:i+1,'Adj_Open'] = k*data['Open'][i]
# data.loc[i:i + 1, 'Adj_High'] = k * data['High'][i]
# data.loc[i:i + 1, 'Adj_Low'] = k * data['Low'][i]
data['Symbol'] = stock
return data
def getStock(id):
stock = Share(str(id) + '.TW')
today = datetime.date.today()
data = stock.get_historical('2017-04-20', str(today))
# data = stock.get_historical('2017-04-20', '2017-04-20')
# data = stock.get_historical(str(today), str(today))
return data
def download_prices_to_today(ticker, prices):
"""Download prices till today.
:param str ticker:
:param [dict] prices: prices sorte by date
"""
try:
share = Share(ticker)
# get from_date
if not prices:
from_date = datetime.strptime('2015-08-03', '%Y-%m-%d')
else:
last_date = datetime.strptime(prices[-1]['Date'], '%Y-%m-%d')
from_date = (last_date + timedelta(days=1))
# get to_date
to_date = datetime.today()
day_of_week = to_date.weekday()
if day_of_week == 6:
to_date = to_date + timedelta(days=-2) # if sunday
elif day_of_week == 5:
to_date = to_date + timedelta(days=-1) # if saturday
# skip if already update-to-date
if from_date >= to_date:
return
str_from_date = from_date.strftime('%Y-%m-%d')
str_to_date = to_date.strftime('%Y-%m-%d')
eprint('downloading prices', ticker, str_from_date, str_to_date)
data = share.get_historical(str_from_date, str_to_date)
return data
except Exception as ex:
eprint(ex)
def start_stock():
# Busca as informações do pregão
data_corrente = start_data()
data_exec = data_corrente[0:4] + '-' + data_corrente[4:6] + '-' + data_corrente[6:8]
lista_acoes = cria_lista()
#lista_acoes = ['PETR3.SA', 'VALE3.SA', 'BBAS3.SA']
nome_arq = 'pregao' + str(data_exec) + '.txt'
arquivo = open(nome_arq, 'w')
texto = ''
informacao = None
for acao in lista_acoes:
try:
yahoo = Share(str(acao))
informacao = yahoo.get_historical(data_exec,data_exec)
except:
pass
if informacao :
simbolo = informacao[0]['Symbol'].split('.SA', 1)[0]
try:
fechamento = informacao[0]['Close']
except KeyError:
fechamento = ''
try:
volume = informacao[0]['Volume']
except KeyError:
volume = ''
texto = simbolo + ';' + fechamento + ';' + volume + "\n"
arquivo.writelines(texto)
arquivo.close()
return "Registros salvos no arquivo: " + nome_arq
def stock_data(start, end,companies):
'''
Function:
Parameters: start date, end date, and
a list of companies names
'''
raw_data = {
'High': [],
'Low': [],
'Open': [],
'Close': [],
'Volume': [],
'Date': []
}
for key,value in companies.items():
company = Share(key)
company.refresh()
data = company.get_historical(start,end)
for item in data:
raw_data['High'].append(float(item['High']))
raw_data['Low'].append(float(item['Low']))
raw_data['Open'].append(float(item['Open']))
raw_data['Close'].append(float(item['Close']))
raw_data['Volume'].append(int(item['Volume']))
raw_data['Date'].append(item['Date'])
return raw_data
def get_point(ticker, start_date, end_date, traits):
stock = Share(ticker)
days = stock.get_historical(start_date, end_date)
x = []
for day in days:
for trait in traits:
x.append(float(day[trait]))
return x
def getHistoricalData():
historicalPrices = []
stock = Share('^VIX')
data = stock.get_historical('2015-01-01', '2016-01-01')
data.reverse()
for elem in data:
historicalPrices.append(float(elem['Adj_Close']))
return historicalPrices
def init():
share = Share('CVE.TO')
hist = share.get_historical('2015-12-03', '2017-01-09')
with open('data-cve.txt', 'w') as file:
for item in hist:
print (json.dumps(item))
file.write(json.dumps(item))
file.write("\n")
def get(self, request):
share = Share('YHOO')
data = share.get_historical('2015-12-03', '2016-02-18')
data = data[::-1]
for d in data:
d['Close'] = float(d['Close'])
return HttpResponse(json.dumps(data))
def getHistoricalPriceFromRange(ticker, start, end):
stocks = Share(ticker.replace("." , "-"))
answerList = []
closeList = []
for item in (stocks.get_historical(start,end)):
answerList.append( (item['Date'],item["Low"] , item["High"] , item["Close"],item["Open"]) )
closeList.append( float(item['Close']) )
pprint (closeList)
