def stock_prices():
print "Begin..."
df = pd.DataFrame()
statspath = path + "/_KeyStats"
stock_list = [x[0] for x in os.walk(statspath)]
print stock_list[0]
for each_dir in stock_list[1:]:
try:
ticker = each_dir.split("//")[0]
print ticker
name = "WIKI/" + ticker.upper()
data = Quandl.get(name,
trim_start="2000-12-12",
trim_end="2014-12-12",
authtoken=auth_tok)
data[ticker.upper()] = data["Adj. Close"]
df = pd.concat([df, data[ticker.upper()]], axis=1)
except Exception as e:
print str(e)
try:
ticker = each_dir.split("//")[0]
print ticker
name = "WIKI/" + ticker.upper()
data = Quandl.get(name,
trim_start="2000-12-12",
trim_end="2014-12-12",
authtoken=auth_tok)
data[ticker.upper()] = data["Adj. Close"]
df = pd.concat([df, data[ticker.upper()]], axis=1)
except Exception as e:
print str(e)
df.to_csv("stock_prices.csv")
def Stock_Prices():
df = pd.DataFrame()
statspath = path+'\\_KeyStats'
stock_list = [x[0] for x in os.walk(statspath)]
for each_dir in stock_list[1:]:
try:
ticker = each_dir.split("\\")[-1]
print(ticker)
name = "WIKI/"+ ticker.upper()
data = Quandl.get(name, trim_start = "2000-12-12",trim_end = "2014-12-30", authtoken= auth_tok)
data[ticker.upper()] = data["Adj. Close"]
df= pd.concat([df, data[ticker.upper()]], axis = 1)
except Exception as e:
print(str(e))
time.sleep(10)
try:
ticker = each_dir.split("\\")[-1]
print(ticker)
name = "WIKI/"+ ticker.upper()
data = Quandl.get(name, trim_start = "2000-12-12",trim_end = "2014-12-30", authtoken= auth_tok)
data[ticker.upper()] = data["Adj. Close"]
df= pd.concat([df, data[ticker.upper()]], axis = 1)
except Exception as e:
print(str(e))
df.to_csv("stock_prices.csv")
def Stock_Prices():
statspath = path
stock_list = [x[0] for x in os.walk(statspath)]
print(stock_list)
for each_dir in stock_list[1:]:
try:
ticker = each_dir.split("\\")[8]
print(ticker)
name = "WIKI/" + ticker.upper()
data = Quandl.get(name,
trim_start="2000-12-12",
trim_end="2014-12-30",
authtoken=auth_tok)
data[ticker.upper()] = data["Adj. Close"]
df = pd.concat([df, data[ticker.upper()]], axis=1)
except Exception as e:
print(str(e))
time.sleep(10)
try:
ticker = each_dir.split("\\")[1]
print(ticker)
name = "WIKI/" + ticker.upper()
data = Quandl.get(name,
trim_start="2000-12-12",
trim_end="2014-12-30",
authtoken=auth_tok)
data[ticker.upper()] = data["Adj. Close"]
df = pd.concat([df, data[ticker.upper()]], axis=1)
df.to_csv("stock_prices.csv")
except Exception as e:
print(str(e))
def Stock_Prices():
df = pd.DataFrame()
statspath = path+'/_KeyStats'
stock_list = [x[0] for x in os.walk(statspath)]
# print(stock_list)
for each_dir in stock_list[1:]:
try:
# ticker = each_dir.split("\\")[1] # Windows only
# ticker = each_dir.split("/")[1] # this didn't work so do this:
ticker = os.path.basename(os.path.normpath(each_dir))
# print(ticker) # uncomment to verify
name = "WIKI/"+ticker.upper()
print(name)
data = Quandl.get(
name,
trim_start = "2000-12-12",
trim_end = "2014-12-30",
authtoken=auth_tok
)
data[ticker.upper()] = data["Adj. Close"]
df = pd.concat([df, data[ticker.upper()]], axis = 1)
except Exception as e:
# this except is just a simple retry...
print(str(e))
time.sleep(10)
try:
# ticker = each_dir.split("\\")[1] # Windows only
# ticker = each_dir.split("/")[1] # this didn't work on *nix so do this:
ticker = os.path.basename(os.path.normpath(each_dir))
# print(ticker) # uncomment to verify
name = "WIKI/"+ticker.upper()
data = Quandl.get(
name,
trim_start = "2000-12-12",
trim_end = "2014-12-30",
authtoken=auth_tok
)
data[ticker.upper()] = data["Adj. Close"]
df = pd.concat([df, data[ticker.upper()]], axis = 1)
except Exception as e:
print(str(e))
df.to_csv("stock_prices.csv")
def index_data():
if request.method=='GET':
return render_template('datarequest.html')
else:
# request was a POST
app_data.vars['symbol']=request.form['stock_data']
auth_tok = "VJmYz-1u3Zf6P3ocrkUN"
string1="WIKI/"
string2=app_data.vars['symbol']
string3=string1+string2
data = Quandl.get(string3, trim_start = "2015-09-01", trim_end = "2015-12-01", authtoken=auth_tok)
dateaxis = data.reset_index()['Date']
close_data=data['Close']
x=dateaxis
y=close_data
fig=plt.figure()
axes=fig.add_subplot(1,1,1)
axes.plot(x, y, marker='.')
#labels= axes.set_xticklabels(rotation=60, fontsize='small')
axes.tick_params(axis='x',labelsize=8)
f=tempfile.NamedTemporaryFile(dir='static/',suffix='png',delete=False)
plt.savefig(f)
f.close()
plotPng=f.name.split('/')[-1]
return(render_template('figures.html',y=y, plotPng=plotPng))
def __init__(self,startDate):
#create dictionary mapping ED contract to #, 1st, 2nd, 3rd, etc
d = {1:"EDH2014", 2:"EDM2014", 3:"EDU2014", 4:"EDZ2014",
5:"EDH2015", 6:"EDM2015", 7:"EDU2015", 8:"EDZ2015",
9:"EDH2016", 10:"EDM2016", 11:"EDU2016", 12:"EDZ2016",
13:"EDH2017", 14:"EDM2017", 15:"EDU2017", 16:"EDZ2017",
17:"EDH2018", 18:"EDM2018", 19:"EDU2018", 20:"EDZ2018",
21:"EDH2019", 22:"EDM2019", 23:"EDU2019", 24:"EDZ2019",
25:"EDH2020", 26:"EDM2020", 27:"EDU2020", 28:"EDZ2020"}
self.dates = {1:'2014-03-19',2:'2014-06-18',3:'2014-09-17',4:'2014-12-17',
5:'2015-03-18',6:'2015-06-17',7:'2015-09-16',8:'2015-12-16',
9:'2016-03-16',10:'2016-06-15',11:'2016-09-21',12:'2016-12-21',
13:'2017-03-15',14:'2017-06-21',15:'2017-09-20',16:'2017-12-20',
17:'2018-03-21',18:'2018-06-20',19:'2018-09-19',20:'2018-12-19',
21:'2019-03-20',22:'2019-06-19',23:'2019-09-18',24:'2019-12-18',
25:'2020-03-18',26:'2020-06-17',27:'2020-09-16',28:'2020-12-16'}
self.startDate = startDate
self.edDict = dict(d)
ed1 = Quandl.get("CME/EDH2020", trim_start = self.startDate, authtoken = auth_tok)
self.edRates = 100 - pandas.DataFrame(ed1["Settle"])
self.edRates.columns = ["EDH2020"]
self.edVolume = pandas.DataFrame(ed1["Volume"])
self.edVolume.columns = ["EDH2020"]
self.edOI = pandas.DataFrame(ed1["Open Interest"])
self.edOI.columns = ["EDH2020"]
def main(self):
results = {}
for symbol in self.symbols:
code = self.generate_quandl_code(symbol)
results[symbol] = Quandl.get(code, authtoken=self.base_config['api_key'], returns='pandas')
results[symbol] = results[symbol].reset_index()
return results
def graph():
stock = app.vars['stock']
stockq = "/".join(("WIKI", stock))
data = Quandl.get(stockq, rows=20, authtoken=auth_tok, returns="pandas")
df = data[['Close']]
p = figure(width=700, height=500, title=stock, x_axis_type='datetime')
p.circle(x=df.index, y=df[['Close']])
script, div = components(p)
return render_template('graph.html', script=script, div=div)
def Stock_Prices():
df = pd.DataFrame()
statspath = path + '/_KeyStats'
stock_list = [x[0] for x in os.walk(statspath)]
# print(stock_list)
for each_dir in stock_list[1:]:
try:
# ticker = each_dir.split("\\")[1] # Windows only
# ticker = each_dir.split("/")[1] # this didn't work so do this:
ticker = os.path.basename(os.path.normpath(each_dir))
# print(ticker) # uncomment to verify
name = "WIKI/" + ticker.upper()
print(name)
data = Quandl.get(name,
trim_start="2000-12-12",
trim_end="2014-12-30",
authtoken=auth_tok)
data[ticker.upper()] = data["Adj. Close"]
df = pd.concat([df, data[ticker.upper()]], axis=1)
except Exception as e:
# this except is just a simple retry...
print(str(e))
time.sleep(10)
try:
# ticker = each_dir.split("\\")[1] # Windows only
# ticker = each_dir.split("/")[1] # this didn't work on *nix so do this:
ticker = os.path.basename(os.path.normpath(each_dir))
# print(ticker) # uncomment to verify
name = "WIKI/" + ticker.upper()
data = Quandl.get(name,
trim_start="2000-12-12",
trim_end="2014-12-30",
authtoken=auth_tok)
data[ticker.upper()] = data["Adj. Close"]
df = pd.concat([df, data[ticker.upper()]], axis=1)
except Exception as e:
print(str(e))
df.to_csv("stock_prices.csv")
def loadAll(self):
for i in range(4,29):
label = "CME/" + self.edDict[i]
temp = Quandl.get(label, trim_start = self.startDate, authtoken = auth_tok)
self.edRates[self.edDict[i]] = 100 - temp["Settle"]
self.edVolume[self.edDict[i]] = temp["Volume"]
self.edOI[self.edDict[i]] = temp["Open Interest"]
#issues with the data (even though its from the cme) - manual fix
self.edRates["EDZ2017"]["2014-02-24"] = 3
def pullSwapData(self, inputDate):
swapArray = numpy.zeros(9)
swapArray[0] = Quandl.get("FRED/USD3MTD156N", trim_start = inputDate, trim_end = inputDate, authtoken = auth_tok)["VALUE"][0]
swapArray[1] = fred.get_series('DSWP1', observation_start=inputDate, observation_end=inputDate)[0]
swapArray[2] = fred.get_series('DSWP2', observation_start=inputDate, observation_end=inputDate)[0]
swapArray[3] = fred.get_series('DSWP3', observation_start=inputDate, observation_end=inputDate)[0]
swapArray[4] = fred.get_series('DSWP4', observation_start=inputDate, observation_end=inputDate)[0]
swapArray[5] = fred.get_series('DSWP5', observation_start=inputDate, observation_end=inputDate)[0]
swapArray[6] = fred.get_series('DSWP7', observation_start=inputDate, observation_end=inputDate)[0]
swapArray[7] = fred.get_series('DSWP10', observation_start=inputDate, observation_end=inputDate)[0]
swapArray[8] = fred.get_series('DSWP30', observation_start=inputDate, observation_end=inputDate)[0]
return swapArray/100
def getHistoricalData(ticker):
onlineData = Quandl.get("YAHOO/" + ticker, authtoken = authtoken, collapse='daily')
localData = []
for i in dates:
singleDayData = {'Date': i}
for j in stockHeaders:
try:
singleDayData[j] = str(onlineData.loc[i, j])
except KeyError:
# print "The date " + i + " has no data."
break
else:
localData.append(singleDayData)
return localData
def hichart_quandl(request):
from Quandl import Quandl
import json
myAAPL_data = Quandl.get("WIKI/AAPL", returns="pandas", column="11",
authtoken="L5A6rmU9FGvyss9F7Eym", trim_start='2006/06/15', trim_end='2007/06/15')
data = json.loads(myAAPL_data.to_json()) # convert to JSON object...
#### Below logic is quite imp as
# this is transforming pandas dataframe to the High charts input structure
data_list = list(sorted(data['Adj. Close'].items()))
highcharts_data = []
for x in range(len(data_list)):
dl = list(data_list[x])
dl[0] = int(dl[0])
highcharts_data.append(dl)
### This is important to note json.dumps() convert python data structure to JSON form
return HttpResponse(json.dumps(highcharts_data), content_type='application/json')
def getIndicatorFromQuandl(indicator_name):
resolution = 'monthly'
print "Getting "+indicator_name+" indicator from Quandl"
today = str(datetime.datetime.now().year)+"-"+str(datetime.datetime.now().month)+"-"+str(datetime.datetime.now().day)
data = Quandl.get(indicator_name, authtoken='5UND8fCsD7oWNEs1fpfa', trim_start='1900-01-01', trim_end=today, collapse=resolution, transformation='none', returns='numpy')
datapoints = []
for datapoint in data:
dt = datapoint[0]
datapoints.append({"name":indicator_name, "date":(dt.year, dt.month, dt.day), "value": datapoint[1] })
# insert multiple records to the table
tblIndicators.insert_multiple(datapoints)
# get name i.e. description of the indicator
meta = searchQuandl(indicator_name);
desc = meta[0]["name"]
# insert new record into indicators index
tblIndex.insert({"desc": desc, "name": indicator_name, "resolution": resolution, "count": len(datapoints)})
print "Sucessful insert of "+str(len(datapoints))+" records."+indicator_name
return datapoints
def getCorporates(self, trim_start, trim_end, WORKING_DIR='.'):
self.OIS = OIS(trim_start=trim_start,
trim_end=trim_end,
WORKING_DIR=WORKING_DIR)
self.datesAll = self.OIS.datesAll
self.OISData = self.OIS.getOIS()
self.WORKING_DIR = WORKING_DIR
for rating in self.ratings:
index = 'ML/' + rating + 'TRI'
try:
corpSpreads = 1e-4 * (Quandl.get(
index,
authtoken="Lqsxas8ieaKqpztgYHxk",
trim_start=trim_start,
trim_end=trim_end))
corpSpreads.reset_index(level=0, inplace=True)
corpSpreads = pd.merge(left=self.datesAll,
right=corpSpreads,
how='left')
corpSpreads = corpSpreads.fillna(method='ffill').fillna(
method='bfill')
self.corpSpreads[rating] = corpSpreads.T.fillna(
method='ffill').fillna(method='bfill').T
except:
print(index, " not found")
self.corpSpreads = pd.Panel.from_dict(self.corpSpreads)
self.corporates = {}
self.OISData.drop('Date', axis=1, inplace=True)
ntenors = np.shape(self.OISData)[1]
for rating in self.ratings:
try:
tiledCorps = np.tile(self.corpSpreads[rating]['Value'],
ntenors)
tiledCorps = tiledCorps.reshape(np.shape(self.OISData),
order="F")
self.corporates[rating] = pd.DataFrame(
data=(tiledCorps + self.OISData.values))
self.corporates[rating].drop(
self.corporates[rating].columns[[0]], axis=1, inplace=True)
except:
print("Error in addition of Corp Spreads")
self.corporates = pd.Panel(self.corporates)
return self.corporates
def __init__(self,
trim_start="2005-01-10",
trim_end="2010-01-10",
WORKING_DIR='.'):
self.OIS = 0.01 * Quandl.get("USTREASURY/YIELD",
authtoken="Lqsxas8ieaKqpztgYHxk",
trim_start=trim_start,
trim_end=trim_end)
self.OIS.reset_index(level=0, inplace=True)
self.datesAll = pd.DataFrame(pd.date_range(trim_start, trim_end),
columns=['Date'])
self.datesAll.reset_index(level=0, inplace=True)
self.OIS = pd.merge(left=self.datesAll, right=self.OIS, how='left')
self.OIS = self.OIS.fillna(method='ffill').fillna(method='bfill')
self.OIS = self.OIS.T.fillna(method='ffill').fillna(
method='bfill').T # Fill NA forward and backward to eliminate NA
self.WORKING_DIR = WORKING_DIR
self.OIS_dict = {}
self.t_step = 1 / 365.0
def Stock_Prices():
df = pd.DataFrame()
statspath = path + "/_KeyStats"
stock_list = [x[0] for x in os.walk(statspath)]
stock_list.sort()
for each_dir in stock_list[1:]:
try:
ticker = each_dir.split('/')[6]
print(ticker)
name = "WIKI/" + ticker.upper()
data = Quandl.get(name,
trim_start="2000-12-12",
trim_end="2014-12-30",
authtoken=auth_tok)
data[ticker.upper()] = data["Adj. Close"]
df = pd.concat([df, data[ticker.upper()]], axis=1)
except Exception as e:
print(str(e))
#time.sleep(10)
df.to_csv("Stock_Prices.csv")
def Stock_Prices():
df = pd.DataFrame()
statspath = path+'Yahoo/intraQuarter/_KeyStats'
stock_list = [x[0] for x in os.walk(statspath)]
for each_dir in stock_list[1:]:
try:
ticker = each_dir.split('_KeyStats/')[1]
print ticker
name = 'WIKI/' + ticker.upper()
data = Quandl.get(name, trim_start=datetime.strptime('2000-12-12', '%Y-%m-%d'),
trim_end=date.today()-timedelta(1),
authtoken=auth_token)
data[ticker.upper()] = data['Adj. Close']
df = pd.concat([df, data[ticker.upper()]], axis=1)
except Exception as e:
print('Error polling Quandl: ' +str(e))
df.to_csv('stock_prices.csv')
def stock_prices():
df = pd.DataFrame()
statspath = path + '/_KeyStats'
stock_list = [x[0] for x in os.walk(statspath)]
for each_dir in stock_list[1:]:
try:
ticker = each_dir.split("/Users/User/intraQuarter/_KeyStats/")[1]
print(ticker)
name = "WIKI/" + ticker.upper()
# Query Quandl, using the standard format, e.g WIKI/AAPL.
data = Quandl.get(name,
trim_start="2000-12-12",
trim_end="2014-12-30",
authtoken=auth_tok)
data[ticker.upper()] = data["Adj. Close"]
df = pd.concat([df, data[ticker.upper()]], axis=1)
except Exception as e:
print(str(e))
df.to_csv("stock_prices.csv")
def hichart_quandl(request):
from Quandl import Quandl
import json
myAAPL_data = Quandl.get("WIKI/AAPL",
returns="pandas",
column="11",
authtoken="L5A6rmU9FGvyss9F7Eym",
trim_start='2006/06/15',
trim_end='2007/06/15')
data = json.loads(myAAPL_data.to_json()) # convert to JSON object...
#### Below logic is quite imp as
# this is transforming pandas dataframe to the High charts input structure
data_list = list(sorted(data['Adj. Close'].items()))
highcharts_data = []
for x in range(len(data_list)):
dl = list(data_list[x])
dl[0] = int(dl[0])
highcharts_data.append(dl)
### This is important to note json.dumps() convert python data structure to JSON form
return HttpResponse(json.dumps(highcharts_data),
content_type='application/json')
def Stock_Prices():
df = pd.DataFrame()
statspath = path + 'Yahoo/intraQuarter/_KeyStats'
stock_list = [x[0] for x in os.walk(statspath)]
for each_dir in stock_list[1:]:
try:
ticker = each_dir.split('_KeyStats/')[1]
print ticker
name = 'WIKI/' + ticker.upper()
data = Quandl.get(name,
trim_start=datetime.strptime(
'2000-12-12', '%Y-%m-%d'),
trim_end=date.today() - timedelta(1),
authtoken=auth_token)
data[ticker.upper()] = data['Adj. Close']
df = pd.concat([df, data[ticker.upper()]], axis=1)
except Exception as e:
print('Error polling Quandl: ' + str(e))
df.to_csv('stock_prices.csv')
import pandas as pd
from Quandl import Quandl
import math, datetime, pickle
import numpy as np
from sklearn import preprocessing, cross_validation, svm
from sklearn.linear_model import LinearRegression
import matplotlib.pyplot as plt
from matplotlib import style
style.use('ggplot')
df = Quandl.get('WIKI/GOOGL')
df = df[['Adj. Open','Adj. High','Adj. Low','Adj. Close','Adj. Volume',]]
df['HL_PCT'] = (df['Adj. High']-df['Adj. Low'])/df['Adj. Low'] *100.0
df['PCT_Change'] = (df['Adj. Close']-df['Adj. Open'])/df['Adj. Open'] *100.0
df=df[['Adj. Close','HL_PCT','PCT_Change','Adj. Volume']]
forecast_col = 'Adj. Close'
df.fillna(-99999, inplace=True)
forecast_out = int(math.ceil(0.01*len(df)))
df['label']=df[forecast_col].shift(-forecast_out)
X = np.array(df.drop(['label'],1))
X = preprocessing.scale(X)
X_lately = X[-forecast_out:]
X = X[:-forecast_out]
import pandas as pd
from Quandl import Quandl
import math, datetime, pickle
import numpy as np
from sklearn import preprocessing, cross_validation, svm
from sklearn.linear_model import LinearRegression
import matplotlib.pyplot as plt
from matplotlib import style
style.use('ggplot')
df = Quandl.get('WIKI/GOOGL')
df = df[[
'Adj. Open',
'Adj. High',
'Adj. Low',
'Adj. Close',
'Adj. Volume',
]]
df['HL_PCT'] = (df['Adj. High'] - df['Adj. Low']) / df['Adj. Low'] * 100.0
df['PCT_Change'] = (df['Adj. Close'] -
df['Adj. Open']) / df['Adj. Open'] * 100.0
df = df[['Adj. Close', 'HL_PCT', 'PCT_Change', 'Adj. Volume']]
forecast_col = 'Adj. Close'
df.fillna(-99999, inplace=True)
forecast_out = int(math.ceil(0.01 * len(df)))
import pandas as pd
import os
from Quandl import Quandl
import time
# auth_tok = "your_auth_here"
auth_tok = open("quandl_auth_tok.txt", "r").read()
data = Quandl.get("WIKI/KO",
trim_start="2000-12-12",
trim_end="2014-12-30",
authtoken=auth_tok)
print(data)
def searchQuandl(term):
res = Quandl.search(query = term, page = 1)
return res
country_array.append(m)
x = dict(country_array)
return x
for key in form_dictionary():
data_string = 'WORLDBANK/' + key + '_SP_DYN_SMAM_FE' # format of the data on Quandl database
country_keys.append(data_string)
'''
# Test data - Afghanistan, Albania, Algeria, Antigua and Barbada, Argentina, USA, Channel Islands (Channel Islands not in database)
test_country_keys = ['WORLDBANK/AFG_SP_DYN_SMAM_FE', 'WORLDBANK/ALB_SP_DYN_SMAM_FE', 'WORLDBANK/DZA_SP_DYN_SMAM_FE', 'WORLDBANK/ATG_SP_DYN_SMAM_FE', 'WORLDBANK/ARG_SP_DYN_SMAM_FE', 'WORLDBANK/USA_SP_DYN_SMAM_FE', 'WORLDBANK/CHI_SP_DYN_SMAM_FE']
firstmarriage = Quandl.get(test_country_keys, authtoken=auth_token, order='desc')
'''
firstmarriage = Quandl.get(country_keys, authtoken=auth_token, order='desc')
firstmarriage_trimmed = {} # type is dict, no longer pandas DataFrame - DataFrame doesn't allow me to strip out null values, must be even matrix
for key in firstmarriage:
if key[-9:] == 'NOT FOUND': # skipping the countries we have no data for
continue
trimmed_data = firstmarriage[key].dropna() # removes the null values from the series
country_longform = key[10:13]
trimmed_data.name = form_dictionary()[country_longform] # Name attribute is now human readable country, note that name goes below corresponding data
firstmarriage_trimmed[key] = trimmed_data # forming a new dict with the trimmed series
f = open('marriage', 'w')
f.write(str(firstmarriage_trimmed))
f.close()
# TODO: Remove extraneous Name, dtype entries from the dataset
Randomizing()
#############################################################################
#Getting More and More Data
import pandas as pd
import os
from Quandl import Quandl
import time
auth_tok = open("quandlekey.txt", "r").read()
data = Quandl.get("WIKI/KO", trim_start = "2000-12-12", trim_end = "2014-12-30", authtoken=auth_tok)
print(data)
#####################################################################
df = pd.DataFrame()
path = "E:\\IIT KHARAGPUR\\Semester II\\Machine Learning\\DataSciencePython\\Dataset\\yahoofinance\\_KeyStats"
def Stock_Prices():
statspath = path
stock_list = [x[0] for x in os.walk(statspath)]
print(stock_list)
