def stock_prices(): print "Begin..." df = pd.DataFrame() statspath = path + "/_KeyStats" stock_list = [x[0] for x in os.walk(statspath)] print stock_list[0] for each_dir in stock_list[1:]: try: ticker = each_dir.split("//")[0] print ticker name = "WIKI/" + ticker.upper() data = Quandl.get(name, trim_start="2000-12-12", trim_end="2014-12-12", authtoken=auth_tok) data[ticker.upper()] = data["Adj. Close"] df = pd.concat([df, data[ticker.upper()]], axis=1) except Exception as e: print str(e) try: ticker = each_dir.split("//")[0] print ticker name = "WIKI/" + ticker.upper() data = Quandl.get(name, trim_start="2000-12-12", trim_end="2014-12-12", authtoken=auth_tok) data[ticker.upper()] = data["Adj. Close"] df = pd.concat([df, data[ticker.upper()]], axis=1) except Exception as e: print str(e) df.to_csv("stock_prices.csv")
def Stock_Prices(): df = pd.DataFrame() statspath = path+'\\_KeyStats' stock_list = [x[0] for x in os.walk(statspath)] for each_dir in stock_list[1:]: try: ticker = each_dir.split("\\")[-1] print(ticker) name = "WIKI/"+ ticker.upper() data = Quandl.get(name, trim_start = "2000-12-12",trim_end = "2014-12-30", authtoken= auth_tok) data[ticker.upper()] = data["Adj. Close"] df= pd.concat([df, data[ticker.upper()]], axis = 1) except Exception as e: print(str(e)) time.sleep(10) try: ticker = each_dir.split("\\")[-1] print(ticker) name = "WIKI/"+ ticker.upper() data = Quandl.get(name, trim_start = "2000-12-12",trim_end = "2014-12-30", authtoken= auth_tok) data[ticker.upper()] = data["Adj. Close"] df= pd.concat([df, data[ticker.upper()]], axis = 1) except Exception as e: print(str(e)) df.to_csv("stock_prices.csv")
def Stock_Prices(): statspath = path stock_list = [x[0] for x in os.walk(statspath)] print(stock_list) for each_dir in stock_list[1:]: try: ticker = each_dir.split("\\")[8] print(ticker) name = "WIKI/" + ticker.upper() data = Quandl.get(name, trim_start="2000-12-12", trim_end="2014-12-30", authtoken=auth_tok) data[ticker.upper()] = data["Adj. Close"] df = pd.concat([df, data[ticker.upper()]], axis=1) except Exception as e: print(str(e)) time.sleep(10) try: ticker = each_dir.split("\\")[1] print(ticker) name = "WIKI/" + ticker.upper() data = Quandl.get(name, trim_start="2000-12-12", trim_end="2014-12-30", authtoken=auth_tok) data[ticker.upper()] = data["Adj. Close"] df = pd.concat([df, data[ticker.upper()]], axis=1) df.to_csv("stock_prices.csv") except Exception as e: print(str(e))
def Stock_Prices(): df = pd.DataFrame() statspath = path+'/_KeyStats' stock_list = [x[0] for x in os.walk(statspath)] # print(stock_list) for each_dir in stock_list[1:]: try: # ticker = each_dir.split("\\")[1] # Windows only # ticker = each_dir.split("/")[1] # this didn't work so do this: ticker = os.path.basename(os.path.normpath(each_dir)) # print(ticker) # uncomment to verify name = "WIKI/"+ticker.upper() print(name) data = Quandl.get( name, trim_start = "2000-12-12", trim_end = "2014-12-30", authtoken=auth_tok ) data[ticker.upper()] = data["Adj. Close"] df = pd.concat([df, data[ticker.upper()]], axis = 1) except Exception as e: # this except is just a simple retry... print(str(e)) time.sleep(10) try: # ticker = each_dir.split("\\")[1] # Windows only # ticker = each_dir.split("/")[1] # this didn't work on *nix so do this: ticker = os.path.basename(os.path.normpath(each_dir)) # print(ticker) # uncomment to verify name = "WIKI/"+ticker.upper() data = Quandl.get( name, trim_start = "2000-12-12", trim_end = "2014-12-30", authtoken=auth_tok ) data[ticker.upper()] = data["Adj. Close"] df = pd.concat([df, data[ticker.upper()]], axis = 1) except Exception as e: print(str(e)) df.to_csv("stock_prices.csv")
def index_data(): if request.method=='GET': return render_template('datarequest.html') else: # request was a POST app_data.vars['symbol']=request.form['stock_data'] auth_tok = "VJmYz-1u3Zf6P3ocrkUN" string1="WIKI/" string2=app_data.vars['symbol'] string3=string1+string2 data = Quandl.get(string3, trim_start = "2015-09-01", trim_end = "2015-12-01", authtoken=auth_tok) dateaxis = data.reset_index()['Date'] close_data=data['Close'] x=dateaxis y=close_data fig=plt.figure() axes=fig.add_subplot(1,1,1) axes.plot(x, y, marker='.') #labels= axes.set_xticklabels(rotation=60, fontsize='small') axes.tick_params(axis='x',labelsize=8) f=tempfile.NamedTemporaryFile(dir='static/',suffix='png',delete=False) plt.savefig(f) f.close() plotPng=f.name.split('/')[-1] return(render_template('figures.html',y=y, plotPng=plotPng))
def __init__(self,startDate): #create dictionary mapping ED contract to #, 1st, 2nd, 3rd, etc d = {1:"EDH2014", 2:"EDM2014", 3:"EDU2014", 4:"EDZ2014", 5:"EDH2015", 6:"EDM2015", 7:"EDU2015", 8:"EDZ2015", 9:"EDH2016", 10:"EDM2016", 11:"EDU2016", 12:"EDZ2016", 13:"EDH2017", 14:"EDM2017", 15:"EDU2017", 16:"EDZ2017", 17:"EDH2018", 18:"EDM2018", 19:"EDU2018", 20:"EDZ2018", 21:"EDH2019", 22:"EDM2019", 23:"EDU2019", 24:"EDZ2019", 25:"EDH2020", 26:"EDM2020", 27:"EDU2020", 28:"EDZ2020"} self.dates = {1:'2014-03-19',2:'2014-06-18',3:'2014-09-17',4:'2014-12-17', 5:'2015-03-18',6:'2015-06-17',7:'2015-09-16',8:'2015-12-16', 9:'2016-03-16',10:'2016-06-15',11:'2016-09-21',12:'2016-12-21', 13:'2017-03-15',14:'2017-06-21',15:'2017-09-20',16:'2017-12-20', 17:'2018-03-21',18:'2018-06-20',19:'2018-09-19',20:'2018-12-19', 21:'2019-03-20',22:'2019-06-19',23:'2019-09-18',24:'2019-12-18', 25:'2020-03-18',26:'2020-06-17',27:'2020-09-16',28:'2020-12-16'} self.startDate = startDate self.edDict = dict(d) ed1 = Quandl.get("CME/EDH2020", trim_start = self.startDate, authtoken = auth_tok) self.edRates = 100 - pandas.DataFrame(ed1["Settle"]) self.edRates.columns = ["EDH2020"] self.edVolume = pandas.DataFrame(ed1["Volume"]) self.edVolume.columns = ["EDH2020"] self.edOI = pandas.DataFrame(ed1["Open Interest"]) self.edOI.columns = ["EDH2020"]
def main(self): results = {} for symbol in self.symbols: code = self.generate_quandl_code(symbol) results[symbol] = Quandl.get(code, authtoken=self.base_config['api_key'], returns='pandas') results[symbol] = results[symbol].reset_index() return results
def graph(): stock = app.vars['stock'] stockq = "/".join(("WIKI", stock)) data = Quandl.get(stockq, rows=20, authtoken=auth_tok, returns="pandas") df = data[['Close']] p = figure(width=700, height=500, title=stock, x_axis_type='datetime') p.circle(x=df.index, y=df[['Close']]) script, div = components(p) return render_template('graph.html', script=script, div=div)
def Stock_Prices(): df = pd.DataFrame() statspath = path + '/_KeyStats' stock_list = [x[0] for x in os.walk(statspath)] # print(stock_list) for each_dir in stock_list[1:]: try: # ticker = each_dir.split("\\")[1] # Windows only # ticker = each_dir.split("/")[1] # this didn't work so do this: ticker = os.path.basename(os.path.normpath(each_dir)) # print(ticker) # uncomment to verify name = "WIKI/" + ticker.upper() print(name) data = Quandl.get(name, trim_start="2000-12-12", trim_end="2014-12-30", authtoken=auth_tok) data[ticker.upper()] = data["Adj. Close"] df = pd.concat([df, data[ticker.upper()]], axis=1) except Exception as e: # this except is just a simple retry... print(str(e)) time.sleep(10) try: # ticker = each_dir.split("\\")[1] # Windows only # ticker = each_dir.split("/")[1] # this didn't work on *nix so do this: ticker = os.path.basename(os.path.normpath(each_dir)) # print(ticker) # uncomment to verify name = "WIKI/" + ticker.upper() data = Quandl.get(name, trim_start="2000-12-12", trim_end="2014-12-30", authtoken=auth_tok) data[ticker.upper()] = data["Adj. Close"] df = pd.concat([df, data[ticker.upper()]], axis=1) except Exception as e: print(str(e)) df.to_csv("stock_prices.csv")
def loadAll(self): for i in range(4,29): label = "CME/" + self.edDict[i] temp = Quandl.get(label, trim_start = self.startDate, authtoken = auth_tok) self.edRates[self.edDict[i]] = 100 - temp["Settle"] self.edVolume[self.edDict[i]] = temp["Volume"] self.edOI[self.edDict[i]] = temp["Open Interest"] #issues with the data (even though its from the cme) - manual fix self.edRates["EDZ2017"]["2014-02-24"] = 3
def pullSwapData(self, inputDate): swapArray = numpy.zeros(9) swapArray[0] = Quandl.get("FRED/USD3MTD156N", trim_start = inputDate, trim_end = inputDate, authtoken = auth_tok)["VALUE"][0] swapArray[1] = fred.get_series('DSWP1', observation_start=inputDate, observation_end=inputDate)[0] swapArray[2] = fred.get_series('DSWP2', observation_start=inputDate, observation_end=inputDate)[0] swapArray[3] = fred.get_series('DSWP3', observation_start=inputDate, observation_end=inputDate)[0] swapArray[4] = fred.get_series('DSWP4', observation_start=inputDate, observation_end=inputDate)[0] swapArray[5] = fred.get_series('DSWP5', observation_start=inputDate, observation_end=inputDate)[0] swapArray[6] = fred.get_series('DSWP7', observation_start=inputDate, observation_end=inputDate)[0] swapArray[7] = fred.get_series('DSWP10', observation_start=inputDate, observation_end=inputDate)[0] swapArray[8] = fred.get_series('DSWP30', observation_start=inputDate, observation_end=inputDate)[0] return swapArray/100
def getHistoricalData(ticker): onlineData = Quandl.get("YAHOO/" + ticker, authtoken = authtoken, collapse='daily') localData = [] for i in dates: singleDayData = {'Date': i} for j in stockHeaders: try: singleDayData[j] = str(onlineData.loc[i, j]) except KeyError: # print "The date " + i + " has no data." break else: localData.append(singleDayData) return localData
def getIndicatorFromQuandl(indicator_name): resolution = 'monthly' print "Getting "+indicator_name+" indicator from Quandl" today = str(datetime.datetime.now().year)+"-"+str(datetime.datetime.now().month)+"-"+str(datetime.datetime.now().day) data = Quandl.get(indicator_name, authtoken='5UND8fCsD7oWNEs1fpfa', trim_start='1900-01-01', trim_end=today, collapse=resolution, transformation='none', returns='numpy') datapoints = [] for datapoint in data: dt = datapoint[0] datapoints.append({"name":indicator_name, "date":(dt.year, dt.month, dt.day), "value": datapoint[1] }) # insert multiple records to the table tblIndicators.insert_multiple(datapoints) # get name i.e. description of the indicator meta = searchQuandl(indicator_name); desc = meta[0]["name"] # insert new record into indicators index tblIndex.insert({"desc": desc, "name": indicator_name, "resolution": resolution, "count": len(datapoints)}) print "Sucessful insert of "+str(len(datapoints))+" records."+indicator_name return datapoints
def hichart_quandl(request): from Quandl import Quandl import json myAAPL_data = Quandl.get("WIKI/AAPL", returns="pandas", column="11", authtoken="L5A6rmU9FGvyss9F7Eym", trim_start='2006/06/15', trim_end='2007/06/15') data = json.loads(myAAPL_data.to_json()) # convert to JSON object... #### Below logic is quite imp as # this is transforming pandas dataframe to the High charts input structure data_list = list(sorted(data['Adj. Close'].items())) highcharts_data = [] for x in range(len(data_list)): dl = list(data_list[x]) dl[0] = int(dl[0]) highcharts_data.append(dl) ### This is important to note json.dumps() convert python data structure to JSON form return HttpResponse(json.dumps(highcharts_data), content_type='application/json')
def getCorporates(self, trim_start, trim_end, WORKING_DIR='.'): self.OIS = OIS(trim_start=trim_start, trim_end=trim_end, WORKING_DIR=WORKING_DIR) self.datesAll = self.OIS.datesAll self.OISData = self.OIS.getOIS() self.WORKING_DIR = WORKING_DIR for rating in self.ratings: index = 'ML/' + rating + 'TRI' try: corpSpreads = 1e-4 * (Quandl.get( index, authtoken="Lqsxas8ieaKqpztgYHxk", trim_start=trim_start, trim_end=trim_end)) corpSpreads.reset_index(level=0, inplace=True) corpSpreads = pd.merge(left=self.datesAll, right=corpSpreads, how='left') corpSpreads = corpSpreads.fillna(method='ffill').fillna( method='bfill') self.corpSpreads[rating] = corpSpreads.T.fillna( method='ffill').fillna(method='bfill').T except: print(index, " not found") self.corpSpreads = pd.Panel.from_dict(self.corpSpreads) self.corporates = {} self.OISData.drop('Date', axis=1, inplace=True) ntenors = np.shape(self.OISData)[1] for rating in self.ratings: try: tiledCorps = np.tile(self.corpSpreads[rating]['Value'], ntenors) tiledCorps = tiledCorps.reshape(np.shape(self.OISData), order="F") self.corporates[rating] = pd.DataFrame( data=(tiledCorps + self.OISData.values)) self.corporates[rating].drop( self.corporates[rating].columns[[0]], axis=1, inplace=True) except: print("Error in addition of Corp Spreads") self.corporates = pd.Panel(self.corporates) return self.corporates
def __init__(self, trim_start="2005-01-10", trim_end="2010-01-10", WORKING_DIR='.'): self.OIS = 0.01 * Quandl.get("USTREASURY/YIELD", authtoken="Lqsxas8ieaKqpztgYHxk", trim_start=trim_start, trim_end=trim_end) self.OIS.reset_index(level=0, inplace=True) self.datesAll = pd.DataFrame(pd.date_range(trim_start, trim_end), columns=['Date']) self.datesAll.reset_index(level=0, inplace=True) self.OIS = pd.merge(left=self.datesAll, right=self.OIS, how='left') self.OIS = self.OIS.fillna(method='ffill').fillna(method='bfill') self.OIS = self.OIS.T.fillna(method='ffill').fillna( method='bfill').T # Fill NA forward and backward to eliminate NA self.WORKING_DIR = WORKING_DIR self.OIS_dict = {} self.t_step = 1 / 365.0
def Stock_Prices(): df = pd.DataFrame() statspath = path + "/_KeyStats" stock_list = [x[0] for x in os.walk(statspath)] stock_list.sort() for each_dir in stock_list[1:]: try: ticker = each_dir.split('/')[6] print(ticker) name = "WIKI/" + ticker.upper() data = Quandl.get(name, trim_start="2000-12-12", trim_end="2014-12-30", authtoken=auth_tok) data[ticker.upper()] = data["Adj. Close"] df = pd.concat([df, data[ticker.upper()]], axis=1) except Exception as e: print(str(e)) #time.sleep(10) df.to_csv("Stock_Prices.csv")
def Stock_Prices(): df = pd.DataFrame() statspath = path+'Yahoo/intraQuarter/_KeyStats' stock_list = [x[0] for x in os.walk(statspath)] for each_dir in stock_list[1:]: try: ticker = each_dir.split('_KeyStats/')[1] print ticker name = 'WIKI/' + ticker.upper() data = Quandl.get(name, trim_start=datetime.strptime('2000-12-12', '%Y-%m-%d'), trim_end=date.today()-timedelta(1), authtoken=auth_token) data[ticker.upper()] = data['Adj. Close'] df = pd.concat([df, data[ticker.upper()]], axis=1) except Exception as e: print('Error polling Quandl: ' +str(e)) df.to_csv('stock_prices.csv')
def stock_prices(): df = pd.DataFrame() statspath = path + '/_KeyStats' stock_list = [x[0] for x in os.walk(statspath)] for each_dir in stock_list[1:]: try: ticker = each_dir.split("/Users/User/intraQuarter/_KeyStats/")[1] print(ticker) name = "WIKI/" + ticker.upper() # Query Quandl, using the standard format, e.g WIKI/AAPL. data = Quandl.get(name, trim_start="2000-12-12", trim_end="2014-12-30", authtoken=auth_tok) data[ticker.upper()] = data["Adj. Close"] df = pd.concat([df, data[ticker.upper()]], axis=1) except Exception as e: print(str(e)) df.to_csv("stock_prices.csv")
def Stock_Prices(): df = pd.DataFrame() statspath = path + 'Yahoo/intraQuarter/_KeyStats' stock_list = [x[0] for x in os.walk(statspath)] for each_dir in stock_list[1:]: try: ticker = each_dir.split('_KeyStats/')[1] print ticker name = 'WIKI/' + ticker.upper() data = Quandl.get(name, trim_start=datetime.strptime( '2000-12-12', '%Y-%m-%d'), trim_end=date.today() - timedelta(1), authtoken=auth_token) data[ticker.upper()] = data['Adj. Close'] df = pd.concat([df, data[ticker.upper()]], axis=1) except Exception as e: print('Error polling Quandl: ' + str(e)) df.to_csv('stock_prices.csv')
import pandas as pd from Quandl import Quandl import math, datetime, pickle import numpy as np from sklearn import preprocessing, cross_validation, svm from sklearn.linear_model import LinearRegression import matplotlib.pyplot as plt from matplotlib import style style.use('ggplot') df = Quandl.get('WIKI/GOOGL') df = df[['Adj. Open','Adj. High','Adj. Low','Adj. Close','Adj. Volume',]] df['HL_PCT'] = (df['Adj. High']-df['Adj. Low'])/df['Adj. Low'] *100.0 df['PCT_Change'] = (df['Adj. Close']-df['Adj. Open'])/df['Adj. Open'] *100.0 df=df[['Adj. Close','HL_PCT','PCT_Change','Adj. Volume']] forecast_col = 'Adj. Close' df.fillna(-99999, inplace=True) forecast_out = int(math.ceil(0.01*len(df))) df['label']=df[forecast_col].shift(-forecast_out) X = np.array(df.drop(['label'],1)) X = preprocessing.scale(X) X_lately = X[-forecast_out:] X = X[:-forecast_out]
import pandas as pd from Quandl import Quandl import math, datetime, pickle import numpy as np from sklearn import preprocessing, cross_validation, svm from sklearn.linear_model import LinearRegression import matplotlib.pyplot as plt from matplotlib import style style.use('ggplot') df = Quandl.get('WIKI/GOOGL') df = df[[ 'Adj. Open', 'Adj. High', 'Adj. Low', 'Adj. Close', 'Adj. Volume', ]] df['HL_PCT'] = (df['Adj. High'] - df['Adj. Low']) / df['Adj. Low'] * 100.0 df['PCT_Change'] = (df['Adj. Close'] - df['Adj. Open']) / df['Adj. Open'] * 100.0 df = df[['Adj. Close', 'HL_PCT', 'PCT_Change', 'Adj. Volume']] forecast_col = 'Adj. Close' df.fillna(-99999, inplace=True) forecast_out = int(math.ceil(0.01 * len(df)))
import pandas as pd import os from Quandl import Quandl import time # auth_tok = "your_auth_here" auth_tok = open("quandl_auth_tok.txt", "r").read() data = Quandl.get("WIKI/KO", trim_start="2000-12-12", trim_end="2014-12-30", authtoken=auth_tok) print(data)
Randomizing() ############################################################################# #Getting More and More Data import pandas as pd import os from Quandl import Quandl import time auth_tok = open("quandlekey.txt", "r").read() data = Quandl.get("WIKI/KO", trim_start = "2000-12-12", trim_end = "2014-12-30", authtoken=auth_tok) print(data) ##################################################################### df = pd.DataFrame() path = "E:\\IIT KHARAGPUR\\Semester II\\Machine Learning\\DataSciencePython\\Dataset\\yahoofinance\\_KeyStats" def Stock_Prices(): statspath = path stock_list = [x[0] for x in os.walk(statspath)] print(stock_list)
country_array.append(m) x = dict(country_array) return x for key in form_dictionary(): data_string = 'WORLDBANK/' + key + '_SP_DYN_SMAM_FE' # format of the data on Quandl database country_keys.append(data_string) ''' # Test data - Afghanistan, Albania, Algeria, Antigua and Barbada, Argentina, USA, Channel Islands (Channel Islands not in database) test_country_keys = ['WORLDBANK/AFG_SP_DYN_SMAM_FE', 'WORLDBANK/ALB_SP_DYN_SMAM_FE', 'WORLDBANK/DZA_SP_DYN_SMAM_FE', 'WORLDBANK/ATG_SP_DYN_SMAM_FE', 'WORLDBANK/ARG_SP_DYN_SMAM_FE', 'WORLDBANK/USA_SP_DYN_SMAM_FE', 'WORLDBANK/CHI_SP_DYN_SMAM_FE'] firstmarriage = Quandl.get(test_country_keys, authtoken=auth_token, order='desc') ''' firstmarriage = Quandl.get(country_keys, authtoken=auth_token, order='desc') firstmarriage_trimmed = {} # type is dict, no longer pandas DataFrame - DataFrame doesn't allow me to strip out null values, must be even matrix for key in firstmarriage: if key[-9:] == 'NOT FOUND': # skipping the countries we have no data for continue trimmed_data = firstmarriage[key].dropna() # removes the null values from the series country_longform = key[10:13] trimmed_data.name = form_dictionary()[country_longform] # Name attribute is now human readable country, note that name goes below corresponding data firstmarriage_trimmed[key] = trimmed_data # forming a new dict with the trimmed series f = open('marriage', 'w') f.write(str(firstmarriage_trimmed)) f.close() # TODO: Remove extraneous Name, dtype entries from the dataset