def test_string_output(self):
sp500 = symbols.get_sp500_symbols()
company = sp500[0]
assert isinstance(company['company'],
string_types) == True, 'Company dict: {}.\
Expected output to be string'.format(company)
def get_symbols():
sp_500 = symbols.get_sp500_symbols()
symbolList = list()
for e in sp_500:
symbolList.append(e.get("symbol"))
return symbolList
def get_sp500_tickers():
if len(Ticker._sp500_tickers) > 0:
# print len(Ticker._sp500_tickers)
return Ticker._sp500_tickers
sp500 = symbols.get_sp500_symbols()
# print sp500
Ticker._sp500_tickers = [e['symbol'] for e in sp500]
# print Ticker._sp500_tickers
return Ticker._sp500_tickers
async def run_IEX():
from finsymbols import symbols
#create or connect to existing db
db = sqlite3.connect('stock_tables.db')
# Get the symbol list from wiki
all_symbols = symbols.get_sp500_symbols()
# Longer lists of symbols are available
# all_symbols = symbols.get_nyse_symbols() 3,139
# all_symbols = symbols.get_nasdaq_symbols() 3,295
# all_symbols = symbols.get_amex_symbols() 315
# Google and Fox have dual class shares so drop superfluous symbols
drop_symbols = ['GOOG', 'FOX']
good_symbols = [
x['symbol'] for x in all_symbols
if x['symbol'] not in set(drop_symbols)
]
# good_symbols = good_symbols[480:] # Turn this on for testing...only runs last ~20 tickers
# Save symbols and their data to db
symbol_frame = pd.DataFrame(all_symbols)
symbol_frame.to_sql('Names', db)
executor = concurrent.futures.ThreadPoolExecutor(max_workers=50)
futures = [
loop.run_in_executor(executor, get_IEX, i) for i in good_symbols
]
results = await asyncio.gather(*futures)
for (i, result) in zip(good_symbols, results):
p = result.iloc[-1]['close']
last = result.index.values[-1]
first = result.index.values[0]
#print(results)
print('{0}: {1} close = {2} - First day available {3}'.format(
last, i, p, first))
# Add return columns
# https://stackoverflow.com/questions/31287552/logarithmic-returns-in-pandas-dataframe
result['pct_change'] = result.close.pct_change()
result['log_return'] = np.log(1 + result['pct_change'])
# Add ATR to the frame too
result = add_atr_to_dataframe(result)
# Save to table with symbol as the name
result.to_sql(i, db)
# Commit DB
db.commit()
# Close the DB connection
db.close()
def create_symbol_lists():
sp500 = sym.get_sp500_symbols()
nyse = sym.get_nyse_symbols()
amex = sym.get_amex_symbols()
nasdaq = sym.get_nasdaq_symbols()
Df_sp500 = pd.DataFrame(sp500)
# Df_nyse = pd.DataFrame(nyse)
# Df_amex = pd.DataFrame(amex)
# Df_nasdaq = pd.DataFrame(nasdaq)
start = ""
with open('/root/jTWSdump_707/requests/base.txt', 'r') as f:
start = f.read()
# start += "\n"
f.close()
s_sp500 = ""
for symbol in Df_sp500.symbol:
if "^" in symbol or "." in symbol:
continue
#s_sp500 += '"'+symbol.strip()+'" "STK" "SMART" "" "" "" "USD" "" "10 D" "1 min" "1" "TRADES" "10" ""'+'\n'
s_sp500 += '"' + symbol.strip(
) + '" "STK" "SMART" "" "" "" "USD" "" "5 Y" "1 day" "1" "TRADES" "1" ""' + '\n'
s_nyse = ""
# for symbol in Df_nyse.symbol:
# if "^" in symbol or "." in symbol:
# continue
# # s_nyse += '"'+symbol.strip()+'" "STK" "SMART/NYSE" "" "" "" "USD" "" "10 D" "1 min" "1" "TRADES" "10" ""'+'\n'
# s_nyse += '"'+symbol.strip()+'" "STK" "SMART/NYSE" "" "" "" "USD" "" "5 Y" "1 day" "1" "TRADES" "1" ""'+'\n'
#
s_amex = ""
# for symbol in Df_amex.symbol:
# if "^" in symbol or "." in symbol:
# continue
# #s_amex += '"'+symbol.strip()+'" "STK" "SMART" "" "" "" "USD" "" "10 D" "1 min" "1" "TRADES" "10" ""'+'\n'
# s_amex += '"'+symbol.strip()+'" "STK" "SMART" "" "" "" "USD" "" "5 Y" "1 day" "1" "TRADES" "1" ""'+'\n'
#
s_nasdaq = ""
#
# for symbol in Df_nasdaq.symbol:
# if "^" in symbol or "." in symbol:
# continue
# #s_nasdaq += '"'+symbol.strip()+'" "STK" "SMART/NASDAQ" "" "" "" "USD" "" "10 D" "1 min" "1" "TRADES" "10" ""'+'\n'
# s_nasdaq += '"'+symbol.strip()+'" "STK" "SMART/NASDAQ" "" "" "" "USD" "" "5 Y" "1 day" "1" "TRADES" "1" ""'+'\n'
with open('/root/jTWSdump_707/requests/zipline.txt', 'w+') as f:
f.write(start)
f.write(s_sp500)
f.write(s_nyse)
f.write(s_amex)
f.write(s_nasdaq)
f.close()
def getSymbols(self):
idxs = self.cfg["general"]["idxs"].split(",")
symbols_list = []
sectors_map = {}
if "sp500" in idxs:
symbols_list += symbols.get_sp500_symbols()
if "nyse" in idxs:
symbols_list += symbols.get_nyse_symbols()
if "nasdaq" in idxs:
symbols_list += symbols.get_nasdaq_symbols()
if "amex" in idxs:
symbols_list += symbols.get_amex_symbols()
for s in symbols_list:
sectors_map[s["symbol"]] = s["sector"]
with open(self.pwd + "cfg/tickers.json", "w") as f:
json.dump(sectors_map, f)
import imp
import os
import datetime
import json
import pandas_datareader as web
import pandas_montecarlo
from finsymbols import symbols
import random
import matplotlib.pyplot as plt
%matplotlib inline
import quandl
#quandl.ApiConfig.api_key = "YbsGkVvwF-nJbuf7cbC4"
quandl.ApiConfig.api_key = "natezkiWArzE7_mz-oc5"
#------------------------------------------------------------------------------
#Scrape Symbols
sp500_JSON = symbols.get_sp500_symbols()
sp500_DF = pd.read_json(json.dumps(sp500_JSON))
sp500_list = list(sp500_DF['symbol'])
sp500 = sorted([str(i).replace('.','_') for i in sp500_list])
#------------------------------------------------------------------------------
start='2012-01-01'
end= '2018-04-13'
#------------------------------------------------------------------------------
#SP500 Constituent Returns
sp500_test=sp500[0:250]
data1 = quandl.get_table('WIKI/PRICES', ticker = sp500_test,
qopts = { 'columns': ['ticker','adj_close'] },
date = { 'gte': start, 'lte': end},
def test_sp500_size(self):
sp500 = symbols.get_sp500_symbols()
assert len(sp500) == 505, 'len gathered data: {}.\
Expected len: 504'.format(len(sp500))
def sp500():
return symbols.get_sp500_symbols()
# In[16]:
intrinio_get_company_financials_csv("GE", "2006", "Q3")
# In[5]:
import sys
import json
import finsymbols
sys.path.append('/home/skillachie/Desktop/')
from finsymbols import symbols
sp500 = symbols.get_sp500_symbols()
companies = json.dumps(sp500, indent=1, sort_keys=True)
# In[6]:
# import os
# count = 0
# def get_SandP_metadata():
# if not os.path.exists("data"):
# os.makedirs("data")
# for entry in sp500:
# datafileName = "./data/data_" + str(entry["symbol"])
# with open(datafileName, "w") as outfile:
# data = intrinio_get_company_metadata(entry["symbol"])
# json.dump(data, outfile, indent=1, sort_keys=True)
# count += 1
def test_string_output(self):
sp500 = symbols.get_sp500_symbols()
company = sp500[0]
assert isinstance(company['company'], string_types) == True, 'Company dict: {}.\
Expected output to be string'.format(company)
def test_sp500_size(self):
sp500 = symbols.get_sp500_symbols()
assert len(sp500) == 504, 'len gathered data: {}.\
Expected len: 504'.format(len(sp500))
