def generate_eod_tasks():
'''
Task responsible for generating work items used to obtain end of day
data for stocks using get_eod_data() task
'''
db = MongoDBUtil()
symbol_sets = set()
#Gets all symbols
sp500 = finsymbols.get_sp500_symbols()
amex = finsymbols.get_amex_symbols()
nyse = finsymbols.get_nyse_symbols()
nasdaq = finsymbols.get_nasdaq_symbols()
#Adds all symbols to set which removes duplicates
symbol_sets.update(_get_symbol_set(sp500))
symbol_sets.update(_get_symbol_set(amex))
symbol_sets.update(_get_symbol_set(nyse))
symbol_sets.update(_get_symbol_set(nasdaq))
now = datetime.datetime.now()
end_date = '-'.join([str(now.year),str(now.month),str(now.day)])
his_symbols = db.has_historical_data(symbol_sets)
if(len(his_symbols) >= 1):
start_date = '1980-01-01'
hist_job = group(get_eod_data.s(symbol,start_date,end_date) for symbol in symbol_sets)
hist_job.apply_async()
# Obtain data for current date
job = group(get_eod_data.s(symbol,end_date,end_date) for symbol in symbol_sets)
job.apply_async()
def getData(startDate, endDate, period):
tickerSymbols = []
sp500 = finsymbols.get_sp500_symbols()
nasdaq = finsymbols.get_nasdaq_symbols()
amex = finsymbols.get_amex_symbols()
nyse = finsymbols.get_nyse_symbols()
for companies in amex:
tickerSymbols.append(companies.get("symbol"))
print tickerSymbols
return
# start a period before the startdate to obtain the data
print startDate, " ", endDate
startDate = startDate - timedelta(days=period)
print startDate
data = {}
for company in tickerSymbols:
print company
c = Share(company)
histData = c.get_historical(str(startDate), str(endDate))
histData = histData[::-1] # reverse data so its easier to use
data[company] = histData
addTools(data, period)
with open("Data/Amex_data.json", "w") as fp:
json.dump(data, fp, indent=2)
return data
def get_stock_symbols(): nyse=[company['symbol'] for company in ss.get_nyse_symbols() if '$' not in company['symbol']] amex=[company['symbol'] for company in ss.get_amex_symbols() if '$' not in company['symbol']] ndaq=[company['symbol'] for company in ss.get_nasdaq_symbols() if '$' not in company['symbol']] sp500=[company['symbol'] for company in ss.get_sp500_symbols() if '$' not in company['symbol']] tsx=get_tsx_companies() return sp500,amex,nyse,ndaq,tsx
def generate_eod_tasks():
'''
Task responsible for generating work items used to obtain end of day
data for stocks using get_eod_data() task
'''
db = MongoDBUtil()
symbol_sets = set()
#Gets all symbols
sp500 = finsymbols.get_sp500_symbols()
amex = finsymbols.get_amex_symbols()
nyse = finsymbols.get_nyse_symbols()
nasdaq = finsymbols.get_nasdaq_symbols()
#Adds all symbols to set which removes duplicates
symbol_sets.update(_get_symbol_set(sp500))
symbol_sets.update(_get_symbol_set(amex))
symbol_sets.update(_get_symbol_set(nyse))
symbol_sets.update(_get_symbol_set(nasdaq))
now = datetime.datetime.now()
end_date = '-'.join([str(now.year), str(now.month), str(now.day)])
his_symbols = db.has_historical_data(symbol_sets)
if (len(his_symbols) >= 1):
start_date = '1980-01-01'
hist_job = group(
get_eod_data.s(symbol, start_date, end_date)
for symbol in symbol_sets)
hist_job.apply_async()
# Obtain data for current date
job = group(
get_eod_data.s(symbol, end_date, end_date) for symbol in symbol_sets)
job.apply_async()
def get_all_symbols():
sp500 = finsymbols.get_sp500_symbols()
df_sp500=pd.DataFrame(sp500)
df_sp500['exchange']='SPY500'
amex_symbols=finsymbols.get_amex_symbols()
df_amex=pd.DataFrame(amex_symbols)
df_amex['exchange']='AMEX'
nyse_symbols=finsymbols.get_nyse_symbols()
df_nyse=pd.DataFrame(nyse_symbols)
df_nyse['exchange']='NYSE'
nasdaq_symbols=finsymbols.get_nasdaq_symbols()
df_nasdaq=pd.DataFrame(nasdaq_symbols)
df_nasdaq['exchange']='NASDAQ'
result=pd.concat([df_sp500,df_amex,df_nyse,df_nasdaq])
dbdao.save_dataframe(result, "df_symbol_new")
def createListOfStocks():
initialAmex = finsymbols.get_amex_symbols()
initialNasdaq = finsymbols.get_nasdaq_symbols()
initialNyse = finsymbols.get_nyse_symbols()
nyse = [stockInfo['symbol'].strip() for stockInfo in initialNyse]
nasdaq = [stockInfo['symbol'].strip() for stockInfo in initialNasdaq]
amex = [stockInfo['symbol'].strip() for stockInfo in initialAmex]
return nasdaq
def main():
# cum_label = 0
# cum_non_label = 0
# # result = [[140308, 2417065], [316270, 1792759], [476341, 1422235], [663469, 1221244], [833018, 999542], [1055281, 841267], [1351301, 711644], [1674877, 558758], [2157544, 392063], [1833046, 140792]]
# result = [[61261, 1037613], [136272, 766345], [206275, 609507], [285135, 523400], [358020, 428905], [453139, 360710], [578466, 304938], [716058, 240122], [922954, 169132], [783448, 60819]]
# l1 = map(lambda x : x[0], result)
# l2 = map(lambda x: x[1], result)
#
# total_lables = sum(l1)
# total_non_labels = sum(l2)
# print "total_labels", total_lables, " total_non_labels ", total_non_labels
# max_diff = 0.0
# for j in result:
# cum_label += j[0]
# cum_non_label += j[1]
# diff = np.abs(100.0 * float(cum_label) / total_lables - 100.0 * float(cum_non_label) / total_non_labels)
# print "cum_lables, " , j[0], ",", j[1], ",", cum_label, ",", cum_non_label, ",", 100.0 * float(j[0])/total_lables, ",", 100.0 * float(j[1])/total_lables, ",", 100.0 * float(cum_label) / total_lables, ",", 100.0 * float(cum_non_label) / total_non_labels
# if diff > max_diff:
# max_diff = diff
start_time = datetime.datetime(1990, 10, 1)
end_time = datetime.datetime(2016, 10, 8)
# all_stocks = finsymbols.get_sp500_symbols()
all_stocks = finsymbols.get_sp500_symbols()
all_stocks.append(finsymbols.get_nasdaq_symbols())
all_stocks.append(finsymbols.get_nyse_symbols())
logger.info("starting %s ", datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
num_rounds = 5
for j in range(num_rounds):
print 'start_round\t', j
logger.info('start round %s', j)
run_round(all_stocks, start_time, end_time, 1)
print 'end_round\t', j
print "the one end"
# print q.head()
# ys = ystockquote.get_all("%5EGSPC")
# print ys
historyRoute = True
lastDaysRoute = 2
stocks_to_buy_historic_low_header = ["Stock Symbol", "Start", "End", "Stock Lowest Ever", "Stock Current Price", "Buy"]
# stocks_to_buy_historic_low = []
# lowest_in_history_header_row = ["Stock Symbol", "Start", "End", "Stock Lowest Ever", "Stock Current Price", "Buy"]
stocks_to_buy_going_down = [["Stock Symbol", "Days", "Stock Previous Price", "Stock Current Price", "Buy"]]
# stocks_not_to_buy = []
amex = finsymbols.get_amex_symbols()
nyse = finsymbols.get_nyse_symbols()
nasdaq = finsymbols.get_nasdaq_symbols()
sp500 = finsymbols.get_sp500_symbols()
# Dow_Jones_Symbol = "%5EDJI"
# SP_500_Symbol = "%5EGSPC"
# Nasdaq_Symbol1 = "%5EIXIC"
# Nasdaq_Symbol2 = "%5ENDX"
def isCurrentLowest(stock):
# print type(stock.get_price())
current = float(stock.get_price())
lowest = lowestPriceEver(stock)
if cmp(current, lowest) < 0:
# info = stock.get_info()
# temp_stock_info.append(info)
# stocks_to_buy_historic_low.append(stock)
setupForCSV(stock, True, lowest)
print "Error : use -h to see all supported commands"
exit(0)
if exchange == 'all' or exchange == '-nyse':
nyse = finsymbols.get_nyse_symbols()
nyse_json = open('nyse.json', 'w')
nyse_json.write(json.dumps(nyse))
if exchange == 'all' or exchange == '-nasdaq':
nasdaq = finsymbols.get_nasdaq_symbols()
nasdaq_json = open('nasdaq.json', 'w')
nasdaq_json.write(json.dumps(nyse))
if exchange == 'all' or exchange == '-amex':
amex = finsymbols.get_amex_symbols()
amex_json = open('amex.json', 'w')
amex_json.write(json.dumps(nyse))
if exchange == 'all' or exchange == '-sp500':
raise
try:
os.makedirs("saved/")
except OSError, e:
if e.errno != errno.EEXIST:
raise
stock_ex = raw_input("Which stock market would you like to screen: ")
repull_data = raw_input("Pull stock tickers names(y/n): ")
if "y" in repull_data:
if "nyse" in stock_ex:
stock_list = fin.get_nyse_symbols()
if "sp500" in stock_ex:
stock_list = fin.get_sp500_symbols()
if "nasdaq" in stock_ex:
stock_list = fin.get_nasdaq_symbols()
if "amex" in stock_ex:
stock_list = fin.get_amex_symbols()
ticker_list = []
for stock_n in stock_list:
ticker_list.append(str(stock_n["symbol"]))
target = open(stock_ex + "_list.txt", "w")
for ticker in ticker_list:
target.write(ticker + "\n")
target.close()
if "n" in repull_data:
ticker_list = []
target = open(stock_ex + "_list.txt", "r")
pull_list = target.readlines()
target.close()
for pulled in pull_list:
#! /usr/bin/env python
import os
import sys
import json
import finsymbols as data
if __name__ == '__main__':
#a =data.get_sp500_symbols()
a = data.get_amex_symbols()
b = data.get_nyse_symbols()
c = data.get_nasdaq_symbols()
a +=b+c
print(type(a), len(a))
with open('ticker.json', 'w+') as outfile:
json.dump(a, outfile)
with open('ticker.txt', 'w+') as f:
for line in a:
f.write(str(line)+'\n')