def scrape( filter ) :
""" Scrape all symbols from eoddata.com """
verbose = False
base_url = "http://eoddata.com/stocklist/XXX/YYY.htm"
alphabet = map(chr,range(65,91)) + map(str,range(0,10))
total = 0
exch = {}
for key,value in exchanges(True).items() :
if verbose : print "Exchange:",value,"Code:",key
symb = {}
for letter in alphabet :
url = base_url.replace('XXX',key).replace('YYY',letter)
if verbose : print value,key,letter,url
symb = dict(symb.items() + symbols(url,True).items())
exch[key] = symb
total = total + len(exch[key].items())
print "#:",len(exch),", Exchange:",value,", Code:",key,", Number of symbols:",len(exch[key])
print "Total number of symbols:",total
from utils import printPrettyDict
printPrettyDict(exch,40)
import pickle
pickle.dump( exch, open("symbols.pkl","w") )
return output_filename
def pretty( input_filename ) :
import pickle
input_file = open(input_filename,"r")
input_dict = pickle.load(input_file)
output_filename = input_filename.split('.',1)
output_filename = output_filename[0]+"_pretty.txt"
from utils import printPrettyDict
printPrettyDict( input_dict, 40, output_filename )
return output_filename
def exchanges( print_pretty = False ) :
""" Scrape list of exchanges from eoddata.com """
page = urllib.urlopen("http://eoddata.com/symbols.aspx")
soup = BeautifulSoup(page,"lxml")
exchanges = {}
for option in soup.find_all('option') :
exchanges[str(option['value'])] = str(option.string)
if print_pretty :
from utils import printPrettyDict
printPrettyDict( exchanges, 40 )
return exchanges
total = 0
exch = {}
for key,value in exchanges(True).items() :
if verbose : print "Exchange:",value,"Code:",key
symb = {}
for letter in alphabet :
url = base_url.replace('XXX',key).replace('YYY',letter)
if verbose : print value,key,letter,url
symb = dict(symb.items() + symbols(url,True).items())
exch[key] = symb
total = total + len(exch[key].items())
print "#:",len(exch),", Exchange:",value,", Code:",key,", Number of symbols:",len(exch[key])
print "Total number of symbols:",total
from utils import printPrettyDict
printPrettyDict(exch,40)
import pickle
pickle.dump( exch, open("ticker_symbols","w") )
elif option == 2 :
tmp = exchanges(True)
import pickle
from utils import printPrettyDict
file = open("ticker_symbols","r")
exch = pickle.load(file)
#printPrettyDict( exch, 40 )
total = 0