Exemple #1
0
def scrape( filter ) :

    """ Scrape all symbols from eoddata.com """
    
    verbose = False
    base_url = "http://eoddata.com/stocklist/XXX/YYY.htm"
    alphabet = map(chr,range(65,91)) + map(str,range(0,10))
    
    total = 0
    exch = {}
    for key,value in exchanges(True).items() :
        if verbose : print "Exchange:",value,"Code:",key
        symb = {}
        for letter in alphabet :
            url = base_url.replace('XXX',key).replace('YYY',letter)
            if verbose : print value,key,letter,url
            symb = dict(symb.items() + symbols(url,True).items())
        exch[key] = symb
        total = total + len(exch[key].items())
        print "#:",len(exch),", Exchange:",value,", Code:",key,", Number of symbols:",len(exch[key])
    print "Total number of symbols:",total
            
    from utils import printPrettyDict
    printPrettyDict(exch,40)
    
    import pickle
    pickle.dump( exch, open("symbols.pkl","w") )

    return output_filename
Exemple #2
0
def pretty( input_filename ) :

    import pickle
    input_file = open(input_filename,"r")
    input_dict = pickle.load(input_file)

    output_filename = input_filename.split('.',1)
    output_filename = output_filename[0]+"_pretty.txt"

    from utils import printPrettyDict
    printPrettyDict( input_dict, 40, output_filename )

    return output_filename
Exemple #3
0
def exchanges( print_pretty = False ) :

    """ Scrape list of exchanges from eoddata.com """

    page = urllib.urlopen("http://eoddata.com/symbols.aspx")
    soup = BeautifulSoup(page,"lxml")
    exchanges = {}
    for option in soup.find_all('option') :
        exchanges[str(option['value'])] = str(option.string)

    if print_pretty :
        from utils import printPrettyDict
        printPrettyDict( exchanges, 40 )

    return exchanges
Exemple #4
0
    total = 0
    exch = {}
    for key,value in exchanges(True).items() :
        if verbose : print "Exchange:",value,"Code:",key
        symb = {}
        for letter in alphabet :
            url = base_url.replace('XXX',key).replace('YYY',letter)
            if verbose : print value,key,letter,url
            symb = dict(symb.items() + symbols(url,True).items())
        exch[key] = symb
        total = total + len(exch[key].items())
        print "#:",len(exch),", Exchange:",value,", Code:",key,", Number of symbols:",len(exch[key])
    print "Total number of symbols:",total
            
    from utils import printPrettyDict
    printPrettyDict(exch,40)
    
    import pickle
    pickle.dump( exch, open("ticker_symbols","w") )

elif option == 2 : 

    tmp = exchanges(True)

    import pickle
    from utils import printPrettyDict
    file = open("ticker_symbols","r")
    exch = pickle.load(file)
    #printPrettyDict( exch, 40 )

    total = 0