Exemplo n.º 1
0
def downloadSplitData():
    ip2host = dllib.getIpMap()
    syms = map(lambda n: n.lstrip('_').split('.')[0],
               os.listdir(dllib.EOD_DIR))
    for symbol in syms:
        ip2host = dllib.getIpMap()
        success = False
        for ip in ip2host:
            params = (ip, symbol)
            url = "http://%s/x?s=%s&a=0&b=1&c=1900&d=11&e=31&f=2099&g=v&y=0&z=99999" % params
            loc = urllib2.Request(url)
            loc.add_header('Accept-Encoding', 'gzip, deflate')
            loc.add_header('Host', ip2host[ip])
            opener = urllib2.build_opener()
            print 'requesting', url
            try:
                csv_txt = dllib.tryDecompress(opener.open(loc).read())
                open(
                    os.path.join(const.SPLIT_DIR,
                                 'divsplit_' + symbol + '.csv'),
                    'wb').write(csv_txt)
                success = True
                break
            except urllib2.HTTPError:
                continue
        if success:
            print 'success', symbol
            loadSplitData(csv_txt, symbol)
        else:
            print 'FAIL', symbol
Exemplo n.º 2
0
def downloadSplitData():
    ip2host = dllib.getIpMap()
    syms = map(lambda n: n.lstrip("_").split(".")[0], os.listdir(dllib.EOD_DIR))
    for symbol in syms:
        ip2host = dllib.getIpMap()
        success = False
        for ip in ip2host:
            params = (ip, symbol)
            url = "http://%s/x?s=%s&a=0&b=1&c=1900&d=11&e=31&f=2099&g=v&y=0&z=99999" % params
            loc = urllib2.Request(url)
            loc.add_header("Accept-Encoding", "gzip, deflate")
            loc.add_header("Host", ip2host[ip])
            opener = urllib2.build_opener()
            print "requesting", url
            try:
                csv_txt = dllib.tryDecompress(opener.open(loc).read())
                open(os.path.join(const.SPLIT_DIR, "divsplit_" + symbol + ".csv"), "wb").write(csv_txt)
                success = True
                break
            except urllib2.HTTPError:
                continue
        if success:
            print "success", symbol
            loadSplitData(csv_txt, symbol)
        else:
            print "FAIL", symbol
Exemplo n.º 3
0
def downloadEodData():
	symconn = sqlite3.connect(os.path.join(dllib.DB_DIR, "symbols.db"))
	symcurs = symconn.cursor()
	symselect = "select SYMBOL from nasdaqlisted union select NASDAQ_SYMBOL from otherlisted"
	db_syms = map(lambda t: '_' + t[0], symcurs.execute(symselect).fetchall())
	fs_syms = map(lambda n: n.split('.')[0], os.listdir(const.RAW_DIR))
	syms = filter(lambda i: i.isalpha(), map(lambda i: i.lstrip('_'), list(set(db_syms) - set(fs_syms))))
	for symbol in syms:
		ip2host = dllib.getIpMap()
		success = False
		for ip in ip2host:
			params = (ip, symbol)
			url = "http://%s/table.csv?s=%s&a=0&b=1&c=1900&d=11&e=31&f=2099&g=d" % params 
			loc = urllib2.Request(url)
			loc.add_header('Accept-Encoding', 'gzip, deflate')
			loc.add_header('Host', ip2host[ip])
			opener = urllib2.build_opener()
			print 'requesting', url
			try:
				csv_txt = dllib.tryDecompress(opener.open(loc).read())
				open(os.path.join(const.RAW_DIR, '_' + symbol + '.csv'), 'wb').write(csv_txt)
				success = True
				break
			except urllib2.HTTPError:
				continue
		if success:
			print 'success', symbol
			loadEodData(csv_txt, symbol)
		else:
			print 'FAIL', symbol
Exemplo n.º 4
0
def downloadEodData():
    symconn = sqlite3.connect(os.path.join(dllib.DB_DIR, "symbols.db"))
    symcurs = symconn.cursor()
    symselect = "select SYMBOL from nasdaqlisted union select NASDAQ_SYMBOL from otherlisted"
    db_syms = map(lambda t: '_' + t[0], symcurs.execute(symselect).fetchall())
    fs_syms = map(lambda n: n.split('.')[0], os.listdir(const.RAW_DIR))
    syms = filter(
        lambda i: i.isalpha(),
        map(lambda i: i.lstrip('_'), list(set(db_syms) - set(fs_syms))))
    for symbol in syms:
        ip2host = dllib.getIpMap()
        success = False
        for ip in ip2host:
            params = (ip, symbol)
            url = "http://%s/table.csv?s=%s&a=0&b=1&c=1900&d=11&e=31&f=2099&g=d" % params
            loc = urllib2.Request(url)
            loc.add_header('Accept-Encoding', 'gzip, deflate')
            loc.add_header('Host', ip2host[ip])
            opener = urllib2.build_opener()
            print 'requesting', url
            try:
                csv_txt = dllib.tryDecompress(opener.open(loc).read())
                open(os.path.join(const.RAW_DIR, '_' + symbol + '.csv'),
                     'wb').write(csv_txt)
                success = True
                break
            except urllib2.HTTPError:
                continue
        if success:
            print 'success', symbol
            loadEodData(csv_txt, symbol)
        else:
            print 'FAIL', symbol
Exemplo n.º 5
0
def updateEodData():
	txts = filter(lambda fn: fnmatch(fn, '*listed_*.txt'), os.listdir(const.SYM_DIR))
	
	nsdl = filter(lambda fn: fnmatch(fn, 'nasdaqlisted_*.txt'), txts)[0]
	allnsd = list(csv.DictReader(open(pjoin(const.SYM_DIR, nsdl), 'rb'), delimiter="|"))
	nsdsyms = filter(lambda s: s.isalpha(), map(lambda d: d['Symbol'], allnsd))
	
	othl = filter(lambda fn: fnmatch(fn, 'otherlisted_*.txt'), txts)[0]
	allother = list(csv.DictReader(open(pjoin(const.SYM_DIR, othl), 'rb'), delimiter="|"))
	othsyms = filter(lambda s: s.isalpha(), map(lambda d: d['ACT Symbol'], allother))
	
	txt_syms = nsdsyms + othsyms
	
	raw_syms = map(lambda n: n.split('.')[0][1:], os.listdir(const.RAW_DIR))
	syms = list(set(txt_syms + raw_syms))
	for sym in BAD_SYMS:
		syms.remove(sym)
	syms.sort()
	
	# # TEMPORARY TRUNCATION, REMOVE
# 	LAST = 'OHAI'
# 	syms = syms[syms.index(LAST) + 1 :]

	rrobin = {}
	for symbol in syms:
		mo, dy, yr, seq = getStartDate(symbol)
		ip2host = dllib.getIpMap()

		# Get target ip in round-robin fashion
		for ip in ip2host:
			if ip not in rrobin:
				rrobin[ip] = 0
				targetip = ip
			else:
				ld = [{count : ip} for ip, count in rrobin.items()]
				ld = filter(lambda d: d.values()[0] in ip2host, ld)
				ld.sort()
				targetip = ld[0].values()[0]

		params = (targetip, symbol, mo, dy, yr)
		url = "http://%s/table.csv?s=%s&a=%s&b=%s&c=%s&d=11&e=31&f=2099&g=d" % params
		loc = urllib2.Request(url)
		loc.add_header('Accept-Encoding', 'gzip, deflate')
		loc.add_header('Host', ip2host[targetip])
		opener = urllib2.build_opener()
		print 'requesting', url
		try:
			csv_txt = dllib.tryDecompress(opener.open(loc).read())
			if list(csv.DictReader(strio(csv_txt))):
				rrobin[targetip] += 1
				_name = '_' + symbol
				zname = _name + '.csv.zip'
				with ZipFile(pjoin(const.RAW_DIR, zname), 'a', ZIP_DEFLATED) as _zip:
					_zip.writestr(_name + '_' + seq + '.csv', csv_txt)
				print 'success', symbol
				mergeAndAdjust(symbol)
		except urllib2.HTTPError:
			print 'FAIL', symbol