def downloadSplitData():
ip2host = dllib.getIpMap()
syms = map(lambda n: n.lstrip('_').split('.')[0],
os.listdir(dllib.EOD_DIR))
for symbol in syms:
ip2host = dllib.getIpMap()
success = False
for ip in ip2host:
params = (ip, symbol)
url = "http://%s/x?s=%s&a=0&b=1&c=1900&d=11&e=31&f=2099&g=v&y=0&z=99999" % params
loc = urllib2.Request(url)
loc.add_header('Accept-Encoding', 'gzip, deflate')
loc.add_header('Host', ip2host[ip])
opener = urllib2.build_opener()
print 'requesting', url
try:
csv_txt = dllib.tryDecompress(opener.open(loc).read())
open(
os.path.join(const.SPLIT_DIR,
'divsplit_' + symbol + '.csv'),
'wb').write(csv_txt)
success = True
break
except urllib2.HTTPError:
continue
if success:
print 'success', symbol
loadSplitData(csv_txt, symbol)
else:
print 'FAIL', symbol
def downloadEodData():
symconn = sqlite3.connect(os.path.join(dllib.DB_DIR, "symbols.db"))
symcurs = symconn.cursor()
symselect = "select SYMBOL from nasdaqlisted union select NASDAQ_SYMBOL from otherlisted"
db_syms = map(lambda t: '_' + t[0], symcurs.execute(symselect).fetchall())
fs_syms = map(lambda n: n.split('.')[0], os.listdir(const.RAW_DIR))
syms = filter(lambda i: i.isalpha(), map(lambda i: i.lstrip('_'), list(set(db_syms) - set(fs_syms))))
for symbol in syms:
ip2host = dllib.getIpMap()
success = False
for ip in ip2host:
params = (ip, symbol)
url = "http://%s/table.csv?s=%s&a=0&b=1&c=1900&d=11&e=31&f=2099&g=d" % params
loc = urllib2.Request(url)
loc.add_header('Accept-Encoding', 'gzip, deflate')
loc.add_header('Host', ip2host[ip])
opener = urllib2.build_opener()
print 'requesting', url
try:
csv_txt = dllib.tryDecompress(opener.open(loc).read())
open(os.path.join(const.RAW_DIR, '_' + symbol + '.csv'), 'wb').write(csv_txt)
success = True
break
except urllib2.HTTPError:
continue
if success:
print 'success', symbol
loadEodData(csv_txt, symbol)
else:
print 'FAIL', symbol
def downloadSplitData():
ip2host = dllib.getIpMap()
syms = map(lambda n: n.lstrip("_").split(".")[0], os.listdir(dllib.EOD_DIR))
for symbol in syms:
ip2host = dllib.getIpMap()
success = False
for ip in ip2host:
params = (ip, symbol)
url = "http://%s/x?s=%s&a=0&b=1&c=1900&d=11&e=31&f=2099&g=v&y=0&z=99999" % params
loc = urllib2.Request(url)
loc.add_header("Accept-Encoding", "gzip, deflate")
loc.add_header("Host", ip2host[ip])
opener = urllib2.build_opener()
print "requesting", url
try:
csv_txt = dllib.tryDecompress(opener.open(loc).read())
open(os.path.join(const.SPLIT_DIR, "divsplit_" + symbol + ".csv"), "wb").write(csv_txt)
success = True
break
except urllib2.HTTPError:
continue
if success:
print "success", symbol
loadSplitData(csv_txt, symbol)
else:
print "FAIL", symbol
def downloadEodData():
symconn = sqlite3.connect(os.path.join(dllib.DB_DIR, "symbols.db"))
symcurs = symconn.cursor()
symselect = "select SYMBOL from nasdaqlisted union select NASDAQ_SYMBOL from otherlisted"
db_syms = map(lambda t: '_' + t[0], symcurs.execute(symselect).fetchall())
fs_syms = map(lambda n: n.split('.')[0], os.listdir(const.RAW_DIR))
syms = filter(
lambda i: i.isalpha(),
map(lambda i: i.lstrip('_'), list(set(db_syms) - set(fs_syms))))
for symbol in syms:
ip2host = dllib.getIpMap()
success = False
for ip in ip2host:
params = (ip, symbol)
url = "http://%s/table.csv?s=%s&a=0&b=1&c=1900&d=11&e=31&f=2099&g=d" % params
loc = urllib2.Request(url)
loc.add_header('Accept-Encoding', 'gzip, deflate')
loc.add_header('Host', ip2host[ip])
opener = urllib2.build_opener()
print 'requesting', url
try:
csv_txt = dllib.tryDecompress(opener.open(loc).read())
open(os.path.join(const.RAW_DIR, '_' + symbol + '.csv'),
'wb').write(csv_txt)
success = True
break
except urllib2.HTTPError:
continue
if success:
print 'success', symbol
loadEodData(csv_txt, symbol)
else:
print 'FAIL', symbol
def updateEodData():
txts = filter(lambda fn: fnmatch(fn, '*listed_*.txt'), os.listdir(const.SYM_DIR))
nsdl = filter(lambda fn: fnmatch(fn, 'nasdaqlisted_*.txt'), txts)[0]
allnsd = list(csv.DictReader(open(pjoin(const.SYM_DIR, nsdl), 'rb'), delimiter="|"))
nsdsyms = filter(lambda s: s.isalpha(), map(lambda d: d['Symbol'], allnsd))
othl = filter(lambda fn: fnmatch(fn, 'otherlisted_*.txt'), txts)[0]
allother = list(csv.DictReader(open(pjoin(const.SYM_DIR, othl), 'rb'), delimiter="|"))
othsyms = filter(lambda s: s.isalpha(), map(lambda d: d['ACT Symbol'], allother))
txt_syms = nsdsyms + othsyms
raw_syms = map(lambda n: n.split('.')[0][1:], os.listdir(const.RAW_DIR))
syms = list(set(txt_syms + raw_syms))
for sym in BAD_SYMS:
syms.remove(sym)
syms.sort()
# # TEMPORARY TRUNCATION, REMOVE
# LAST = 'OHAI'
# syms = syms[syms.index(LAST) + 1 :]
rrobin = {}
for symbol in syms:
mo, dy, yr, seq = getStartDate(symbol)
ip2host = dllib.getIpMap()
# Get target ip in round-robin fashion
for ip in ip2host:
if ip not in rrobin:
rrobin[ip] = 0
targetip = ip
else:
ld = [{count : ip} for ip, count in rrobin.items()]
ld = filter(lambda d: d.values()[0] in ip2host, ld)
ld.sort()
targetip = ld[0].values()[0]
params = (targetip, symbol, mo, dy, yr)
url = "http://%s/table.csv?s=%s&a=%s&b=%s&c=%s&d=11&e=31&f=2099&g=d" % params
loc = urllib2.Request(url)
loc.add_header('Accept-Encoding', 'gzip, deflate')
loc.add_header('Host', ip2host[targetip])
opener = urllib2.build_opener()
print 'requesting', url
try:
csv_txt = dllib.tryDecompress(opener.open(loc).read())
if list(csv.DictReader(strio(csv_txt))):
rrobin[targetip] += 1
_name = '_' + symbol
zname = _name + '.csv.zip'
with ZipFile(pjoin(const.RAW_DIR, zname), 'a', ZIP_DEFLATED) as _zip:
_zip.writestr(_name + '_' + seq + '.csv', csv_txt)
print 'success', symbol
mergeAndAdjust(symbol)
except urllib2.HTTPError:
print 'FAIL', symbol
