def import_kabuka(startd="", endd=""): if startd == "": startd = dt.today() else: startd = dt.strptime(dtf.datestr2fdate(startd), "%Y-%m-%d") if endd == "": endd = dt.today() else: endd = dt.strptime(dtf.datestr2fdate(endd), "%Y-%m-%d") pdate = startd while pdate <= endd: w = pdate.isoweekday() if w == 6 or w == 7: pdate = pdate + datetime.timedelta(days=1) continue import2kdb(dtf.date2str(pdate)) pdate = pdate + datetime.timedelta(days=1)
def import2kdb(_datestr): ddate = dtf.datestr2fdate(_datestr) cnt = 0 while cnt < RETRY_CNT: try: response = urllib2.urlopen("http://k-db.com/stocks/" + ddate + "?download=csv") html = response.read() f = StringIO.StringIO(html) reader = csv.reader(f, delimiter=',') break except Exception as e: print "date:%s type:%s args:%s e:%s" % (_datestr, str(type(e)), str(e.args), str(e)) cnt += 1 if cnt >= RETRY_CNT: return strsql = "" i = 0 for row in reader: if i == 0: i += 1 continue #d = row[0] #d = d[0:4] + d[6:8] + d[10:12] #if d != _datestr: # i = i + 1 # return code = row[CODE_POS] if i == 1: if len(code) != 6: return print _datestr tmp = code.split("-") if len(tmp) == 2 and tmp[1] == "T": #Use only Tosho codes code = code.split("-")[0] else: i = i + 1 continue openv = row[OPEN_POS] highv = row[HIGH_POS] lowv = row[LOW_POS] closev = row[CLOSE_POS] volv = row[VOL_POS] amtv = row[AMT_POS] if openv == "-" or highv == "-" or lowv == "-" or closev == "-" or volv == "-" or amtv == "-": continue if openv == "" or highv == "" or lowv == "" or closev == "" or volv == "" or amtv == "": continue if openv <= 0 or highv <= 0 or lowv <= 0 or closev <= 0 or volv <= 0 or amtv <= 0: continue strsql = strsql + "REPLACE INTO trade.kabuka VALUES(\'" strsql = strsql + row[CODE_POS].split("-")[0] + "\'" strsql = strsql + ", \'" + _datestr + "\'" strsql = strsql + ", \'" + str(openv) + "\'" strsql = strsql + ", \'" + str(highv) + "\'" strsql = strsql + ", \'" + str(lowv) + "\'" strsql = strsql + ", \'" + str(closev) + "\'" strsql = strsql + ", \'" + str(volv) + "\'" strsql = strsql + ", \'" + str(amtv) + "\'" strsql = strsql + ");\n" i = i + 1 if strsql != "": sql.exec_updsql(strsql)