def crawlSecurityData_AtFront(daysCount): securities = fd.get_all_securities() min_date = dao.select("select min(date) min_date from security_data", ())[0]['min_date'] if min_date is not None: endDate = fd.preOpenDate(min_date, 1) else: endDate = fd.getLastestOpenDate() startDate = fd.preOpenDate(endDate, daysCount) if startDate > endDate: return for code in securities: count = 0 df = ts.get_k_data(code, startDate, endDate) arr_values = [] while count < df.index.__len__(): open = str(df['open'].values[count]) close = str(df['close'].values[count]) high = str(df['high'].values[count]) low = str(df['low'].values[count]) volume = str(df['volume'].values[count]) date = str(df['date'].values[count]) arr_values.append((code, date, open, close, high, low, volume)) count = count + 1 print("Date: " + date + " Code: " + code) #dao.update("delete from security_data where code=%s", (code,)) dao.updatemany( "insert into security_data(code, date, open, close, high, low, volume) values(%s,%s,%s,%s,%s,%s,%s)", arr_values)
def crawlSecurityData_AtRear(dayCount): securities = fd.get_all_securities() max_date = dao.select("select max(date) max_date from security_data", ())[0]['max_date'] endDate = fd.getLastestOpenDate() if max_date is not None: startDate = fd.nextOpenDate(max_date, 1) else: startDate = fd.preOpenDate(endDate, dayCount) isFromHist = False if startDate > endDate: return for code in securities: count = 0 df = ts.get_k_data(code, startDate, endDate) if df.index.__len__() == 0: isFromHist = True df = ts.get_hist_data(code, startDate, endDate) arr_values = [] while count < df.index.__len__(): open = str(df['open'].values[count]) close = str(df['close'].values[count]) high = str(df['high'].values[count]) low = str(df['low'].values[count]) volume = str(df['volume'].values[count]) if isFromHist is True: date = df.index[count] else: date = str(df['date'].values[count]) arr_values.append((code, date, open, close, high, low, volume)) count = count + 1 print("Date: " + date + " Code: " + code) dao.update("delete from security_data where code=%s and date=%s", (code, date)) dao.updatemany( "insert into security_data(code, date, open, close, high, low, volume) values(%s,%s,%s,%s,%s,%s,%s)", arr_values)