def tdx_import_min_data_from_file(connect, filename, h5file, market, stock_record): """从通达信盘后数据导入1分钟或5分钟K线 :param connect : sqlite3连接实例 :param filename: 通达信K线数据文件名 :param h5file : HDF5 file :param stock_record: 股票的相关数据 (stockid, marketid, code, valid, type) :return: 导入的记录数 """ add_record_count = 0 if not os.path.exists(filename): return add_record_count stockid, marketid, code, valid, stktype = stock_record[0], stock_record[1], stock_record[2], stock_record[3],stock_record[4] table = get_h5table(h5file, market, code) if table.nrows > 0: lastdatetime = table[-1]['datetime'] else: lastdatetime = None row = table.row with open(filename, 'rb') as src_file: def trans_date(yymm, hhmm): tmp_date = yymm >> 11 remainder = yymm & 0x7ff year = tmp_date + 2004 month = remainder // 100 day = remainder % 100 hh = hhmm // 60 mm = hhmm % 60 return year * 100000000 + month * 1000000 + day * 10000 + hh * 100 + mm def get_date(pos): src_file.seek(pos * 32, SEEK_SET) data = src_file.read(4) a = struct.unpack('hh', data) return trans_date(a[0], a[1]) def find_pos(): src_file.seek(0, SEEK_END) pos = src_file.tell() total = pos // 32 if lastdatetime is None: return total, 0 low, high = 0, total - 1 mid = high // 2 while mid <= high: cur_date = get_date(low) if cur_date > lastdatetime: mid = low break cur_date = get_date(high) if cur_date <= lastdatetime: mid = high + 1 break cur_date = get_date(mid) if cur_date <= lastdatetime: low = mid + 1 else: high = mid - 1 mid = (low + high) // 2 return total, mid file_total, pos = find_pos() if pos < file_total: src_file.seek(pos * 32, SEEK_SET) data = src_file.read(32) while data: record = struct.unpack('hhfffffii', data) if 0 not in record[2:6]: if record[3] >= record[2] >= record[4] \ and record[3] >= record[5] >= record[4]: row['datetime'] = trans_date(record[0], record[1]) row['openPrice'] = record[2] * 1000 row['highPrice'] = record[3] * 1000 row['lowPrice'] = record[4] * 1000 row['closePrice'] = record[5] * 1000 row['transAmount'] = round(record[6] * 0.001) if stktype == 2: # 指数 row['transCount'] = record[7] else: row['transCount'] = round(record[6] * 0.01) row.append() add_record_count += 1 data = src_file.read(32) if add_record_count > 0: table.flush() return add_record_count
def import_one_stock_data(connect, api, h5file, market, ktype, stock_record, startDate=199012191500): market = market.upper() pytdx_market = to_pytdx_market(market) stockid, marketid, code, valid, stktype = stock_record[0], stock_record[1], stock_record[2], stock_record[3], \ stock_record[4] table = get_h5table(h5file, market, code) last_datetime = table[-1]['datetime'] if table.nrows > 0 else startDate today = datetime.date.today() if ktype == 'DAY': n, step = guess_day_n_step(last_datetime) pytdx_kline_type = TDXParams.KLINE_TYPE_RI_K today_datetime = (today.year * 10000 + today.month * 100 + today.day) * 10000 elif ktype == '1MIN': n, step = guess_1min_n_step(last_datetime) pytdx_kline_type = TDXParams.KLINE_TYPE_1MIN today_datetime = (today.year * 10000 + today.month * 100 + today.day) * 10000 + 1500 elif ktype == '5MIN': n, step = guess_5min_n_step(last_datetime) pytdx_kline_type = TDXParams.KLINE_TYPE_5MIN today_datetime = (today.year * 10000 + today.month * 100 + today.day) * 10000 + 1500 else: return 0 if today_datetime <= last_datetime: return 0 get_bars = api.get_index_bars if stktype == STOCKTYPE.INDEX else api.get_security_bars add_record_count = 0 row = table.row while n >= 0: bar_list = get_bars(pytdx_kline_type, pytdx_market, code, n * 800, step) n -= 1 if bar_list is None: #print(code, "invalid!!") continue for bar in bar_list: try: tmp = datetime.date(bar['year'], bar['month'], bar['day']) bar_datetime = (tmp.year * 10000 + tmp.month * 100 + tmp.day) * 10000 if ktype != 'DAY': bar_datetime += bar['hour'] * 100 + bar['minute'] except: continue if today_datetime >= bar_datetime > last_datetime \ and bar['high'] >= bar['open'] >= bar['low'] > 0 \ and bar['high'] >= bar['close'] >= bar['low'] > 0 \ and bar['vol'] != 0 and bar['amount'] != 0: row['datetime'] = bar_datetime row['openPrice'] = bar['open'] * 1000 row['highPrice'] = bar['high'] * 1000 row['lowPrice'] = bar['low'] * 1000 row['closePrice'] = bar['close'] * 1000 row['transAmount'] = int(bar['amount'] * 0.001) row['transCount'] = bar['vol'] row.append() add_record_count += 1 if add_record_count > 0: table.flush() #print(market, stock_record) if ktype == 'DAY': # 更新基础信息数据库中股票对应的起止日期及其有效标志 if valid == 0: cur = connect.cursor() cur.execute( "update stock set valid=1, startdate=%i, enddate=%i where stockid=%i" % (table[0]['datetime'], 99999999, stockid)) connect.commit() cur.close() # 记录最新更新日期 if (code == '000001' and marketid == MARKETID.SH) \ or (code == '399001' and marketid == MARKETID.SZ): update_last_date(connect, marketid, table[-1]['datetime'] / 10000) elif table.nrows == 0: #print(market, stock_record) table.remove() pass #table.close() return add_record_count
def tdx_import_day_data_from_file(connect, filename, h5file, market, stock_record): """从通达信盘后数据导入日K线 :param connect : sqlite3连接实例 :param filename: 通达信日线数据文件名 :param h5file : HDF5 file :param stock_record: 股票的相关数据 (stockid, marketid, code, valid, type) :return: 导入的记录数 """ add_record_count = 0 if not os.path.exists(filename): return add_record_count stockid, marketid, code, valid, stktype = stock_record[0], stock_record[1], stock_record[2], stock_record[3],stock_record[4] table = get_h5table(h5file, market, code) if table.nrows > 0: lastdatetime = table[-1]['datetime']/10000 else: lastdatetime = None row = table.row with open(filename, 'rb') as src_file: data = src_file.read(32) while data: record = struct.unpack('iiiiifii', data) if lastdatetime and record[0] <= lastdatetime: data = src_file.read(32) continue if 0 not in record[1:5]: if record[2] >= record[1] >= record[3] \ and record[2] >= record[4] >= record[3]: row['datetime'] = record[0] * 10000 row['openPrice'] = record[1] * 10 row['highPrice'] = record[2] * 10 row['lowPrice'] = record[3] * 10 row['closePrice'] = record[4] * 10 row['transAmount'] = round(record[5] * 0.001) if stktype == 2: # 指数 row['transCount'] = record[6] else: row['transCount'] = round(record[6] * 0.01) row.append() add_record_count += 1 data = src_file.read(32) if add_record_count > 0: table.flush() #更新基础信息数据库中股票对应的起止日期及其有效标志 if valid == 0: cur = connect.cursor() cur.execute("update stock set valid=1, startdate=%i, enddate=%i where stockid=%i" % (table[0]['datetime'], 99999999, stockid)) connect.commit() cur.close() #记录最新更新日期 if (code == '000001' and marketid == MARKETID.SH) \ or (code == '399001' and marketid == MARKETID.SZ) : update_last_date(connect, marketid, table[-1]['datetime'] / 10000) return add_record_count