def tdx_import_min_data_from_file(connect, filename, h5file, market, stock_record):
"""从通达信盘后数据导入1分钟或5分钟K线
:param connect : sqlite3连接实例
:param filename: 通达信K线数据文件名
:param h5file : HDF5 file
:param stock_record: 股票的相关数据 (stockid, marketid, code, valid, type)
:return: 导入的记录数
"""
add_record_count = 0
if not os.path.exists(filename):
return add_record_count
stockid, marketid, code, valid, stktype = stock_record[0], stock_record[1], stock_record[2], stock_record[3],stock_record[4]
table = get_h5table(h5file, market, code)
if table.nrows > 0:
lastdatetime = table[-1]['datetime']
else:
lastdatetime = None
row = table.row
with open(filename, 'rb') as src_file:
def trans_date(yymm, hhmm):
tmp_date = yymm >> 11
remainder = yymm & 0x7ff
year = tmp_date + 2004
month = remainder // 100
day = remainder % 100
hh = hhmm // 60
mm = hhmm % 60
return year * 100000000 + month * 1000000 + day * 10000 + hh * 100 + mm
def get_date(pos):
src_file.seek(pos * 32, SEEK_SET)
data = src_file.read(4)
a = struct.unpack('hh', data)
return trans_date(a[0], a[1])
def find_pos():
src_file.seek(0, SEEK_END)
pos = src_file.tell()
total = pos // 32
if lastdatetime is None:
return total, 0
low, high = 0, total - 1
mid = high // 2
while mid <= high:
cur_date = get_date(low)
if cur_date > lastdatetime:
mid = low
break
cur_date = get_date(high)
if cur_date <= lastdatetime:
mid = high + 1
break
cur_date = get_date(mid)
if cur_date <= lastdatetime:
low = mid + 1
else:
high = mid - 1
mid = (low + high) // 2
return total, mid
file_total, pos = find_pos()
if pos < file_total:
src_file.seek(pos * 32, SEEK_SET)
data = src_file.read(32)
while data:
record = struct.unpack('hhfffffii', data)
if 0 not in record[2:6]:
if record[3] >= record[2] >= record[4] \
and record[3] >= record[5] >= record[4]:
row['datetime'] = trans_date(record[0], record[1])
row['openPrice'] = record[2] * 1000
row['highPrice'] = record[3] * 1000
row['lowPrice'] = record[4] * 1000
row['closePrice'] = record[5] * 1000
row['transAmount'] = round(record[6] * 0.001)
if stktype == 2:
# 指数
row['transCount'] = record[7]
else:
row['transCount'] = round(record[6] * 0.01)
row.append()
add_record_count += 1
data = src_file.read(32)
if add_record_count > 0:
table.flush()
return add_record_count
def import_one_stock_data(connect,
api,
h5file,
market,
ktype,
stock_record,
startDate=199012191500):
market = market.upper()
pytdx_market = to_pytdx_market(market)
stockid, marketid, code, valid, stktype = stock_record[0], stock_record[1], stock_record[2], stock_record[3], \
stock_record[4]
table = get_h5table(h5file, market, code)
last_datetime = table[-1]['datetime'] if table.nrows > 0 else startDate
today = datetime.date.today()
if ktype == 'DAY':
n, step = guess_day_n_step(last_datetime)
pytdx_kline_type = TDXParams.KLINE_TYPE_RI_K
today_datetime = (today.year * 10000 + today.month * 100 +
today.day) * 10000
elif ktype == '1MIN':
n, step = guess_1min_n_step(last_datetime)
pytdx_kline_type = TDXParams.KLINE_TYPE_1MIN
today_datetime = (today.year * 10000 + today.month * 100 +
today.day) * 10000 + 1500
elif ktype == '5MIN':
n, step = guess_5min_n_step(last_datetime)
pytdx_kline_type = TDXParams.KLINE_TYPE_5MIN
today_datetime = (today.year * 10000 + today.month * 100 +
today.day) * 10000 + 1500
else:
return 0
if today_datetime <= last_datetime:
return 0
get_bars = api.get_index_bars if stktype == STOCKTYPE.INDEX else api.get_security_bars
add_record_count = 0
row = table.row
while n >= 0:
bar_list = get_bars(pytdx_kline_type, pytdx_market, code, n * 800,
step)
n -= 1
if bar_list is None:
#print(code, "invalid!!")
continue
for bar in bar_list:
try:
tmp = datetime.date(bar['year'], bar['month'], bar['day'])
bar_datetime = (tmp.year * 10000 + tmp.month * 100 +
tmp.day) * 10000
if ktype != 'DAY':
bar_datetime += bar['hour'] * 100 + bar['minute']
except:
continue
if today_datetime >= bar_datetime > last_datetime \
and bar['high'] >= bar['open'] >= bar['low'] > 0 \
and bar['high'] >= bar['close'] >= bar['low'] > 0 \
and bar['vol'] != 0 and bar['amount'] != 0:
row['datetime'] = bar_datetime
row['openPrice'] = bar['open'] * 1000
row['highPrice'] = bar['high'] * 1000
row['lowPrice'] = bar['low'] * 1000
row['closePrice'] = bar['close'] * 1000
row['transAmount'] = int(bar['amount'] * 0.001)
row['transCount'] = bar['vol']
row.append()
add_record_count += 1
if add_record_count > 0:
table.flush()
#print(market, stock_record)
if ktype == 'DAY':
# 更新基础信息数据库中股票对应的起止日期及其有效标志
if valid == 0:
cur = connect.cursor()
cur.execute(
"update stock set valid=1, startdate=%i, enddate=%i where stockid=%i"
% (table[0]['datetime'], 99999999, stockid))
connect.commit()
cur.close()
# 记录最新更新日期
if (code == '000001' and marketid == MARKETID.SH) \
or (code == '399001' and marketid == MARKETID.SZ):
update_last_date(connect, marketid,
table[-1]['datetime'] / 10000)
elif table.nrows == 0:
#print(market, stock_record)
table.remove()
pass
#table.close()
return add_record_count
def tdx_import_day_data_from_file(connect, filename, h5file, market, stock_record):
"""从通达信盘后数据导入日K线
:param connect : sqlite3连接实例
:param filename: 通达信日线数据文件名
:param h5file : HDF5 file
:param stock_record: 股票的相关数据 (stockid, marketid, code, valid, type)
:return: 导入的记录数
"""
add_record_count = 0
if not os.path.exists(filename):
return add_record_count
stockid, marketid, code, valid, stktype = stock_record[0], stock_record[1], stock_record[2], stock_record[3],stock_record[4]
table = get_h5table(h5file, market, code)
if table.nrows > 0:
lastdatetime = table[-1]['datetime']/10000
else:
lastdatetime = None
row = table.row
with open(filename, 'rb') as src_file:
data = src_file.read(32)
while data:
record = struct.unpack('iiiiifii', data)
if lastdatetime and record[0] <= lastdatetime:
data = src_file.read(32)
continue
if 0 not in record[1:5]:
if record[2] >= record[1] >= record[3] \
and record[2] >= record[4] >= record[3]:
row['datetime'] = record[0] * 10000
row['openPrice'] = record[1] * 10
row['highPrice'] = record[2] * 10
row['lowPrice'] = record[3] * 10
row['closePrice'] = record[4] * 10
row['transAmount'] = round(record[5] * 0.001)
if stktype == 2:
# 指数
row['transCount'] = record[6]
else:
row['transCount'] = round(record[6] * 0.01)
row.append()
add_record_count += 1
data = src_file.read(32)
if add_record_count > 0:
table.flush()
#更新基础信息数据库中股票对应的起止日期及其有效标志
if valid == 0:
cur = connect.cursor()
cur.execute("update stock set valid=1, startdate=%i, enddate=%i where stockid=%i" %
(table[0]['datetime'], 99999999, stockid))
connect.commit()
cur.close()
#记录最新更新日期
if (code == '000001' and marketid == MARKETID.SH) \
or (code == '399001' and marketid == MARKETID.SZ) :
update_last_date(connect, marketid, table[-1]['datetime'] / 10000)
return add_record_count
