def next_delta(start_time, interval, phase1_delta, phase2_delta,
ignore_weekend):
interval_delta = TimeDelta(seconds=interval)
phase1_start_delta, phase1_end_delta = phase1_delta
phase2_start_delta, phase2_end_delta = phase2_delta
today = Datetime.today()
phase1_start, phase1_end = today + phase1_start_delta, today + phase1_end_delta
phase2_start, phase2_end = today + phase2_start_delta, today + phase2_end_delta
current_time = Datetime.now()
maybe_time = current_time + interval_delta
if phase1_start_delta == phase1_end_delta and phase2_start_delta == phase2_end_delta:
# 两个时间周期各自的起止时间相等，则认为未做限制
delta = interval_delta - (current_time - start_time)
elif maybe_time > phase2_end:
# 如果预计的下一次时间大于周期2的结束日期, 则取下一日的周期1起始时间计算
next_time = today + TimeDelta(1) + phase1_start_delta
if ignore_weekend and next_time.day_of_week() in (0, 6):
next_time = today.next_week() + phase1_start_delta
delta = next_time - start_time
elif maybe_time in (phase1_start, phase1_end, phase2_start, phase2_end):
# 如果下一次时间刚好等于时间周期的起止点，则直接返回预计的时间间隔
delta = interval_delta
elif start_time < phase1_start and phase1_start < maybe_time < phase1_end:
# 如果本次的时间小于周期1的起始时间且预计下一次的时间在phase1内，则取phase1起始时间计算
delta = phase1_start - start_time
elif phase1_end < maybe_time < phase2_start:
delta = phase2_start - start_time
else:
delta = interval_delta - (current_time - start_time)
return delta
def __init__(self, config, hku_config_file, market='SH'):
super(self.__class__, self).__init__()
self.working = True
self._config = config
self.hku_config_file = hku_config_file
self.market = market.lower()
self.marketid = None
self._interval = TimeDelta(seconds=config.getint('collect', 'interval', fallback=60 * 60))
self._phase1_start_time = Datetime(
datetime.datetime.combine(
datetime.date.today(),
datetime.time.fromisoformat(
(config.get('collect', 'phase1_start', fallback='09:05'))
)
)
)
self._phase1_end_time = Datetime(
datetime.datetime.combine(
datetime.date.today(),
datetime.time.fromisoformat(
(config.get('collect', 'phase1_end', fallback='09:05'))
)
)
)
self._use_zhima_proxy = config.getboolean('collect', 'use_zhima_proxy', fallback=False)
self.cond = QWaitCondition()
self.mutex = QMutex()
def insert_bars(self, market, bars, ktype):
"""批量插入一批Bar, 如果 bar['market'] 和 指定的 market 不一致，将被忽略
:param str market: 市场简称
:param bar: 可通过键值方式取值，必须包含以下key：market, code, datetime, open, high
low, close, amount, volumn, bid1,bid1_amount .. bid5, ask1, ask1_amount...
"""
kline = [
(bar['code'], Datetime(bar['datetime']).start_of_day().number
if ktype == 'day' else Datetime(bar['datetime']).number,
bar['open'], bar['high'], bar['low'], bar['close'], bar['amount'],
bar['volumn'], bar['bid1'], bar['bid1_amount'], bar['bid2'],
bar['bid2_amount'], bar['bid3'], bar['bid3_amount'], bar['bid4'],
bar['bid4_amount'], bar['bid5'], bar['bid5_amount'], bar['ask1'],
bar['ask1_amount'], bar['ask2'], bar['ask2_amount'], bar['ask3'],
bar['ask3_amount'], bar['ask4'], bar['ask4_amount'], bar['ask5'],
bar['ask5_amount']) for bar in bars
if market.lower() == bar['market'].lower()
]
sql = 'INSERT OR REPLACE INTO {}_{}(code, datetime, open, high, low, close, amount, \
volumn, bid1, bid1_amount, bid2, bid2_amount, bid3, bid3_amount, bid4, bid4_amount, \
bid5, bid5_amount, ask1, ask1_amount, ask2, ask2_amount, ask3, ask3_amount, \
ask4, ask4_amount, ask5, ask5_amount) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,\
?,?,?,?,?,?,?,?,?,?)'.format(market, ktype)
self.cursor.executemany(sql, kline)
def run(self):
self.logger.info("{} 数据采集同步线程启动".format(self.market))
self.db = SqliteMemDriver()
hikyuu_init(self.hku_config_file, ignore_preload=True)
stk_list = self.get_stock_list()
hku_warn_if(not stk_list, "从数据库中获取的股票列表为空！", logger=self.logger)
self.mutex.tryLock()
end_delta = self._phase1_end_time - self._phase1_end_time.start_of_day(
)
start_delta = self._phase1_start_time - self._phase1_start_time.start_of_day(
)
start = Datetime.now()
if self._phase1_start_time >= self._phase1_end_time:
delta = TimeDelta(0)
elif start >= self._phase1_end_time:
delta = (self._phase1_start_time + TimeDelta(1) - start)
elif start < self._phase1_start_time:
delta = (self._phase1_start_time - start)
else:
delta = self._interval * ceil(
(start - self._phase1_start_time) / self._interval) - (
start - self._phase1_start_time)
hku_info('{} 下次采集时间：{}',
self.market,
start + delta,
logger=self.logger)
delta = int(delta.total_milliseconds())
while self.working and not self.cond.wait(self.mutex, int(delta)):
last_time = Datetime.today() + end_delta
start = Datetime.now()
if start >= last_time:
next_time = Datetime.today() + TimeDelta(1) + start_delta
hku_info('{} 明日采集时间：{}',
self.market,
next_time,
logger=self.logger)
delta = next_time - start
delta = int(delta.total_milliseconds())
continue
hku_trace("start：{}".format(start))
self.collect(stk_list)
end = Datetime.now()
x = end - start
if x + TimeDelta(seconds=1) < self._interval:
delta = self._interval - x - TimeDelta(seconds=1)
delta = int(delta.total_milliseconds())
self.logger.info("{} {:<.2f} 秒后重新采集".format(
self.market, delta / 1000))
#self.sleep(delta)
self.db.close()
self.logger.info("{} 数据采集同步线程终止!".format(self.market))
def next_time_delta(self):
now = Datetime.now()
next_time = Datetime(now.year, now.month, now.day, self.hour,
self.minute)
if next_time < now:
current_date = Datetime(now.year, now.month, now.day)
if current_date.day_of_week() == 5:
next_time = next_time + TimeDelta(3)
else:
next_time = next_time + TimeDelta(1)
return (next_time, next_time - now)
def loadKData(self, market, code, ktype, start_ix, end_ix, out_buffer):
"""
【重载接口】（必须）按指定的位置[start_ix, end_ix)读取K线数据至out_buffer
:param str market: 市场标识
:param str code: 证券代码
:param KQuery.KType ktype: K线类型
:param int start_ix: 起始位置
:param int end_ix: 结束位置
:param KRecordListPtr out_buffer: 传入的数据缓存，读取数据后使用
out_buffer.append(krecord) 加入数据
"""
if start_ix >= end_ix or start_ix <0 or end_ix <0:
return
data = self._get_bars(market, code, ktype)
if len(data) < start_ix:
return
total = end_ix if end_ix < len(data) else len(data)
for i in range(start_ix, total):
record = KRecord()
record.datetime = Datetime(data[i].get('datetime'))
record.openPrice = data[i].get('open')
record.highPrice = data[i].get('high')
record.lowPrice = data[i].get('low')
record.closePrice = data[i].get('close')
record.transAmount = data[i].get('amount')
record.transCount = data[i].get('vol')
out_buffer.append(record)
def getKRecordList(self, market, code, query):
"""
【重载接口】（必须）按指定的位置[start_ix, end_ix)读取K线数据至out_buffer
:param str market: 市场标识
:param str code: 证券代码
:param Query query: 查询条件
:rtype: KRecordList
"""
if query.query_type == Query.DATE:
print("未实现按日期查询")
return KRecordList()
start_ix = query.start
end_ix = query.end
if start_ix >= end_ix or start_ix < 0 or end_ix < 0:
return KRecordList()
data = self._get_bars(market, code, query.ktype)
if len(data) < start_ix:
return KRecordList()
total = end_ix if end_ix < len(data) else len(data)
result = KRecordList()
for i in range(start_ix, total):
record = KRecord()
record.datetime = Datetime(data[i].get('datetime'))
record.open = data[i].get('open')
record.high = data[i].get('high')
record.low = data[i].get('low')
record.close = data[i].get('close')
record.amount = data[i].get('amount')
record.volume = data[i].get('vol')
result.append(record)
return result
def getKRecord(self, market, code, pos, ktype):
"""
【重载接口】（必须）获取指定位置的K线记录
:param str market: 市场标识
:param str code: 证券代码
:param int pos: 指定位置（大于等于0）
:param Query.KType ktype: K线类型
"""
record = KRecord()
if pos < 0:
return record
data = self._get_bars(market, code, ktype)
if data is None:
return record
if pos < len(data):
record.datetime = Datetime(data.index[pos])
record.open = data['open'][pos]
record.high = data['high'][pos]
record.low = data['low'][pos]
record.close = data['close'][pos]
record.amount = data['money'][pos]
record.volume = data['volume'][pos]
return record
def loadKData(self, market, code, ktype, start_ix, end_ix, out_buffer):
"""
【重载接口】（必须）按指定的位置[start_ix, end_ix)读取K线数据至out_buffer
:param str market: 市场标识
:param str code: 证券代码
:param Query.KType ktype: K线类型
:param int start_ix: 起始位置
:param int end_ix: 结束位置
:param KRecordListPtr out_buffer: 传入的数据缓存，读取数据后使用
out_buffer.append(krecord) 加入数据
"""
if start_ix >= end_ix or start_ix < 0 or end_ix < 0:
return
data = self._get_bars(market, code, ktype)
if len(data) < start_ix:
return
total = end_ix if end_ix < len(data) else len(data)
for i in range(start_ix, total):
record = KRecord()
record.datetime = Datetime(data.index[i])
record.open = data['open'][i]
record.high = data['high'][i]
record.low = data['low'][i]
record.close = data['close'][i]
record.amount = data['money'][i]
record.volume = data['volume'][i]
out_buffer.append(record)
def getKRecord(self, market, code, pos, ktype):
"""
【重载接口】（必须）获取指定位置的K线记录
:param str market: 市场标识
:param str code: 证券代码
:param int pos: 指定位置（大于等于0）
:param KQuery.KType ktype: K线类型
"""
record = KRecord()
if pos < 0:
return record
data = self._get_bars(market, code, ktype)
if data is None:
return record
if pos < len(data):
record.datetime = Datetime(data[pos].get('datetime'))
record.openPrice = data[pos].get('open')
record.highPrice = data[pos].get('high')
record.lowPrice = data[pos].get('low')
record.closePrice = data[pos].get('close')
record.transAmount = data[pos].get('amount')
record.transCount = data[pos].get('vol')
return record
def PositionList_to_np(pos_list):
"""转化为numpy结构数组"""
t_type = np.dtype({
'names': [
'证券代码', '证券名称', '买入日期', '已持仓天数', '持仓数量', '投入金额', '当前市值',
'盈亏金额', '盈亏比例'
],
'formats':
['U10', 'U20', 'datetime64[D]', 'i', 'i', 'd', 'd', 'd', 'd']
})
sm = StockManager.instance()
query = Query(-1)
data = []
for pos in pos_list:
invest = pos.buyMoney - pos.sellMoney + pos.totalCost
k = pos.stock.getKData(query)
cur_val = k[0].closePrice * pos.number
bonus = cur_val - invest
date_list = sm.getTradingCalendar(
QueryByDate(Datetime(pos.takeDatetime.date())))
data.append(
(pos.stock.market_code, pos.stock.name, pos.takeDatetime,
len(date_list), pos.number, invest, cur_val, bonus,
100 * bonus / invest))
return np.array(data, dtype=t_type)
def __init__(self, config, market='SH'):
super(self.__class__, self).__init__()
self.working = True
self._config = config
self.market = market.lower()
self.marketid = None
assert config.getboolean("mysql", "enable", fallback=False)
self._db_config = {
'user': config['mysql']['usr'],
'password': config['mysql']['pwd'],
'host': config['mysql']['host'],
'port': config['mysql']['port']
}
self._connect = None
self.quotations = [
'stock',
] # sina 不支持基金，qq支持，此处屏蔽基金
#if config['quotation']['stock']:
# self.quotations.append('stock')
#if config['quotation']['fund']:
# self.quotations.append('fund')
#self.logger.info(self.quotations)
self._interval = TimeDelta(
seconds=config.getint('collect', 'interval', fallback=60 * 60))
self._phase1_start_time = Datetime(
datetime.datetime.combine(
datetime.date.today(),
datetime.time.fromisoformat((config.get('collect',
'phase1_start',
fallback='09:05')))))
self._phase1_end_time = Datetime(
datetime.datetime.combine(
datetime.date.today(),
datetime.time.fromisoformat((config.get('collect',
'phase1_end',
fallback='09:05')))))
self._use_zhima_proxy = config.getboolean('collect',
'use_zhima_proxy',
fallback=False)
self.cond = QWaitCondition()
self.mutex = QMutex()
def _sell(self, datetime, market, code, price, num):
"""实现 OrderBrokerBase 的 _sell 接口"""
if self._real:
import tushare as ts
df = ts.get_realtime_quotes(code)
new_price = float(df.ix[0]['bid'])
self._broker.sell(code, new_price, num)
return Datetime.now()
else:
self._broker.sell(code, price, num)
return datetime
def _sell(self, datetime, market, code, price, num):
"""实现 OrderBrokerBase 的 _sell 接口"""
if self._real:
import tushare as ts
df = ts.get_realtime_quotes(code)
new_price = float(df.ix[0]['bid'])
self._broker.sell(code, new_price, num)
return Datetime.now()
else:
self._broker.sell(code, price, num)
return datetime
def process_one_record(self, record):
if not self.record_is_valid(record):
return (0, 0)
connect = self.get_connect()
current_date = Datetime(record['datetime'].date()).number
table = get_table(connect, self.market, record['code'], 'day')
sql = 'replace into {} (date, open, high, low, close, amount, count) \
values ({}, {}, {}, {}, {}, {}, {})'.format(
table, current_date, record['open'], record['high'], record['low'],
record['close'], record['amount'], record['volumn'])
cur = connect.cursor()
cur.execute(sql)
cur.close()
return
def _buy(self, datetime, market, code, price, num):
"""实现 OrderBrokerBase 的 _buy 接口"""
if self._real:
import tushare as ts
df = ts.get_realtime_quotes(code)
new_price = float(df.ix[0]['ask'])
if (abs(new_price - price) <= self._slip):
self._broker.buy(code, new_price, num)
else:
print("out of slip, not buy!!!!!!!!!!")
return Datetime.now()
else:
self._broker.buy(code, price, num)
return datetime
def _buy(self, datetime, market, code, price, num):
"""实现 OrderBrokerBase 的 _buy 接口"""
if self._real:
import tushare as ts
df = ts.get_realtime_quotes(code)
new_price = float(df.ix[0]['ask'])
if (abs(new_price - price) <= self._slip):
self._broker.buy(code, new_price, num)
else:
print("out of slip, not buy!!!!!!!!!!")
return Datetime.now()
else:
self._broker.buy(code, price, num)
return datetime
def indicator_getitem(data, i):
"""
:param i: int | Datetime | slice | str 类型
"""
if isinstance(i, int):
length = len(data)
index = length + i if i < 0 else i
if index < 0 or index >= length:
raise IndexError("index out of range: %d" % i)
return data.get(index)
elif isinstance(i, slice):
return [data.get(x) for x in range(*i.indices(len(data)))]
elif isinstance(i, Datetime):
return data.get_by_date(i)
elif isinstance(i, str):
return data.get_by_date(Datetime(i))
else:
raise IndexError("Error index type")
def run(use_proxy, source, seconds, phase1, phase2, ignore_weekend):
phase1_delta = parse_phase(phase1)
hku_error_if(phase1_delta is None or len(phase1_delta) != 2,
"无效参数 phase1: {}".format(phase1),
callback=lambda: exit(1))
hku_error_if(phase1_delta[0] > phase1_delta[1],
"无效参数 phase1: {}, 结束时间应大于等于起始时间".format(phase1),
callback=lambda: exit(1))
phase2_delta = parse_phase(phase2)
hku_error_if(phase2_delta is None or len(phase2_delta) != 2,
"无效参数 phase2: {}".format(phase2),
callback=lambda: exit(1))
hku_error_if(phase2_delta[0] > phase2_delta[1],
"无效参数 phase2: {}, 结束时间应大于等于起始时间".format(phase2),
callback=lambda: exit(1))
hku_error_if(
phase1_delta[1] > phase2_delta[0],
"无效参数 phase1: {}, phase2: {}, phase2 起始时间应大于等于 phase1 结束时间".format(
phase1, phase2),
callback=lambda: exit(1))
hku_logger.info("采集时间段1：{}".format(phase1))
hku_logger.info("采集时间段2：{}".format(phase2))
config_file = os.path.expanduser('~') + '/.hikyuu/hikyuu.ini'
if not os.path.exists(config_file):
print("未找到配置文件，请先运行 HikyuuTDX 进行配置与数据导入")
exit(1)
hikyuu_init(config_file, ignore_preload=True)
print("采集程序运行中，可使用 Ctrl-C 终止！")
sm = StockManager.instance()
stk_list = [
stk.market_code.lower() for stk in sm
if stk.valid and stk.type in (constant.STOCKTYPE_A,
constant.STOCKTYPE_INDEX,
constant.STOCKTYPE_GEM)
]
spot_topic = ':spot:'
def batch_func(records):
spot = bytearray(spot_topic.encode('utf-8'))
buf = create_fb_spot(records)
spot.extend(buf)
pub_sock.send(bytes(spot))
address = "ipc:///tmp/hikyuu_real_pub.ipc"
pub_sock = pynng.Pub0()
pub_sock.listen(address)
today = Datetime.today()
phase1_time = [today + x for x in phase1_delta]
phase2_time = [today + x for x in phase2_delta]
start_time = Datetime.now()
if not (phase1_time[0] <= start_time <= phase1_time[1]
or phase2_time[0] <= start_time <= phase2_time[1]):
delta = next_delta(start_time, seconds, phase1_delta, phase2_delta,
ignore_weekend)
next_time = start_time + delta
hku_info("启动采集时间：{}".format(next_time))
time.sleep(delta.total_seconds())
while True:
try:
start_time = Datetime.now()
pub_sock.send("{}{}".format(spot_topic,
'[start spot]').encode('utf-8'))
records = get_spot_parallel(stk_list, source, use_proxy,
batch_func)
hku_info("{}:{}:{} 采集数量: {}".format(start_time.hour,
start_time.minute,
start_time.second,
len(records)))
pub_sock.send('{}{}'.format(spot_topic,
'[end spot]').encode('utf-8'))
delta = next_delta(start_time, seconds, phase1_delta, phase2_delta,
ignore_weekend)
time.sleep(delta.total_seconds())
except Exception as e:
hku_error(e)
time.sleep(10)
class CollectToMemThread(QThread):
def __init__(self, config, hku_config_file, market='SH'):
super(self.__class__, self).__init__()
self.working = True
self._config = config
self.hku_config_file = hku_config_file
self.market = market.lower()
self.marketid = None
self._interval = TimeDelta(seconds=config.getint('collect', 'interval', fallback=60 * 60))
self._phase1_start_time = Datetime(
datetime.datetime.combine(
datetime.date.today(),
datetime.time.fromisoformat(
(config.get('collect', 'phase1_start', fallback='09:05'))
)
)
)
self._phase1_end_time = Datetime(
datetime.datetime.combine(
datetime.date.today(),
datetime.time.fromisoformat(
(config.get('collect', 'phase1_end', fallback='09:05'))
)
)
)
self._use_zhima_proxy = config.getboolean('collect', 'use_zhima_proxy', fallback=False)
self.cond = QWaitCondition()
self.mutex = QMutex()
def stop(self):
if self.working:
self.working = False
self.cond.wakeAll()
self.wait()
def __del__(self):
self.stop()
@hku_catch()
def run(self):
self.logger.info("{} 数据采集同步线程启动".format(self.market))
self.db = SqliteMemDriver()
hikyuu_init(self.hku_config_file, ignore_preload=True)
stk_list = self.get_stock_list()
hku_warn_if(not stk_list, "从数据库中获取的股票列表为空！", self.logger)
self.mutex.tryLock()
end_delta = self._phase1_end_time - self._phase1_end_time.start_of_day()
start_delta = self._phase1_start_time - self._phase1_start_time.start_of_day()
start = Datetime.now()
if self._phase1_start_time >= self._phase1_end_time:
delta = TimeDelta(0)
elif start >= self._phase1_end_time:
delta = (self._phase1_start_time + TimeDelta(1) - start)
elif start < self._phase1_start_time:
delta = (self._phase1_start_time - start)
else:
delta = self._interval * ceil((start - self._phase1_start_time) / self._interval
) - (start - self._phase1_start_time)
hku_info('{} 下次采集时间：{}'.format(self.market, start + delta), self.logger)
delta = int(delta.total_milliseconds())
while self.working and not self.cond.wait(self.mutex, int(delta)):
last_time = Datetime.today() + end_delta
start = Datetime.now()
if start >= last_time:
next_time = Datetime.today() + TimeDelta(1) + start_delta
hku_info('{} 明日采集时间：{}'.format(self.market, next_time), self.logger)
delta = next_time - start
delta = int(delta.total_milliseconds())
continue
hku_trace("start：{}".format(start))
self.collect(stk_list)
end = Datetime.now()
x = end - start
if x + TimeDelta(seconds=1) < self._interval:
delta = self._interval - x - TimeDelta(seconds=1)
delta = int(delta.total_milliseconds())
self.logger.info("{} {:<.2f} 秒后重新采集".format(self.market, delta / 1000))
#self.sleep(delta)
self.db.close()
self.logger.info("{} 数据采集同步线程终止!".format(self.market))
@hku_catch(retry=2, ret=[])
def get_stock_list(self):
sm = StockManager.instance()
return [
stk.market_code.lower() for stk in sm
if stk.type in (constant.STOCKTYPE_A, constant.STOCKTYPE_INDEX, constant.STOCKTYPE_GEM)
and stk.valid and stk.market.lower() == self.market.lower()
]
def record_is_valid(self, record):
return record['amount'] > 0.0 and record['volumn'] > 0.0 \
and record['high'] >= record['open'] >= record['low'] > 0.0 \
and record['high'] >= record['close'] >= record['low']
@hku_catch()
def collect(self, stk_list):
record_list = get_spot_parallel(stk_list, source='sina', use_proxy=self._use_zhima_proxy)
hku_info("{} 网络获取数量：{}".format(self.market, len(record_list)))
record_list = [r for r in record_list if self.record_is_valid(r)]
self.db.insert_bars(self.market, record_list, 'day')
self.db.insert_bars(self.market, record_list, 'min')
class CollectToMySQLThread(QThread):
def __init__(self, config, market='SH'):
super(self.__class__, self).__init__()
self.working = True
self._config = config
self.market = market.lower()
self.marketid = None
assert config.getboolean("mysql", "enable", fallback=False)
self._db_config = {
'user': config['mysql']['usr'],
'password': config['mysql']['pwd'],
'host': config['mysql']['host'],
'port': config['mysql']['port']
}
self._connect = None
self.quotations = [
'stock',
] # sina 不支持基金，qq支持，此处屏蔽基金
#if config['quotation']['stock']:
# self.quotations.append('stock')
#if config['quotation']['fund']:
# self.quotations.append('fund')
#self.logger.info(self.quotations)
self._interval = TimeDelta(
seconds=config.getint('collect', 'interval', fallback=60 * 60))
self._phase1_start_time = Datetime(
datetime.datetime.combine(
datetime.date.today(),
datetime.time.fromisoformat((config.get('collect',
'phase1_start',
fallback='09:05')))))
self._phase1_end_time = Datetime(
datetime.datetime.combine(
datetime.date.today(),
datetime.time.fromisoformat((config.get('collect',
'phase1_end',
fallback='09:05')))))
self._use_zhima_proxy = config.getboolean('collect',
'use_zhima_proxy',
fallback=False)
self.cond = QWaitCondition()
self.mutex = QMutex()
def stop(self):
if self.working:
self.working = False
self.cond.wakeAll()
self.wait()
if self._connect is not None:
hku_trace('关闭数据库连接', logger=self.logger)
self._connect.close()
def __del__(self):
self.stop()
@hku_catch()
def run(self):
self.logger.info("{} 数据采集同步线程启动".format(self.market))
stk_list = self.get_stock_list()
hku_warn_if(not stk_list, "从数据库中获取的股票列表为空！", logger=self.logger)
self.mutex.tryLock()
end_delta = self._phase1_end_time - self._phase1_end_time.start_of_day(
)
start_delta = self._phase1_start_time - self._phase1_start_time.start_of_day(
)
start = Datetime.now()
if self._phase1_start_time >= self._phase1_end_time:
delta = TimeDelta(0)
elif start >= self._phase1_end_time:
delta = (self._phase1_start_time + TimeDelta(1) - start)
elif start < self._phase1_start_time:
delta = (self._phase1_start_time - start)
else:
delta = self._interval * ceil(
(start - self._phase1_start_time) / self._interval) - (
start - self._phase1_start_time)
hku_info('{} 下次采集时间：{}',
self.market,
start + delta,
logger=self.logger)
delta = int(delta.total_milliseconds())
while self.working and not self.cond.wait(self.mutex, int(delta)):
last_time = Datetime.today() + end_delta
start = Datetime.now()
if start >= last_time:
next_time = Datetime.today() + TimeDelta(1) + start_delta
hku_info('{} 明日采集时间：{}',
self.market,
next_time,
logger=self.logger)
delta = next_time - start
delta = int(delta.total_milliseconds())
continue
hku_trace("start：{}".format(start))
self.collect(stk_list)
end = Datetime.now()
x = end - start
if x + TimeDelta(seconds=1) < self._interval:
delta = self._interval - x - TimeDelta(seconds=1)
delta = int(delta.total_milliseconds())
self.logger.info("{} {:<.2f} 秒后重新采集".format(
self.market, delta / 1000))
#self.sleep(delta)
self.logger.info("{} 数据采集同步线程终止!".format(self.market))
def record_is_valid(self, record):
return record['amount'] > 0.0 and record['volumn'] > 0.0 \
and record['high'] >= record['open'] >= record['low'] > 0.0 \
and record['high'] >= record['close'] >= record['low']
@hku_catch(ret=False)
def process_one_record(self, record):
if not self.record_is_valid(record):
return (0, 0)
connect = self.get_connect()
current_date = Datetime(record['datetime'].date()).number
table = get_table(connect, self.market, record['code'], 'day')
sql = 'replace into {} (date, open, high, low, close, amount, count) \
values ({}, {}, {}, {}, {}, {}, {})'.format(
table, current_date, record['open'], record['high'], record['low'],
record['close'], record['amount'], record['volumn'])
cur = connect.cursor()
cur.execute(sql)
cur.close()
return
@hku_catch()
def collect(self, stk_list):
record_list = get_spot_parallel(stk_list,
source='sina',
use_proxy=self._use_zhima_proxy)
hku_info("{} 网络获取数量：{}".format(self.market, len(record_list)))
connect = self.get_connect()
if self.marketid == None:
self.marketid = get_marketid(connect, self.market)
update_count = 0
for record in record_list:
if not self.working:
break
if self.process_one_record(record):
update_count += 1
connect.commit()
hku_info("{} 数据库更新记录数：{}".format(self.market, update_count))
def get_connect(self):
if self._connect:
return self._connect
try:
self._connect = mysql.connector.connect(**self._db_config)
except mysql.connector.Error as err:
if err.errno == errorcode.ER_ACCESS_DENIED_ERROR:
self.logger.error("MYSQL密码或用户名错误！")
elif err.errno == errorcode.ER_BAD_DB_ERROR:
self.logger.error("MySQL数据库不存在！")
else:
self.logger.error("连接数据库失败，{}".format(err.msg))
except:
self.logger.error("未知原因导致无法连接数据库！")
return self._connect
@hku_catch(retry=2, ret=[])
def get_stock_list(self):
connect = self.get_connect()
stk_list = get_stock_list(connect, self.market, self.quotations)
return [
"{}{}".format(self.market.lower(), item[2]) for item in stk_list
if item[3] == 1
]
def _getIndexRangeByDate(self, market, code, query):
"""
【重载接口】（必须）按日期获取指定的K线数据
:param str market: 市场标识
:param str code: 证券代码
:param KQuery query: 日期查询条件（QueryByDate）
"""
print("getIndexRangeByDate")
if query.queryType != Query.DATE:
return (0, 0)
start_datetime = query.startDatetime
end_datetime = query.endDatetime
if start_datetime >= end_datetime or start_datetime > Datetime.max():
return (0, 0)
data = self._get_bars(market, code, query.kType)
total = len(data)
if total == 0:
return (0, 0)
mid, low = 0, 0
high = total-1
while low <= high:
tmp_datetime = Datetime(data[high].get('datetime'))
if start_datetime > tmp_datetime:
mid = high + 1
break
tmp_datetime = Datetime(data[low].get('datetime'))
if tmp_datetime >= start_datetime:
mid = low
break
mid = (low + high) // 2
tmp_datetime = Datetime(data[mid].get('datetime'))
if start_datetime > tmp_datetime:
low = mid + 1
else:
high = mid - 1
if mid >= total:
return (0, 0)
start_pos = mid
low = mid
high = total - 1
while low <= high:
tmp_datetime = Datetime(data[high].get('datetime'))
if end_datetime > tmp_datetime:
mid = high + 1
break
tmp_datetime = Datetime(data[low].get('datetime'))
if tmp_datetime >= end_datetime:
mid = low
break
mid = (low + high) // 2
tmp_datetime = Datetime(data[mid].get('datetime'))
if end_datetime > tmp_datetime:
low = mid + 1
else:
high = mid - 1
end_pos = total if mid >= total else mid
if start_pos >= end_pos:
return (0,0)
return (start_pos, end_pos)