def raw_history_bars(self, instrument, frequency, start_dt=None, end_dt=None, length=None):
symbol = instrument_to_tushare(instrument)
if frequency in ["1d"]:
if start_dt and end_dt:
s_date_int = convert_date_to_int(start_dt.date())
e_date_int = convert_date_to_int(end_dt.date())
elif start_dt and length:
dates = self._dates_index(instrument)
s_date_int = convert_date_to_int(start_dt.date())
s_pos = safe_searchsorted(dates, s_date_int)
s_date_int = int(dates[s_pos])
e_date_int = int(dates[min(s_pos + length, len(dates)) - 1])
elif end_dt and length:
dates = self._dates_index(instrument)
e_date_int = convert_date_to_int(end_dt.date())
e_pos = safe_searchsorted(dates, e_date_int, side="right")
s_date_int = int(dates[max(e_pos - length, 0)])
e_date_int = int(dates[e_pos - 1])
else:
raise RuntimeError("At least two of [start_dt,end_dt,length] should be given.")
data, msg = self._api.daily(symbol, freq=frequency, adjust_mode=None,
start_date=s_date_int // 1000000,
end_date=e_date_int // 1000000)
if isinstance(data, pd.DataFrame) and data.size:
data = data[data["volume"] > 0] # TODO sikp_suspended?
return QuantOsConverter.df2np(data)
else:
if msg:
system_log.warning(msg)
return QuantOsConverter.empty()
else:
return MiniteBarDataSourceMixin.raw_history_bars(
self, instrument, frequency, start_dt=start_dt, end_dt=end_dt, length=length
)
def _get_bars_in_days(self, instrument, frequency, params):
s_date = params[0]["trade_date"]
e_date = params[-1]["trade_date"]
s_time = params[0]["start_time"] if "start_time" in params[0] else 0
e_time = params[-1]["end_time"] if "end_time" in params[-1] else 150000
s_dt_int = convert_date_to_int(s_date) + s_time
e_dt_int = convert_date_to_int(e_date) + e_time
db = self._get_db(instrument=instrument, frequency=frequency)
collection = instrument.order_book_id
filters = {
"_d": {
"$gte": datetime.combine(s_date, time=time()),
"$lte": datetime.combine(e_date, time=time())
}
}
projection = {"_id": 0, "_d": 0}
loop = get_asyncio_event_loop()
bars = loop.run_until_complete(
self._do_get_bars(db, collection, filters, projection))
if bars is not None and bars.size:
bars = DataFrameConverter.df2np(bars)
else:
bars = DataFrameConverter.empty()
s_pos = np.searchsorted(bars["datetime"], s_dt_int)
e_pos = np.searchsorted(bars["datetime"], e_dt_int, side="right")
return bars[s_pos:e_pos]
def history_bars(self, instrument, bar_count, frequency, fields, dt,
skip_suspended=True, include_now=False,
adjust_type='pre', adjust_orig=None):
if frequency != '1d' and frequency != '1w':
raise NotImplementedError
if skip_suspended and instrument.type == 'CS':
bars = self._filtered_day_bars(instrument)
else:
bars = self._all_day_bars_of(instrument)
if not self._are_fields_valid(fields, bars.dtype.names):
raise RQInvalidArgument("invalid fields: {}".format(fields))
if len(bars) <= 0:
return bars
if frequency == '1w':
if include_now:
dt = np.uint64(convert_date_to_int(dt))
i = bars['datetime'].searchsorted(dt, side='right')
else:
monday = dt - timedelta(days=dt.weekday())
monday = np.uint64(convert_date_to_int(monday))
i = bars['datetime'].searchsorted(monday, side='left')
left = i - bar_count * 5 if i >= bar_count * 5 else 0
bars = bars[left:i]
if adjust_type == 'none' or instrument.type in {'Future', 'INDX'}:
# 期货及指数无需复权
week_bars = self.resample_week_bars(bars, bar_count, fields)
return week_bars if fields is None else week_bars[fields]
if isinstance(fields, str) and fields not in FIELDS_REQUIRE_ADJUSTMENT:
week_bars = self.resample_week_bars(bars, bar_count, fields)
return week_bars if fields is None else week_bars[fields]
adjust_bars_date = adjust_bars(bars, self.get_ex_cum_factor(instrument.order_book_id),
fields, adjust_type, adjust_orig)
adjust_week_bars = self.resample_week_bars(adjust_bars_date, bar_count, fields)
return adjust_week_bars if fields is None else adjust_week_bars[fields]
dt = np.uint64(convert_date_to_int(dt))
i = bars['datetime'].searchsorted(dt, side='right')
left = i - bar_count if i >= bar_count else 0
bars = bars[left:i]
if adjust_type == 'none' or instrument.type in {'Future', 'INDX'}:
# 期货及指数无需复权
return bars if fields is None else bars[fields]
if isinstance(fields, str) and fields not in FIELDS_REQUIRE_ADJUSTMENT:
return bars if fields is None else bars[fields]
bars = adjust_bars(bars, self.get_ex_cum_factor(instrument.order_book_id),
fields, adjust_type, adjust_orig)
return bars if fields is None else bars[fields]
def get_bar(self, instrument, dt, frequency):
"""
:type instrument: rqalpha.model.instrument.instrument
:type dt: datetime.datetime
:param str frequency: `1d` or `1m`
:return: numpy.ndarray
"""
if frequency == '1d':
bars = self.get_stock_data_from_mongo(instrument.order_book_id,
CycType.CYC_DAY)
if bars is None:
return
dt = convert_date_to_int(dt)
pos = bars['datetime'].searchsorted(dt)
if pos >= len(bars) or bars['datetime'][pos] != dt:
return None
return bars[pos]
elif frequency == '1m':
bars = self.get_stock_data_from_mongo(instrument.order_book_id,
CycType.CYC_MINUTE)
if bars is None:
return
dt = convert_dt_to_int(dt)
pos = bars['datetime'].searchsorted(dt)
if pos >= len(bars) or bars['datetime'][pos] != dt:
return None
return bars[pos]
else:
raise NotImplementedError
def history_bars(self,
instrument,
bar_count,
frequency,
fields,
dt,
skip_suspended=True):
"""
获取历史数据
:param instrument: 合约对象
:type instrument: :class:`~Instrument`
:param int bar_count: 获取的历史数据数量
:param str frequency: 周期频率，`1d` 表示日周期, `1m` 表示分钟周期
:param str fields: 返回数据字段
========================= ===================================================
fields 字段名
========================= ===================================================
datetime 时间戳
open 开盘价
high 最高价
low 最低价
close 收盘价
volume 成交量
total_turnover 成交额
datetime int类型时间戳
open_interest 持仓量（期货专用）
basis_spread 期现差（股指期货专用）
settlement 结算价（期货日线专用）
prev_settlement 结算价（期货日线专用）
========================= ===================================================
:param datetime.datetime dt: 时间
:param bool skip_suspended: 是否跳过停牌日
:return: `numpy.ndarray`
"""
if frequency != '1d':
raise NotImplementedError
if skip_suspended and instrument.type == 'CS':
bars = self._filtered_day_bars(instrument)
else:
bars = self._all_day_bars_of(instrument)
if bars is None or not self._are_fields_valid(fields,
bars.dtype.names):
return None
dt = convert_date_to_int(dt)
i = bars['datetime'].searchsorted(dt, side='right')
left = i - bar_count if i >= bar_count else 0
if fields is None:
return bars[left:i]
else:
return bars[left:i][fields]
def get_bar(self, instrument, dt, frequency):
"""
根据 dt 来获取对应的 Bar 数据
:param instrument: 合约对象
:type instrument: :class:`~Instrument`
:param datetime.datetime dt: calendar_datetime
:param str frequency: 周期频率，`1d` 表示日周期, `1m` 表示分钟周期
:return: `numpy.ndarray` | `dict`
"""
if frequency != '1d':
raise NotImplementedError
bars = self._all_day_bars_of(instrument)
if bars is None:
return
dt = convert_date_to_int(dt)
pos = bars['datetime'].searchsorted(dt)
if pos >= len(bars) or bars['datetime'][pos] != dt:
return None
return bars[pos]
def __call__(self, path, fields, **kwargs):
with h5py.File(path, 'w') as h5:
i, step = 0, 300
while True:
order_book_ids = self._order_book_ids[i:i + step]
df = rqdatac.get_price(order_book_ids,
START_DATE,
datetime.date.today(),
'1d',
adjust_type='none',
fields=fields,
expect_df=True)
if not (df is None or df.empty):
df.reset_index(inplace=True)
df['datetime'] = [
convert_date_to_int(d) for d in df['date']
]
del df['date']
df.set_index(['order_book_id', 'datetime'], inplace=True)
df.sort_index(inplace=True)
for order_book_id in df.index.levels[0]:
h5.create_dataset(
order_book_id,
data=df.loc[order_book_id].to_records(),
**kwargs)
i += step
yield len(order_book_ids)
if i >= len(self._order_book_ids):
break
def history_bars(self, instrument, bar_count, frequency, fields, dt,
skip_suspended=True, include_now=False,
adjust_type='pre', adjust_orig=None):
if frequency != '1d':
raise NotImplementedError
if skip_suspended and instrument.type == 'CS':
bars = self._filtered_day_bars(instrument)
else:
bars = self._all_day_bars_of(instrument)
if bars is None or not self._are_fields_valid(fields, bars.dtype.names):
return None
dt = np.uint64(convert_date_to_int(dt))
i = bars['datetime'].searchsorted(dt, side='right')
left = i - bar_count if i >= bar_count else 0
bars = bars[left:i]
if adjust_type == 'none' or instrument.type in {'Future', 'INDX'}:
# 期货及指数无需复权
return bars if fields is None else bars[fields]
if isinstance(fields, str) and fields not in FIELDS_REQUIRE_ADJUSTMENT:
return bars if fields is None else bars[fields]
return adjust_bars(bars, self.get_ex_cum_factor(instrument.order_book_id),
fields, adjust_type, adjust_orig)
def history_bars(self, instrument, bar_count, frequency, fields, dt,
skip_suspended=True, include_now=False,
adjust_type='pre', adjust_orig=None):
if frequency != '1d':
raise NotImplementedError
if skip_suspended and instrument.type == 'CS':
bars = self._filtered_day_bars(instrument)
else:
bars = self._all_day_bars_of(instrument)
if bars is None or not self._are_fields_valid(fields, bars.dtype.names):
return None
dt = np.uint64(convert_date_to_int(dt))
i = bars['datetime'].searchsorted(dt, side='right')
left = i - bar_count if i >= bar_count else 0
bars = bars[left:i]
if adjust_type == 'none' or instrument.type in {'Future', 'INDX'}:
# 期货及指数无需复权
return bars if fields is None else bars[fields]
if isinstance(fields, str) and fields not in FIELDS_REQUIRE_ADJUSTMENT:
return bars if fields is None else bars[fields]
return adjust_bars(bars, self.get_ex_cum_factor(instrument.order_book_id),
fields, adjust_type, adjust_orig)
def resample_week_bars(self, bars, bar_count, fields):
df_bars = pd.DataFrame(bars)
df_bars['datetime'] = df_bars.apply(
lambda x: convert_int_to_datetime(x['datetime']), axis=1)
df_bars = df_bars.set_index('datetime')
nead_fields = fields
if isinstance(nead_fields, str):
nead_fields = [nead_fields]
hows = {
field: BAR_RESAMPLE_FIELD_METHODS[field]
for field in nead_fields if field in BAR_RESAMPLE_FIELD_METHODS
}
df_bars = df_bars.resample('W-Fri').agg(hows)
df_bars.index = df_bars.index.map(
self._update_weekly_trading_date_index)
df_bars = df_bars[~df_bars.index.duplicated(keep='first')]
df_bars.sort_index(inplace=True)
df_bars = df_bars[-bar_count:]
df_bars = df_bars.reset_index()
df_bars['datetime'] = df_bars.apply(
lambda x: np.uint64(convert_date_to_int(x['datetime'].date())),
axis=1)
df_bars = df_bars.set_index('datetime')
bars = df_bars.to_records()
return bars
async def _get_bars_in_day(self, instrument=None, frequency=None, trade_date=None, start_time=0, end_time=150000):
# TODO retry when net error occurs
symbol = instrument_to_tushare(instrument)
trade_date = convert_date_to_int(trade_date) // 1000000
start_time = max(start_time, 80000)
end_time = min(end_time, 160000)
return self._api.bar(symbol=symbol, freq=frequency[:-1] + frequency[-1].upper(),
trade_date=trade_date, start_time=start_time, end_time=end_time)
def history_bars(self,
instrument,
bar_count,
frequency,
fields,
dt,
skip_suspended=True,
include_now=False):
"""
:type instrument: rqalpha.model.instrument.instrument
:type bar_count: int
:param str frequency: `1d` or `1m`
:type fields: str or list[str]
:type dt: datetime.datetime
:return: numpy.ndarray
"""
if frequency == '1d':
bars = self.get_stock_data_from_mongo(instrument.order_book_id,
CycType.CYC_DAY)
if bars is None or not self._are_fields_valid(
fields, bars.dtype.names):
return None
if skip_suspended and instrument.type == 'CS':
bars = bars[bars['volume'] > 0]
dt = convert_date_to_int(dt)
i = bars['datetime'].searchsorted(dt, side='right')
left = i - bar_count if i >= bar_count else 0
if fields is None:
return bars[left:i]
else:
return bars[left:i][fields]
elif frequency == '1m':
bars = self.get_stock_data_from_mongo(instrument.order_book_id,
CycType.CYC_MINUTE)
if bars is None or not self._are_fields_valid(
fields, bars.dtype.names):
return None
# if skip_suspended and instrument.type == 'CS':
# bars = bars[bars['volume'] > 0]
dt = convert_dt_to_int(dt)
i = bars['datetime'].searchsorted(dt, side='right')
left = i - bar_count if i >= bar_count else 0
if fields is None:
return bars[left:i]
else:
return bars[left:i][fields]
else:
raise NotImplementedError
def get_split_by_ex_date(self, order_book_id, date):
df = self.get_split(order_book_id)
if df is None or len(df) == 0:
return
dt = convert_date_to_int(date)
pos = df['ex_date'].searchsorted(dt)
if pos == len(df) or df['ex_date'][pos] != dt:
return None
return df['split_factor'][pos]
def get_split_by_ex_date(self, order_book_id, date):
df = self.get_split(order_book_id)
if df is None or len(df) == 0:
return
dt = convert_date_to_int(date)
pos = df['ex_date'].searchsorted(dt)
if pos == len(df) or df['ex_date'][pos] != dt:
return None
return df['split_factor'][pos]
def __call__(self, path, fields, **kwargs):
need_recreate_h5 = False
with h5py.File(path, 'r') as h5:
need_recreate_h5 = not self.h5_has_valid_fields(h5, fields)
if need_recreate_h5:
yield from GenerateDayBarTask(self._order_book_ids)(path, fields,
**kwargs)
else:
with h5py.File(path, 'a') as h5:
for order_book_id in self._order_book_ids:
if order_book_id in h5:
try:
start_date = rqdatac.get_next_trading_date(
int(h5[order_book_id]['datetime'][-1] //
1000000))
except ValueError:
h5.pop(order_book_id)
start_date = START_DATE
else:
start_date = START_DATE
df = rqdatac.get_price(order_book_id,
start_date,
END_DATE,
'1d',
adjust_type='none',
fields=fields,
expect_df=True)
if not (df is None or df.empty):
df = df[
fields] # Future order_book_id like SC888 will auto add 'dominant_id'
df = df.loc[order_book_id]
df.reset_index(inplace=True)
df['datetime'] = [
convert_date_to_int(d) for d in df['date']
]
del df['date']
df.set_index('datetime', inplace=True)
if order_book_id in h5:
data = np.array([
tuple(i) for i in chain(
h5[order_book_id][:], df.to_records())
],
dtype=h5[order_book_id].dtype)
del h5[order_book_id]
h5.create_dataset(order_book_id,
data=data,
**kwargs)
else:
h5.create_dataset(order_book_id,
data=df.to_records(),
**kwargs)
yield 1
def get_stock_data_from_mongo(self, code, cyc_type):
"""
:param str code: WindCode
:param cyc_type: Type from CycType
:return: numpy.ndarray
"""
logger.info('Load data from MongoDB: Code = {}, CycType = {}'.format(
code, cyc_type))
cursor = self.db[get_col_name(code)].find({
'cycType': cyc_type
}, {
'_id': False,
'date': True,
'open': True,
'close': True,
'high': True,
'low': True,
'volume': True,
'amount': True
}).sort("date", pymongo.ASCENDING)
pre_close = np.round(cursor.next()['close'], CONVERTER['close'].round)
data_num = cursor.count()
dtype = np.dtype([(f, FIELDS[f]) for f in FIELDS.keys()])
bars = np.zeros(shape=(data_num, ), dtype=dtype)
i = 0
for doc in cursor:
if cyc_type == CycType.CYC_DAY:
bars[i]['datetime'] = convert_date_to_int(doc['date'])
bars[i]['limit_up'] = np.round(
np.floor(pre_close * 11000) / 10000,
CONVERTER['limit_up'].round)
bars[i]['limit_down'] = np.round(
np.ceil(pre_close * 9000) / 10000,
CONVERTER['limit_down'].round)
elif cyc_type == CycType.CYC_MINUTE:
bars[i]['datetime'] = convert_dt_to_int(doc['date'])
else:
raise NotImplementedError
bars[i]['open'] = np.round(doc['open'], CONVERTER['open'].round)
bars[i]['close'] = np.round(doc['close'], CONVERTER['close'].round)
bars[i]['high'] = np.round(doc['high'], CONVERTER['high'].round)
bars[i]['low'] = np.round(doc['low'], CONVERTER['low'].round)
bars[i]['volume'] = doc['volume']
bars[i]['total_turnover'] = doc['amount']
pre_close = doc['close']
i += 1
logger.info(
'Load data from MongoDB finished: Code = {}, CycType = {}'.format(
code, cyc_type))
return bars
def get_bar(self, instrument, dt, frequency):
if frequency != '1d':
raise NotImplementedError
bars = self._all_day_bars_of(instrument)
if bars is None:
return
dt = np.uint64(convert_date_to_int(dt))
pos = bars['datetime'].searchsorted(dt)
if pos >= len(bars) or bars['datetime'][pos] != dt:
return None
return bars[pos]
def get_bar(self, instrument, dt, frequency):
if frequency != '1d':
raise NotImplementedError
bars = self._all_day_bars_of(instrument)
if bars is None:
return
dt = np.uint64(convert_date_to_int(dt))
pos = bars['datetime'].searchsorted(dt)
if pos >= len(bars) or bars['datetime'][pos] != dt:
return None
return bars[pos]
def get_bar(self, instrument, dt, frequency):
# type: (Instrument, Union[datetime, date], str) -> Optional[np.ndarray]
if frequency != '1d':
raise NotImplementedError
bars = self._all_day_bars_of(instrument)
if len(bars) <= 0:
return
dt = np.uint64(convert_date_to_int(dt))
pos = bars['datetime'].searchsorted(dt)
if pos >= len(bars) or bars['datetime'][pos] != dt:
return None
return bars[pos]
def gen_splits(d):
stocks = rqdatac.all_instruments().order_book_id.tolist()
split = rqdatac.get_split(stocks)
split['split_factor'] = split['split_coefficient_to'] / split[
'split_coefficient_from']
split = split[['split_factor']]
split.reset_index(inplace=True)
split.rename(columns={'ex_dividend_date': 'ex_date'}, inplace=True)
split['ex_date'] = [convert_date_to_int(d) for d in split['ex_date']]
split.set_index(['order_book_id', 'ex_date'], inplace=True)
with h5py.File(os.path.join(d, 'split_factor.h5'), 'w') as h5:
for order_book_id in split.index.levels[0]:
h5[order_book_id] = split.loc[order_book_id].to_records()
def adjust_bars(bars, ex_factors, fields, adjust_type, adjust_orig):
if ex_factors is None or len(bars) == 0:
return bars
dates = ex_factors['start_date']
ex_cum_factors = ex_factors['ex_cum_factor']
if adjust_type == 'pre':
adjust_orig_dt = np.uint64(convert_date_to_int(adjust_orig))
base_adjust_rate = _factor_for_date(dates, ex_cum_factors,
adjust_orig_dt)
else:
base_adjust_rate = 1.0
start_date = bars['datetime'][0]
end_date = bars['datetime'][-1]
if (_factor_for_date(dates, ex_cum_factors, start_date) == base_adjust_rate
and _factor_for_date(dates, ex_cum_factors,
end_date) == base_adjust_rate):
return bars
factors = ex_cum_factors.take(
dates.searchsorted(bars['datetime'], side='right') - 1)
# 复权
factors /= base_adjust_rate
if isinstance(fields, str):
if fields in PRICE_FIELDS:
bars[fields] *= factors
return bars
elif fields == 'volume':
bars[fields] *= (1 / factors)
return bars
# should not got here
return bars
result = np.copy(bars)
for f in result.dtype.names:
if f in PRICE_FIELDS:
result[f] *= factors
elif f == 'volume':
result[f] *= (1 / factors)
return result
def gen_ex_factor(d):
stocks = rqdatac.all_instruments().order_book_id.tolist()
ex_factor = rqdatac.get_ex_factor(stocks)
ex_factor.reset_index(inplace=True)
ex_factor['ex_date'] = [
convert_date_to_int(d) for d in ex_factor['ex_date']
]
ex_factor.rename(columns={'ex_date': 'start_date'}, inplace=True)
ex_factor.set_index(['order_book_id', 'start_date'], inplace=True)
ex_factor = ex_factor[['ex_cum_factor']]
dtype = ex_factor.loc[ex_factor.index.levels[0][0]].to_records().dtype
initial = np.empty((1, ), dtype=dtype)
initial['start_date'] = 0
initial['ex_cum_factor'] = 1.0
with h5py.File(os.path.join(d, 'ex_cum_factor.h5'), 'w') as h5:
for order_book_id in ex_factor.index.levels[0]:
h5[order_book_id] = np.concatenate(
[initial, ex_factor.loc[order_book_id].to_records()])
def __call__(self, path, fields, **kwargs):
with h5py.File(path, 'a') as h5:
for order_book_id in self._order_book_ids:
if order_book_id in h5:
try:
start_date = rqdatac.get_next_trading_date(
int(h5[order_book_id]['datetime'][-1] // 1000000))
except ValueError:
h5.pop(order_book_id)
start_date = START_DATE
else:
start_date = START_DATE
df = rqdatac.get_price(order_book_id,
start_date,
END_DATE,
'1d',
adjust_type='none',
fields=fields,
expect_df=True)
if not (df is None or df.empty):
df = df.loc[order_book_id]
df.reset_index(inplace=True)
df['datetime'] = [
convert_date_to_int(d) for d in df['date']
]
del df['date']
df.set_index('datetime', inplace=True)
if order_book_id in h5:
data = np.array([
tuple(i) for i in chain(h5[order_book_id][:],
df.to_records())
],
dtype=h5[order_book_id].dtype)
del h5[order_book_id]
h5.create_dataset(order_book_id, data=data, **kwargs)
else:
h5.create_dataset(order_book_id,
data=df.to_records(),
**kwargs)
yield 1
def adjust_bars(bars, ex_factors, fields, adjust_type, adjust_orig):
if ex_factors is None or len(bars) == 0:
return bars if fields is None else bars[fields]
dates = ex_factors['start_date']
ex_cum_factors = ex_factors['ex_cum_factor']
if adjust_type == 'pre':
adjust_orig_dt = np.uint64(convert_date_to_int(adjust_orig))
base_adjust_rate = _factor_for_date(dates, ex_cum_factors, adjust_orig_dt)
else:
base_adjust_rate = 1.0
start_date = bars['datetime'][0]
end_date = bars['datetime'][-1]
if (_factor_for_date(dates, ex_cum_factors, start_date) == base_adjust_rate and
_factor_for_date(dates, ex_cum_factors, end_date) == base_adjust_rate):
return bars if fields is None else bars[fields]
factors = ex_cum_factors.take(dates.searchsorted(bars['datetime'], side='right') - 1)
# 复权
factors /= base_adjust_rate
if isinstance(fields, str):
if fields in PRICE_FIELDS:
return bars[fields] * factors
elif fields == 'volume':
return bars[fields] * (1 / factors)
# should not got here
return bars[fields]
result = np.copy(bars if fields is None else bars[fields])
for f in result.dtype.names:
if f in PRICE_FIELDS:
result[f] *= factors
elif f == 'volume':
result[f] *= (1 / factors)
return result
def get_bar(self, instrument, dt, frequency):
# type: (Instrument, Union[datetime, date], str) -> Optional[np.ndarray]
if frequency not in ['1d', '1m', '5m', '15m']:
raise NotImplementedError
if frequency == '1d':
bars = self._all_day_bars_of(instrument)
if len(bars) <= 0:
return
dt = np.uint64(convert_date_to_int(dt))
pos = bars['datetime'].searchsorted(dt)
if pos >= len(bars) or bars['datetime'][pos] != dt:
return None
return bars[pos]
elif frequency == '1m':
dt_day_start = np.uint64(convert_date_to_int(dt))
dt_day_end = np.uint64(convert_date_to_int(dt + timedelta(days=1)))
bars = self._all_min_bars_of(instrument, dt_day_start, dt_day_end)
if len(bars) <= 0:
return
dt = np.uint64(convert_date_min_to_int(dt))
pos = bars['datetime'].searchsorted(dt)
if pos >= len(bars) or bars['datetime'][pos] != dt:
return None
return bars[pos]
elif frequency == '5m':
dt_day_start = np.uint64(convert_date_to_int(dt))
dt_day_end = np.uint64(convert_date_to_int(dt + timedelta(days=1)))
bars = self._all_min_bars_of(instrument, dt_day_start, dt_day_end)
if len(bars) <= 0:
return
merge_bar = self._merge_min_bar(dt, bars, 5)
return merge_bar
elif frequency == '15m':
dt_day_start = np.uint64(convert_date_to_int(dt))
dt_day_end = np.uint64(convert_date_to_int(dt + timedelta(days=1)))
bars = self._all_min_bars_of(instrument, dt_day_start, dt_day_end)
if len(bars) <= 0:
return
merge_bar = self._merge_min_bar(dt, bars, 15)
return merge_bar
def update(self, context):
for stock in context.stocks:
if stock in self.stock_pool:
#忽略已经在股票池中的股票
continue
# 读取历史数据
prices = history_bars(stock, context.DECLINE_TIME_PERIOD + 1, '1d',None,True,False,'post')
if prices.size < (context.DECLINE_TIME_PERIOD+1):
continue
#判断股价当天是否出现最小值
stock_low = prices['low']
MinPrice = np.min(stock_low)
MinPrice_index = stock_low.argmin()
date_time = prices['datetime']
DateTimeMin = date_time[MinPrice_index]
dt = np.uint64(convert_date_to_int(context.now))
if dt != DateTimeMin:
continue
# 如果跌到黄金分割位则加入stock pool
stock_high = prices['high']
MaxPrice = np.max(stock_high)
golden_price = MaxPrice * context.GOLDEN_RATIO
close_price = prices[-1]['close']
if not stock_price_equal(MinPrice, golden_price, context.UNCERTAINTY_RATE):
continue
prices318 = history_bars(stock, 318, '1d', None, True, False, 'post')
if prices318.size < (318):
continue
ma318 = sum(prices318['close']) / 318
prices110 = history_bars(stock, 110, '1d', None, True, False, 'post')
if prices110.size < (110):
continue
ma110 = sum(prices110['close'])/110
prices250 = history_bars(stock, 250, '1d', None, True, False, 'post')
if prices250.size < (250):
continue
ma250 = sum(prices250['close']) / 250
# if ma120<ma318 or ma318>ma250 or ma120<ma250:
# continue
if not (stock_price_equal(MinPrice, ma110, context.UNCERTAINTY_RATE) or stock_price_equal(MinPrice, ma250, context.UNCERTAINTY_RATE) \
or stock_price_equal(MinPrice, ma318, context.UNCERTAINTY_RATE)):
continue
# if stock not in context.sample['StockID'].values:
if True:
self.stock_pool[stock] = DailyStockStatus(stock[0:6],golden_price)
self.stock_pool[stock].DateTimeMin = DateTimeMin
MaxPrice_index = np.where(stock_high == MaxPrice)[-1][-1]
self.stock_pool[stock].DateTimeMax = date_time[MaxPrice_index]
self.stock_pool[stock].MinPrice = MinPrice
self.stock_pool[stock].MaxPrice = MaxPrice
#logger.info(stock + " added to self selected stock pool")
#每天更新加入到了股票池里面的每只股票的状态
for stock in list(self.stock_pool.keys()):
# 读取历史数据
prices = history_bars(stock, context.DECLINE_TIME_PERIOD + 1, '1d',None,True,False,'post')
low_price = prices[-1]['low']
if low_price < self.stock_pool[stock].MinPrice:
# 股价创出新低，更新股票最低价
self.stock_pool[stock].MinPrice = low_price
# del self.stock_pool[stock]
dt = np.uint64(convert_date_to_int(context.now))
self.stock_pool[stock].DateTimeMin = dt
logger.info(stock + " falling with new lowest price")
else:
sampledata={'StockID':stock,
# 'GoldenPrice':self.stock_pool[stock].GoldenPrice,
'MinPrice':self.stock_pool[stock].MinPrice,
'DateTimeMin': [int(self.stock_pool[stock].DateTimeMin / 1000000)],
'DateTimeMax':[int(self.stock_pool[stock].DateTimeMax/1000000)],
# 'MaxPrice':self.stock_pool[stock].MaxPrice,
'Change':'0%',
'symbol':context.stocksmap[stock],
'Downdays':0,
'MaxDrawdown':0,
'DateDownMin':0,
'Updays':0,
'MaxChange':0,
'DateUpMax':0}
context.sample=context.sample.append(pd.DataFrame(sampledata),ignore_index=False)
context.sample.reset_index(drop=True, inplace=True)
# context.sample.set_index('DateTimeMin')
del self.stock_pool[stock]
# context.MonitoringDays
Index2BeDeleted = []
for index, row in context.sample.iterrows():
#1. 获取增长最大幅度，以及增长天数
DateTimeGolden = row['DateTimeMin']
DateNow = str(int((convert_date_to_int(context.now))/1000000))
(DateTimeGoldenPrice2Now,) = get_trading_dates(str(DateTimeGolden), end_date=DateNow).shape
if DateTimeGoldenPrice2Now>context.MonitoringDays:
Index2BeDeleted.append(index)
continue
prices = history_bars(row['StockID'], DateTimeGoldenPrice2Now, '1d',None,True,False,'post')
stock_high = prices['high']
MaxPrice = np.max(stock_high)
# stock_low = prices['low']
# MinPrice = np.min(stock_low)
MinPrice = row['MinPrice']
date_time = prices['datetime']
# date_time = prices['datetime']
MaxPrice_index = np.where(stock_high == MaxPrice)[-1][-1]
# MinPrice_index = np.where(stock_low == MinPrice)[-1][-1]
DateTimeMax = int(date_time[MaxPrice_index]/1000000)
(UpdayNums,) = get_trading_dates(str(DateTimeGolden), str(DateTimeMax)).shape
UpdayNums = UpdayNums-1
Updays_index = list(context.sample.columns).index('Updays')
context.sample.iloc[index,Updays_index] = UpdayNums
if UpdayNums>0:
MaxChange = str(round(((MaxPrice-MinPrice)/MinPrice)*100,2))+'%'
MaxChange_index = list(context.sample.columns).index('MaxChange')
context.sample.iloc[index,MaxChange_index] = MaxChange
DateUpMax_index = list(context.sample.columns).index('DateUpMax')
context.sample.iloc[index,DateUpMax_index] = int(DateTimeMax)
#2. 获取下跌最大幅度以及下跌天数
DateNow = str(int((convert_date_to_int(context.now))/1000000))
(DateNumMinePrice2Now,) = get_trading_dates(str(DateTimeGolden), end_date=DateNow).shape
prices = history_bars(row['StockID'], DateNumMinePrice2Now, '1d',None,True,False,'post')
stock_low = prices['low']
MinPrice = np.min(stock_low)
# stock_low = prices['low']
# MinPrice = np.min(stock_low)
GoldenPrice = row['MinPrice']
date_time = prices['datetime']
# date_time = prices['datetime']
MinPrice_index = np.where(stock_low == MinPrice)[-1][-1]
# MinPrice_index = np.where(stock_low == MinPrice)[-1][-1]
DateTimeMin = int(date_time[MinPrice_index]/1000000)
# if (row['StockID']=='300631.XSHE'):
# DateTimeMin = DateTimeMin
(DowndayNums,) = get_trading_dates(str(DateTimeGolden), str(DateTimeMin)).shape
DowndayNums = DowndayNums-1
Downdays_index = list(context.sample.columns).index('Downdays')
context.sample.iloc[index,Downdays_index] = DowndayNums
if DowndayNums>0:
MaxDrawdown = str(round(((MinPrice-GoldenPrice)/GoldenPrice)*100,2))+'%'
MaxDrawdown_index = list(context.sample.columns).index('MaxDrawdown')
context.sample.iloc[index,MaxDrawdown_index] = MaxDrawdown
DateDownMin_index = list(context.sample.columns).index('DateDownMin')
context.sample.iloc[index,DateDownMin_index] = int(DateTimeMin)
#3. 更新每天股价
MinPrice = row['MinPrice']
Close = prices[-1]['close']
Change = str(round(((Close-MinPrice)/MinPrice)*100,2))+'%'
Change_index = list(context.sample.columns).index('Change')
context.sample.iloc[index,Change_index] = Change
if len(Index2BeDeleted)>0:
context.sample.drop(index=Index2BeDeleted)
def update(self, context):
for stock in context.stocks:
if stock in self.stock_pool:
#忽略已经在股票池中的股票
continue
# 读取历史数据
prices = history_bars(stock, context.DECLINE_TIME_PERIOD + 1, '1d',
None, True, False, 'post') #not defined?
if prices.size < 35:
continue
#判断股价当天是否出现最小值
stock_low = prices['low']
MinPrice = np.min(stock_low)
MinPrice_index = stock_low.argmin()
date_time = prices['datetime']
DateTimeMin = date_time[MinPrice_index]
dt = np.uint64(convert_date_to_int(context.now))
if dt != DateTimeMin:
continue
# 如果跌到黄金分割位则加入stock pool
stock_high = prices['high']
MaxPrice = np.max(stock_high)
golden_price = MaxPrice * context.GOLDEN_RATIO
close_price = prices[-1]['close']
if not stock_price_equal(MinPrice, golden_price,
context.UNCERTAINTY_RATE):
continue
self.stock_pool[stock] = DailyStockStatus(stock[0:6], golden_price)
self.stock_pool[stock].BottomDays = 1
self.stock_pool[stock].DateTimeMin = DateTimeMin
MaxPrice_index = np.where(stock_high == MaxPrice)[-1][-1]
self.stock_pool[stock].DateTimeMax = date_time[MaxPrice_index]
self.stock_pool[stock].MinPrice = MinPrice
self.stock_pool[stock].MaxPrice = MaxPrice
#logger.info(stock + " added to self selected stock pool")
#每天更新加入到了股票池里面的每只股票的状态
for stock in list(self.stock_pool.keys()):
# 读取历史数据
prices = prices = history_bars(stock,
context.DECLINE_TIME_PERIOD + 1,
'1d', None, True, False, 'post')
stock_status = self.stock_pool[stock]
#更新底部驻留天数
self.stock_pool[stock].BottomDays += 1
# 判断底部成交量是否放大2倍
if stock_status.VolumeBoosted == False:
stock_volume = prices['volume']
today_volume = stock_volume[-1]
today_volume_avr = talib.SMA(stock_volume, 5)
today_volume_avr = today_volume_avr[-1]
if today_volume > (today_volume_avr * 2):
self.stock_pool[stock].VolumeBoosted = True
logger.info(stock + " bottom boosted 2 times")
BoostRate = today_volume / today_volume_avr
self.stock_pool[stock].BoostRate = BoostRate
high_price = prices[-1]['high']
low_price = prices[-1]['low']
if low_price < self.stock_pool[stock].MinPrice:
# 股价创出新低，更新股票最低价
self.stock_pool[stock].MinPrice = low_price
# del self.stock_pool[stock]
dt = np.uint64(convert_date_to_int(context.now))
self.stock_pool[stock].DateTimeMin = dt
logger.info(stock + " falling with new lowest price")
if (high_price > (self.stock_pool[stock].MinPrice * 1.2)) and (
self.stock_pool[stock].TrendRevered == False):
# 股票从底部最低点增长20%，趋势可能发生反转
self.stock_pool[stock].TrendRevered = True
self.stock_pool[stock].ReversedHight = high_price
self.stock_pool[stock].DateRevserdHight = np.uint64(
convert_date_to_int(context.now))
#logger.info(stock + " trend posible reversed"+ " DateTimeMax: "+str(self.stock_pool[stock].DateTimeMax)+ " DateTimeMin: "+str(self.stock_pool[stock].DateTimeMin)+" MaxPrice:"+str(self.stock_pool[stock].MaxPrice)+" MinPrice:"+str(self.stock_pool[stock].MinPrice)+" decresed: "+str(((self.stock_pool[stock].MaxPrice-self.stock_pool[stock].MinPrice))/self.stock_pool[stock].MaxPrice))
if self.stock_pool[stock].TrendRevered and self.stock_pool[
stock].ReversedStockPeriod != 0:
if high_price > self.stock_pool[stock].ReversedHight:
self.stock_pool[stock].ReversedHight = high_price
self.stock_pool[stock].ReversedStockPeriod = 8
self.stock_pool[stock].DateRevserdHight = np.uint64(
convert_date_to_int(context.now))
self.stock_pool[stock].ReversedStockPeriod -= 1
if self.stock_pool[stock].TrendRevered and self.stock_pool[
stock].ReversedStockPeriod == 0:
self.stock_pool[stock].decresed_ratio = (
(self.stock_pool[stock].MaxPrice -
self.stock_pool[stock].MinPrice)
) / self.stock_pool[stock].MaxPrice
self.stock_pool[stock].ReversedRatio = (
(self.stock_pool[stock].ReversedHight -
self.stock_pool[stock].MinPrice)
) / self.stock_pool[stock].MinPrice
logger.info(stock + " trend reversed" + " DateTimeMax: " +
str(self.stock_pool[stock].DateTimeMax) +
" DateTimeMin: " +
str(self.stock_pool[stock].DateTimeMin) +
" MaxPrice:" +
str(self.stock_pool[stock].MaxPrice) +
" MinPrice:" +
str(self.stock_pool[stock].MinPrice) +
" decresed: " +
str(((self.stock_pool[stock].MaxPrice -
self.stock_pool[stock].MinPrice)) /
self.stock_pool[stock].MaxPrice))
self.stock_pool[stock].DecreasedRatio = (
(self.stock_pool[stock].MaxPrice -
self.stock_pool[stock].MinPrice)
) / self.stock_pool[stock].MaxPrice
logger.info(stock + " trend reversed" + " DateTimeMin: " +
str(self.stock_pool[stock].DateTimeMin) +
" MinPrice:" +
str(self.stock_pool[stock].MinPrice) +
" ReversedHight:" +
str(self.stock_pool[stock].ReversedHight) +
" reversed: " +
str(((self.stock_pool[stock].ReversedHight -
self.stock_pool[stock].MinPrice)) /
self.stock_pool[stock].MinPrice))
self.stock_pool[stock].ReversedRatio = (
(self.stock_pool[stock].ReversedHight -
self.stock_pool[stock].MinPrice)
) / self.stock_pool[stock].MinPrice
#context.sample[context.sample_id] = self.stock_pool[stock]
sampledata = {
'BottomDays':
self.stock_pool[stock].BottomDays,
'VolumeBoosted':
self.stock_pool[stock].VolumeBoosted,
'GoldenPrice':
self.stock_pool[stock].GoldenPrice,
'BoostRate':
self.stock_pool[stock].BoostRate,
'MinPrice':
self.stock_pool[stock].MinPrice,
'TrendRevered':
self.stock_pool[stock].TrendRevered,
'DateTimeMax':
[int(self.stock_pool[stock].DateTimeMax / 1000000)],
'DateTimeMin':
[int(self.stock_pool[stock].DateTimeMin / 1000000)],
'MaxPrice':
self.stock_pool[stock].MaxPrice,
'ReversedHight':
self.stock_pool[stock].ReversedHight,
'DateRevserdHight':
[int(self.stock_pool[stock].DateRevserdHight / 1000000)],
'ReversedStockPeriod':
self.stock_pool[stock].ReversedStockPeriod,
'DecreasedRatio':
self.stock_pool[stock].DecreasedRatio,
'ReversedRatio':
self.stock_pool[stock].ReversedRatio
}
context.sample = context.sample.append(
pd.DataFrame(sampledata))
del self.stock_pool[stock]
def history_bars(self,
instrument,
bar_count,
frequency,
fields,
dt,
skip_suspended=True,
include_now=False,
adjust_type='pre',
adjust_orig=None):
if frequency not in ['1m', '5m', '15m', '1d']:
raise NotImplementedError
if skip_suspended and instrument.type == 'CS':
bars = self._filtered_day_bars(instrument)
elif frequency == '1d':
bars = self._all_day_bars_of(instrument)
elif frequency in ['1m', '5m', '15m']:
dt_end = np.uint64(convert_date_to_int(dt + timedelta(days=1)))
dt_start = np.uint(
convert_date_to_int(dt - timedelta(days=int(bar_count / 180) +
30))) # @TODO
bars = self._all_min_bars_of(instrument, dt_start, dt_end)
if not self._are_fields_valid(fields, bars.dtype.names):
raise RQInvalidArgument("invalid fileds: {}".format(fields))
if len(bars) <= 0:
return bars
if frequency == '1d':
dt = np.uint64(convert_date_to_int(dt))
elif frequency in ['1m', '5m', '15m']:
dt = np.uint64(convert_date_min_to_int(dt))
count = int(frequency[:-1])
i = bars['datetime'].searchsorted(dt, side='right')
if i <= 0:
return None
left = i - bar_count * count if i >= bar_count * count else 0
if count > 1:
merge_bars = []
for j in range(0, bar_count):
if left + (j + 1) * count > i:
break
merge_bar = self._history_merge_min_bar(
bars[left + j * count:left + (j + 1) * count])
merge_bars.append(merge_bar)
bars = np.array(merge_bars)
else:
bars = bars[left:i]
if adjust_type == 'none' or instrument.type in {'Future', 'INDX'}:
# 期货及指数无需复权
return bars if fields is None else bars[fields]
if isinstance(fields, str) and fields not in FIELDS_REQUIRE_ADJUSTMENT:
return bars if fields is None else bars[fields]
return adjust_bars(bars,
self.get_ex_cum_factor(instrument.order_book_id),
fields, adjust_type, adjust_orig)
def raw_history_bars(self,
instrument,
frequency,
start_dt=None,
end_dt=None,
length=None):
if frequency[-1] == "m":
days = []
if start_dt and end_dt:
assert start_dt <= end_dt, "start datetime later then end datetime!"
s_date, s_time = start_dt.date(), start_dt.time()
e_date, e_time = end_dt.date(), end_dt.time()
s_date_int = convert_date_to_int(s_date)
e_date_int = convert_date_to_int(e_date)
dates = self._dates_index(instrument)
s_pos = safe_searchsorted(dates, s_date_int)
e_pos = safe_searchsorted(dates, e_date_int, side="right") - 1
if s_pos == e_pos:
days.append(
dict(
trade_date=convert_int_to_date(dates[s_pos]),
start_time=s_time,
end_time=e_time,
))
else:
days.append(
dict(trade_date=convert_int_to_date(dates[s_pos]),
start_time=s_time))
days.extend(
map(lambda x: dict(trade_date=convert_int_to_date(x)),
dates[s_pos + 1:e_pos]))
days.append(
dict(trade_date=convert_int_to_date(dates[e_pos]),
end_time=e_time))
post_handler = lambda x: x
elif start_dt and length:
s_date, s_time = start_dt.date(), int(
start_dt.strftime("%H%M%S"))
dates = self._dates_index(instrument)
s_date_int = convert_date_to_int(s_date)
s_pos = safe_searchsorted(dates, s_date_int)
s_bar_count = self.get_bar_count_in_day(instrument,
frequency,
trade_date=s_date,
start_time=s_time)
total_bar_count = self.get_bar_count_in_day(
instrument, frequency)
extra_days = (max(length - s_bar_count, 0) -
1) // total_bar_count + 1
days.append(dict(trade_date=s_date, start_time=s_time))
days.extend(
map(lambda x: dict(trade_date=convert_int_to_date(x)),
dates[s_pos + 1:s_pos + 1 + extra_days]))
post_handler = lambda x: x[:length]
elif end_dt and length:
e_date, e_time = end_dt.date(), int(end_dt.strftime("%H%M%S"))
dates = self._dates_index(instrument)
e_date_int = convert_date_to_int(e_date)
e_pos = safe_searchsorted(dates, e_date_int, side="right") - 1
e_bar_count = self.get_bar_count_in_day(instrument,
frequency,
trade_date=e_date,
end_time=e_time)
total_bar_count = self.get_bar_count_in_day(
instrument, frequency)
extra_days = (max(length - e_bar_count, 0) -
1) // total_bar_count + 1
days.extend(
map(lambda x: dict(trade_date=convert_int_to_date(x)),
dates[max(e_pos - extra_days, 0):e_pos]))
days.append(dict(trade_date=e_date, end_time=e_time))
post_handler = lambda x: x[-length:]
else:
raise RuntimeError(
"At least two of [start_dt,end_dt,length] should be given."
)
data = post_handler(
self._get_bars_in_days(instrument, frequency, days))
return data
else:
return None
