def _should_access_shm(self, security, dt):
# 考虑商品期货的夜盘。
from jqdata.stores.calendar_store import get_calendar_store
cal = get_calendar_store()
trade_date = cal.get_current_trade_date(
security, dt)
return self._today <= trade_date
def get_marginsec_stocks(date=None):
if not date:
calendar_store = get_calendar_store()
date = calendar_store.get_previous_trade_date(None, _get_today())
else:
date = convert_date(date)
return jqdata.apis.get_marginsec_stocks(date)
def calc_minute_indexs(security,
end_dt,
start_dt=None,
count=None,
skip_paused=True,
include_now=True):
'''
计算分钟索引: 返回 numpy.ndarray, 每一项为时间戳
'''
if skip_paused:
if count:
minutes = get_minute_by_count(security,
end_dt,
count,
include_now=include_now)
assert len(minutes) <= count
if len(minutes) == count:
return minutes
# 补全
miss = count - len(minutes)
calendar = get_calendar_store()
prev_minutes = [(calendar.first_date_timestamp - i * 60)
for i in xrange(miss, 0, -1)]
minutes = np.concatenate((prev_minutes, minutes))
return minutes
else:
minutes = get_minute_by_period(security,
start_dt,
end_dt,
include_now=include_now)
return minutes
else:
calendar = get_calendar_store()
minutes = calendar.get_trade_minutes_by_enddt(security,
end_dt,
start_dt=start_dt,
count=count,
include_now=include_now)
if count and len(minutes) < count:
miss = count - len(minutes)
prev_minutes = [(calendar.first_date_timestamp - i * 60)
for i in xrange(miss, 0, -1)]
minutes = np.concatenate((prev_minutes, minutes))
return minutes
else:
return minutes
def get_dominant_future(dt, underlying_symbol):
"""获取某一期货品种策略当前日期的主力合约代码"""
dt = convert_dt(dt)
check_string(underlying_symbol)
store = get_dominant_future_store()
calendar_store = get_calendar_store()
current_trade_date = calendar_store.get_current_trade_date(
get_security_store().get_security('AU9999.XSGE'), dt)
return store.get_dominant_code(current_trade_date, underlying_symbol)
def get_dominant_future(underlying_symbol, dt=None):
if dt and convert_date(dt) != _get_today():
dt = dt
else:
calendar_store = get_calendar_store()
dt = calendar_store.get_previous_trade_date(
get_security_store().get_security('AU9999.XSGE'), _get_today())
return jqdata.apis.get_dominant_future(dt, underlying_symbol)
pass
def get_fundamentals_continuously(query_object,
end_date=None,
count=1): # noqa
if end_date is None:
cal_store = get_calendar_store()
stock = SecurityStore.instance().get_security('000001.XSHE')
end_date = cal_store.get_previous_trade_date(stock, _get_today())
return jqdata.apis.get_fundamentals_continuously(query_object,
end_date=end_date,
count=count)
def get_fundamentals(query_object, date=None, statDate=None): # noqa
if date and statDate:
raise Exception('date和statDate参数只能输入一个')
if date is None and statDate is None:
cal_store = get_calendar_store()
stock = SecurityStore.instance().get_security('000001.XSHE')
date = cal_store.get_previous_trade_date(stock, _get_today())
if date:
yestoday = datetime.date.today() - datetime.timedelta(days=1)
date = min(parse_date(date), yestoday)
return jqdata.apis.get_fundamentals(query_object,
date=date,
statDate=statDate)
def calc_daily_indexs(security,
end_date,
start_date=None,
count=None,
skip_paused=True,
include_now=True):
'''
计算日期索引
'''
if skip_paused:
if count:
dates = get_date_by_count(security, end_date, count)
assert len(dates) <= count
if len(dates) == count:
return dates
# 补全
calendar = get_calendar_store()
miss = count - len(dates)
prev_dates = [(calendar.first_date_timestamp - i * 86400)
for i in xrange(miss, 0, -1)]
dates = np.concatenate((prev_dates, dates))
return dates
else:
dates = get_date_by_period(security, start_date, end_date)
return dates
else:
calendar = get_calendar_store()
dates = calendar.get_trade_days_by_enddt(security,
end_date,
start_date=start_date,
count=count)
if count and len(dates) < count:
miss = count - len(dates)
prev_dates = [(calendar.first_date_timestamp - i * 86400)
for i in xrange(miss, 0, -1)]
dates = np.concatenate((prev_dates, dates))
return dates
else:
return dates
def get_future_contracts(dt, underlying_symbol):
"""期货可交易合约列表"""
dt = convert_dt(dt)
check_string(underlying_symbol)
underlying_symbol = underlying_symbol.upper()
store = get_security_store()
calendar_store = get_calendar_store()
current_trade_date = calendar_store.get_current_trade_date(
get_security_store().get_security('AU9999.XSGE'), dt)
futures = store.get_all_securities(['futures'], current_trade_date)
all_code = list(futures.index)
all_code.sort()
code = []
for n in all_code:
if '9999' not in n and '8888' not in n:
res = re.findall(r"(.*)[0-9]{4}\.[a-zA-Z]+$", n)
if len(res) == 1 and underlying_symbol == res[0].upper():
code.append(n.upper())
return code
def get_trade_days(start_date=None, end_date=None, count=None):
if start_date and count:
raise ParamsError("start_date 参数与 count 参数只能二选一")
if not (count is None or count > 0):
raise ParamsError("count 参数需要大于 0 或者为 None")
if not end_date:
end_date = datetime.date.today()
else:
end_date = parse_date(end_date)
store = get_calendar_store()
if start_date:
start_date = parse_date(start_date)
return store.get_trade_days_between(start_date, end_date)
elif count is not None:
return store.get_trade_days_by_count(end_date, count)
else:
raise ParamsError("start_date 参数与 count 参数必须输入一个")
def calc_factors(securities, factors, start_date, end_date):
"""
securities: a list of str, security code
factors: a list of Factor objects
return a pd.Panel, axes: ['security', 'date', 'factor']
"""
start_date = convert_date(start_date)
end_date = convert_date(end_date)
indus_type = ['sw_l1', 'sw_l2', 'sw_l3', 'zjw', 'jq_l1', 'jq_l2']
industry_raw = []
for it in indus_type:
ids = get_industries(it)
for ds in ids.index:
industry_raw.append(ds)
markets_raw = ['open', 'close', 'high', 'low', 'volume', 'money']
name_fac_dic = {f.name: f for f in factors}
industry = []
markets = ['close']
fundamentals = {}
# 将因子排序
# 将依赖的market和finance/industry因子筛选出
fac_dep_dic = {}
for f in factors:
dep_list = []
for fd in f.dependencies:
if fd in industry_raw:
industry.append(fd)
elif fd in markets_raw:
markets.append(fd)
elif fd in set(finance.keys()):
fundamentals[fd] = finance[fd]
else:
dep_list.append(fd)
fac_dep_dic[f.name] = dep_list
fac_name_sorted = factor_sort(fac_dep_dic, fac_dep_dic.keys())
industry = list(set(industry))
markets = list(set(markets))
# 计算最大窗口
fac_length = {}
for n in fac_name_sorted:
fac_window = name_fac_dic[n].max_window
fac_max = fac_window
for d in name_fac_dic[n].dependencies:
if d in fac_length:
if fac_window + fac_length[d] > fac_max:
fac_max = fac_window + fac_length[d]
fac_length[n] = fac_max
len_value = fac_length.values()
max_len = max(len_value)
# 获取基础数据
cal_store = get_calendar_store()
days = cal_store.get_trade_days_between(start_date, end_date)
days = cal_store.get_trade_days_by_count(end_date, len(days) + max_len)
if len(fundamentals) > 0:
q = query(valuation.code, *fundamentals.values()).filter(
valuation.code.in_(securities))
d = {day: get_fundamentals(q, date=day) for day in days}
for df in d.values():
df.index = df.code.values
del df['code']
pnl = pd.Panel(d)
pnl = pnl.transpose(2, 0, 1)
if len(markets) == 0:
data = pnl
if len(markets) > 0:
p = get_price(securities,
days[0],
days[-1],
'1d',
markets,
fq='post',
pre_factor_ref_date=days[-1])
p.major_axis = [date(x.year, x.month, x.day) for x in p.major_axis]
if len(fundamentals) == 0:
data = p
if len(fundamentals) * len(markets) > 0:
data = pd.concat([pnl, p])
# 获取行业数据-每个行业作为一个因子
fac_dict = {}
if len(industry) > 0:
for ds in industry:
fac_dict[ds] = pd.DataFrame(index=[dt for dt in days],
columns=securities)
fac_dict[ds] = fac_dict[ds].fillna(0)
sec_code = industry_store.get_security_industry_date(
securities, industry)
for dd in days:
for sc in sec_code:
if str(dd) >= sc['stock_startdate'] and str(
dd) <= sc['stock_enddate']:
fac_dict[sc['code']][sc['stock']][dd] = 1
# 填充dict
for item in data.items:
fac_dict[item] = data[item]
for fac in fac_name_sorted:
fac_dict[fac] = pd.DataFrame(index=[dt for dt in days],
columns=securities)
# 转换index格式为timestamp
for df in fac_dict:
fac_dict[df].index = pd.to_datetime(fac_dict[df].index)
# 因子计算
for fac in fac_name_sorted:
factor = name_fac_dic[fac]
for dix in range(max_len, len(days)):
dic_tmp = {
df: fac_dict[df][dix - factor.max_window:dix]
for df in fac_dict
}
calc_f = factor.calc(dic_tmp)
if calc_f is not None:
fac_dict[fac].iloc[dix] = calc_f
fac_dict = {df: fac_dict[df][max_len:] for df in fac_dict}
return fac_dict
pass
def __init__(self):
self.cal_store = get_calendar_store()
self.sec_store = get_security_store()
self.idx_store = get_index_store()
self.idst_store = IndustryStore_Sqlite()
self.extended_store = ExtendedDataGetter()
def get_all_trade_days():
store = get_calendar_store()
return store.get_all_trade_days()
def get_price_daily_single(security,
end_date=None,
fields=None,
start_date=None,
count=None,
skip_paused=True,
fq=None,
include_now=True,
pre_factor_ref_date=None):
fields = list(fields)
if 'price' in fields:
fields = replace_list(fields, 'price', 'avg')
fields = list(set(fields)) # 去掉重复的列
if fq and 'factor' not in fields:
fields.append('factor')
if not count:
assert start_date is not None
cols_dict, dates = fetch_daily_data(security,
end_date,
fields,
start_date=start_date,
count=count,
skip_paused=skip_paused,
fq=fq,
include_now=include_now)
if fq:
factors = cols_dict['factor']
if fq == 'pre':
assert pre_factor_ref_date, '请设置前复权基准日期'
f = get_factor_by_date(security, pre_factor_ref_date)
if f and abs(f - 1.0) >= 1e-6:
factors = copy_if_readonly(factors) / f
price_decimals = security.price_decimals
for f in fields:
if f in [
'open', 'close', 'high', 'low', 'price', 'avg',
'pre_close', 'high_limit', 'low_limit'
]:
cols_dict[f] *= factors
np.round(cols_dict[f], price_decimals, cols_dict[f])
elif f in ['volume']:
cols_dict[f] /= factors
np.round(cols_dict[f], 0, cols_dict[f])
if 'factor' in fields:
if fq:
cols_dict['factor'] = factors
else:
cols_dict['factor'] = np.full(len(dates), 1.0)
# 需要返回的行数
return_dates = calc_daily_indexs(security,
end_date,
start_date=start_date,
count=count,
skip_paused=skip_paused,
include_now=include_now)
return_count = len(return_dates)
if len(dates) < return_count:
# pad nan
miss = return_count - len(dates)
prev = np.full(miss, nan)
for f in fields:
cols_dict[f] = np.concatenate((prev, cols_dict[f]))
calendar = get_calendar_store()
prev_dates = [(calendar.first_date_timestamp - i * 86400)
for i in xrange(miss, 0, -1)]
dates = np.concatenate((prev_dates, dates))
return cols_dict, dates
def fetch_minute_data(security,
end_dt,
fields,
start_dt=None,
count=None,
skip_paused=True,
fq=None,
include_now=True):
# 需要返回的行数
return_count = 0
if skip_paused:
if count:
return_count = count
cols_dict = get_minute_bar_by_count(security,
end_dt,
count,
fields,
include_now=include_now)
minutes = cols_dict.pop('date', EMPTY_ARRAY)
if len(minutes) < count:
miss = return_count - len(minutes)
prev = np.full(miss, nan)
for f in fields:
cols_dict[f] = np.concatenate((prev, cols_dict[f]))
calendar = get_calendar_store()
prev_minutes = [(calendar.first_date_timestamp - i * 60)
for i in xrange(miss, 0, -1)]
minutes = np.concatenate((prev_minutes, minutes))
else:
cols_dict = get_minute_bar_by_period(security,
start_dt,
end_dt,
fields,
include_now=include_now)
minutes = cols_dict.pop('date', EMPTY_ARRAY)
return_count = len(minutes)
if len(minutes) == 0 and return_count <= 0:
cols_dict = {col: EMPTY_ARRAY for col in fields}
return cols_dict, EMPTY_ARRAY
else:
calendar = get_calendar_store()
valid_trade_minutes = calendar.get_trade_minutes_by_enddt(
security,
end_dt,
start_dt=start_dt,
count=count,
include_now=include_now)
if count:
# assert len(valid_trade_minutes) == count
return_count = count
else:
return_count = len(valid_trade_minutes)
if return_count <= 0:
cols_dict = {col: EMPTY_ARRAY for col in fields}
return cols_dict, EMPTY_ARRAY
# 需要上一个交易日的close数据填充
if 'close' not in fields:
cols_dict = get_minute_bar_by_count(security,
end_dt,
return_count + 1,
fields + ['close'],
include_now=include_now)
else:
cols_dict = get_minute_bar_by_count(security,
end_dt,
return_count + 1,
fields,
include_now=include_now)
minutes = cols_dict.pop('date', EMPTY_ARRAY)
if len(minutes) == 0:
cols_dict = {col: np.full(return_count, nan) for col in fields}
return cols_dict, valid_trade_minutes
mask = np.in1d(valid_trade_minutes, minutes, assume_unique=True)
if not np.all(mask):
cols_dict = fill_paused(security, cols_dict, minutes,
valid_trade_minutes, fields)
minutes = valid_trade_minutes
else:
start = np.searchsorted(minutes, valid_trade_minutes[0])
if start > 0:
minutes = minutes[start:]
for col in cols_dict:
cols_dict[col] = cols_dict[col][start:]
return cols_dict, minutes
def fetch_daily_data(security,
end_date,
fields,
start_date=None,
count=None,
skip_paused=True,
fq=None,
include_now=True):
# 需要返回的行数
return_count = None
if skip_paused:
if count:
return_count = count
cols_dict = get_daily_bar_by_count(security,
end_date,
count,
fields,
include_now=include_now)
dates = cols_dict.pop('date', EMPTY_ARRAY)
else:
cols_dict = get_daily_bar_by_period(security,
start_date,
end_date,
fields,
include_now=include_now)
dates = cols_dict.pop('date', EMPTY_ARRAY)
return_count = len(dates)
if len(dates) == 0: # and return_count == 0:
cols_dict = {col: EMPTY_ARRAY for col in fields}
return cols_dict, EMPTY_ARRAY
else:
calendar = get_calendar_store()
valid_trade_days = calendar.get_trade_days_by_enddt(
security,
end_date,
start_date=start_date,
count=count,
include_now=include_now)
if count:
# assert len(valid_trade_days) <= count
return_count = count
else:
return_count = len(valid_trade_days)
if return_count <= 0:
cols_dict = {col: EMPTY_ARRAY for col in fields}
return cols_dict, EMPTY_ARRAY
max_count = len(valid_trade_days) + 1
# 需要上一个交易日的close数据填充
if 'close' not in fields:
cols_dict = get_daily_bar_by_count(security,
end_date,
max_count,
fields + ['close'],
include_now=include_now)
else:
cols_dict = get_daily_bar_by_count(security,
end_date,
max_count,
fields,
include_now=include_now)
dates = cols_dict.pop('date', [])
if len(dates) == 0:
cols_dict = {
col: np.full(len(valid_trade_days), nan)
for col in fields
}
return cols_dict, valid_trade_days
mask = np.in1d(valid_trade_days, dates, assume_unique=True)
# 发生了停牌
if not np.all(mask):
cols_dict = fill_paused(security, cols_dict, dates,
valid_trade_days, fields)
dates = valid_trade_days
else:
start = np.searchsorted(dates, valid_trade_days[0])
if start > 0:
dates = dates[start:]
for col in cols_dict:
cols_dict[col] = cols_dict[col][start:]
return cols_dict, dates
