def QA_data_futuretick_resample_min(tick,
type_='1min',
source='tick_resample',
if_drop=True):
"""tick采样成1分钟级别分钟线，支持QA_DataAggrement_future_transaction数据
Arguments:
tick {[type]} -- transaction
Returns:
[type] -- [description]
"""
tick = tick.sort_values(by=['datetime', 'code'])
tick.index = pd.to_datetime(tick.datetime)
data = tick.resample(type_, closed='left', base=30, loffset=type_).apply({
'price':
'ohlc',
'volume':
'sum',
'zengcang':
'sum',
'code':
'last'
})
data.columns = data.columns.droplevel(0)
# for edit_time_period in MARKET_PRESET.get_resample_edit_periods(code):
# _data_min = _resample_edit_periods(_data_min,edit_time_period)
# resx = resx.append(_data_min)
'''整理成符合分钟数据协议的数据格式'''
data.rename(columns={
'volume': 'trade',
'zengcang': 'position'
},
inplace=True)
for i in ['open', 'high', 'low', 'close']:
data[i] /= 1000
data['datetime'] = list(map(lambda x: str(x)[:19], data.index))
data['tradetime'] = pd.to_datetime(
data['datetime'].apply(QA_util_future_to_tradedatetime))
data = data \
.assign(date=data['datetime'].apply(lambda x: str(x)[0:10])) \
.assign(date_stamp=data['datetime'].apply(lambda x: QA_util_date_stamp(x))) \
.assign(time_stamp=data['datetime'].apply(lambda x: QA_util_time_stamp(x))) \
.assign(type=type_)
data['source'] = source
if if_drop: data = data.dropna()
return select_DataAggrement(DATA_AGGREMENT_NAME.FUTURE_MIN)(DATASOURCE.TDX,
data)
def QA_fetch_get_future_min(code,
start,
end,
frequence='1min',
fill_data_with_tick_database=False,
fill_data_with_tick_online=False,
method='api',
account=None,
password=None,
remember=False):
assert QA_util_dateordatetime_valid(start), 'start input format error'
assert QA_util_dateordatetime_valid(end), 'end input format error'
jqcode = _QA_code_toJQDATA(code, 'futures')
jqfrequence = _QA_freq_toJQDATA(frequence)
if method == 'api':
JQDATA_login(account=account, password=password, remember=remember)
data = jqdatasdk.get_price(security=jqcode,
start_date=start,
end_date=end,
frequency=jqfrequence,
fields=None,
skip_paused=True,
fq=None,
count=None)
# jqdatasdk.logout()
elif method == 'http':
data = jqdatahttp.get_price(security=jqcode,
start_date=start,
end_date=end,
frequency=jqfrequence,
fields=None,
skip_paused=True,
fq=None,
count=None)
data['code'] = code
data['type'] = frequence
return select_DataAggrement(DATABASE_NAME.FUTURE_MIN)(DATASOURCE.JQDATA,
data)
def QA_fetch_get_future_min(code, start, end, frequence=FREQUENCE.ONE_MIN):
'''
TODO 当前仅支持L8和L9
期货数据 分钟线
code = 'IL8'
start = '2014-12-27'
end = '2014-12-31'
'''
market_type = MARKET_TYPE.FUTURE_CN
try:
if code[-2:] in ['L8']:
if frequence == FREQUENCE.ONE_MIN:
'''找到总路径'''
product_type = '股指期货' if code[:-2] in ['IF', 'IC', 'IH'
] else '商品期货'
data_type = '主连' if code[-2:] == 'L8' else '指数'
path = cofund_data_path[(market_type, frequence, product_type,
data_type)]
main_code = code[:-2]
'''确定品种路径'''
if main_code.isupper():
file_code = main_code if main_code in QA_util_listdir(
path) else main_code.lower()
elif main_code.islower():
file_code = main_code if main_code in QA_util_listdir(
path) else main_code.upper()
else:
raise NotImplementedError
path = os.path.abspath(path + file_code)
date_calendar = QA_util_get_trade_range(
max(
start[:10],
QA_util_date_int2str(min(QA_util_listfile(path,
'csv')))),
min((end[:10]),
QA_util_date_int2str(max(QA_util_listfile(path,
'csv')))))
# date_calendar = QA_util_get_trade_range(start[:10],end[:10])
data = pd.DataFrame()
for date in date_calendar:
try:
intdate = QA_util_date_str2int(date)
file_temp = pd.read_csv(
os.path.join(path,
str(intdate) + '.csv'))
data = data.append(file_temp)
except:
pass
# data = data[data['Time']>=85900]
data['type'] = frequence
data['code'] = code
data = data.drop_duplicates(subset=['RealDate', 'Time'])
return select_DataAggrement(DATABASE_NAME.FUTURE_MIN)(
DATASOURCE.COFUND, data)[start:end]
else:
print('当前仅支持1分钟的数据调用')
raise NotImplementedError
else:
print('当前仅支持L8的数据调用')
raise NotImplementedError
except Exception as exp:
print("code is ", code)
print(exp.__str__)
return None
def QA_data_stocktick_resample_1min(tick,
type_='1min',
source='tick_resample',
if_drop=True):
"""
tick 采样为 分钟数据(标准QA数据协议格式)
1. 仅使用将 tick 采样为 1 分钟数据
2. 仅测试过，与通达信 1 分钟数据达成一致
3. 经测试，可以匹配 QA.QA_fetch_get_stock_transaction 得到的数据，其他类型数据未测试
demo:
df = QA.QA_fetch_get_stock_transaction(package='tdx', code='000001',
start='2018-08-01 09:25:00',
end='2018-08-03 15:00:00')
df_min = QA_data_stocktick_resample_1min(df)
"""
tick['volume'] = tick['volume'] * 100.0
tick = tick.assign(amount=tick.price * tick.volume)
resx = pd.DataFrame()
tick.index = pd.to_datetime(tick.datetime)
_dates = set(tick.date)
for date in sorted(list(_dates)):
_data = tick.loc[tick.date == date]
# morning min bar
_data1 = _data[time(9, 25):time(11, 30)].resample(
type_, closed='left', base=30, loffset=type_).apply({
'price': 'ohlc',
'volume': 'sum',
'code': 'last',
'amount': 'sum'
})
_data1.columns = _data1.columns.droplevel(0)
# do fix on the first and last bar
# 某些股票某些日期没有集合竞价信息，譬如 002468 在 2017 年 6 月 5 日的数据
if len(_data.loc[time(9, 25):time(9, 25)]) > 0:
_data1.loc[time(9, 31):time(9, 31),
'open'] = _data1.loc[time(9, 26):time(9, 26),
'open'].values
_data1.loc[time(9, 31):time(9, 31),
'high'] = _data1.loc[time(9, 26):time(9, 31),
'high'].max()
_data1.loc[time(9, 31):time(9, 31),
'low'] = _data1.loc[time(9, 26):time(9, 31),
'low'].min()
_data1.loc[time(9, 31):time(9, 31),
'volume'] = _data1.loc[time(9, 26):time(9, 31),
'volume'].sum()
_data1.loc[time(9, 31):time(9, 31),
'amount'] = _data1.loc[time(9, 26):time(9, 31),
'amount'].sum()
# 通达信分笔数据有的有 11:30 数据，有的没有
if len(_data.loc[time(11, 30):time(11, 30)]) > 0:
_data1.loc[time(11, 30):time(11, 30),
'high'] = _data1.loc[time(11, 30):time(11, 31),
'high'].max()
_data1.loc[time(11, 30):time(11, 30),
'low'] = _data1.loc[time(11, 30):time(11, 31),
'low'].min()
_data1.loc[time(11, 30):time(11, 30),
'close'] = _data1.loc[time(11, 31):time(11, 31),
'close'].values
_data1.loc[time(11, 30):time(11, 30),
'volume'] = _data1.loc[time(11, 30):time(11, 31),
'volume'].sum()
_data1.loc[time(11, 30):time(11, 30),
'amount'] = _data1.loc[time(11, 30):time(11, 31),
'amount'].sum()
_data1 = _data1.loc[time(9, 31):time(11, 30)]
# afternoon min bar
_data2 = _data[time(13, 0):time(15, 0)].resample(type_,
closed='left',
base=30,
loffset=type_).apply({
'price':
'ohlc',
'volume':
'sum',
'code':
'last',
'amount':
'sum'
})
_data2.columns = _data2.columns.droplevel(0)
# 沪市股票在 2018-08-20 起，尾盘 3 分钟集合竞价
if (pd.Timestamp(date) <
pd.Timestamp('2018-08-20')) and (tick.code.iloc[0][0] == '6'):
# 避免出现 tick 数据没有 1:00 的值
if len(_data.loc[time(13, 0):time(13, 0)]) > 0:
_data2.loc[time(15, 0):time(15, 0),
'high'] = _data2.loc[time(15, 0):time(15, 1),
'high'].max()
_data2.loc[time(15, 0):time(15, 0),
'low'] = _data2.loc[time(15, 0):time(15, 1),
'low'].min()
_data2.loc[time(15, 0):time(15, 0),
'close'] = _data2.loc[time(15, 1):time(15, 1),
'close'].values
else:
# 避免出现 tick 数据没有 15:00 的值
if len(_data.loc[time(13, 0):time(13, 0)]) > 0:
_data2.loc[time(15, 0):time(15, 0)] = _data2.loc[
time(15, 1):time(15, 1)].values
_data2 = _data2.loc[time(13, 1):time(15, 0)]
resx = resx.append(_data1).append(_data2)
if if_drop:
resx = resx.dropna()
'''整理数据格式到STOCK_MIN格式'''
resx['datetime'] = resx.index
resx = resx \
.assign(date=resx['datetime'].apply(lambda x: str(x)[0:10]),
date_stamp=resx['datetime'].apply(
lambda x: QA_util_date_stamp(x)),
time_stamp=resx['datetime'].apply(
lambda x: QA_util_time_stamp(x)),
type=type_)
resx['source'] = source
return select_DataAggrement(DATA_AGGREMENT_NAME.STOCK_MIN)(None, resx)
def QA_data_min_resample_stock(min_data,
type_='30min',
source='1min_resample'):
"""分钟线采样成大周期
分钟线采样成子级别的分钟线
time+ OHLC==> resample
Arguments:
min {[type]} -- [description]
raw_type {[type]} -- [description]
new_type {[type]} -- [description]
"""
min_data['datetime'] = pd.to_datetime(min_data['datetime'])
try:
min_data = min_data.reset_index().set_index('datetime', drop=False)
except:
min_data = min_data.set_index('datetime', drop=False)
CONVERSION = {
'code': 'first',
'open': 'first',
'high': 'max',
'low': 'min',
'close': 'last',
'vol': 'sum',
'amount': 'sum'
} if 'vol' in min_data.columns else {
'code': 'first',
'open': 'first',
'high': 'max',
'low': 'min',
'close': 'last',
'volume': 'sum',
'amount': 'sum'
}
resx = pd.DataFrame()
for item in set(min_data.index.date):
min_data_p = min_data.loc[str(item)]
n = min_data_p['{} 21:00:00'.format(item):].resample(
type_, base=30, closed='right', loffset=type_).apply(CONVERSION)
d = min_data_p[:'{} 11:30:00'.format(item)].resample(
type_, base=30, closed='right', loffset=type_).apply(CONVERSION)
f = min_data_p['{} 13:00:00'.format(item):].resample(
type_, closed='right', loffset=type_).apply(CONVERSION)
resx = resx.append(d).append(f)
'''整理数据格式到STOCK_MIN格式'''
resx['datetime'] = resx.index
resx = resx \
.assign(date=resx['datetime'].apply(lambda x: str(x)[0:10]),
date_stamp=resx['datetime'].apply(
lambda x: QA_util_date_stamp(x)),
time_stamp=resx['datetime'].apply(
lambda x: QA_util_time_stamp(x)),
type=type_ if 'min' in str(type_) else str(type_)+'min')
resx['source'] = source
return select_DataAggrement(DATA_AGGREMENT_NAME.STOCK_MIN)(None, resx)