def get_(folder):
# step 2 因子数据预处理
# 加载dataview数据集
dv = DataView()
dataview_folder = folder
dv.load_dataview(dataview_folder)
# 定义信号过滤条件-非指数成分
# df_index_member = dv.get_ts('index_member')
signal = dv.get_ts("pb")
price = dv.get_ts("close_adj")
group = dv.get_ts("sw1")
# mask
mask = dv.get_ts('index_member') == 0 # 定义信号过滤条件-非指数成分
# 定义可买入卖出条件——未停牌、未涨跌停
trade_status = dv.get_ts('trade_status')
can_trade = trade_status == 1 # 可以交易
# 涨停
up_limit = dv.add_formula(
'up_limit',
'(open - Delay(close, 1)) / Delay(close, 1) > 0.095',
is_quarterly=False)
# 跌停
down_limit = dv.add_formula(
'down_limit',
'(open - Delay(close, 1)) / Delay(close, 1) < -0.095',
is_quarterly=False)
can_enter = np.logical_and(up_limit < 1, can_trade) # 未涨停未停牌
can_exit = np.logical_and(down_limit < 1, can_trade) # 未跌停未停牌
benchmark = dv.data_benchmark
return signal, price, mask, group, can_enter, can_exit, benchmark
def test_add_field():
dv = DataView()
dv.load_dataview(folder_path=daily_path)
nrows, ncols = dv.data_d.shape
n_securities = len(dv.data_d.columns.levels[0])
ds = RemoteDataService()
ds.init_from_config(data_config)
dv.add_field('total_share', ds)
assert dv.data_d.shape == (nrows, ncols + 1 * n_securities)
def test_q_add_field():
dv = DataView()
dv.load_dataview(folder_path=quarterly_path)
nrows, ncols = dv.data_q.shape
n_securities = len(dv.data_d.columns.levels[0])
ds = RemoteDataService()
ds.init_from_config(data_config)
dv.add_field("debttoassets", ds)
assert dv.data_q.shape == (nrows, ncols + 1 * n_securities)
def test_add_formula():
dv = DataView()
dv.load_dataview(folder_path=daily_path)
nrows, ncols = dv.data_d.shape
n_securities = len(dv.data_d.columns.levels[0])
formula = 'Delta(high - close, 1)'
dv.add_formula('myvar1', formula, is_quarterly=False, add_data=True)
assert dv.data_d.shape == (nrows, ncols + 1 * n_securities)
formula2 = 'myvar1 - close'
dv.add_formula('myvar2', formula2, is_quarterly=False, add_data=True)
assert dv.data_d.shape == (nrows, ncols + 2 * n_securities)
def test_q_add_formula():
dv = DataView()
folder_path = '../output/prepared/20160609_20170601_freq=1D'
dv.load_dataview(folder_path=quarterly_path)
nrows, ncols = dv.data_d.shape
n_securities = len(dv.data_d.columns.levels[0])
formula = 'total_oper_rev / close'
dv.add_formula('myvar1', formula, is_quarterly=False, add_data=True)
df1 = dv.get_ts('myvar1')
assert not df1.empty
formula2 = 'Delta(oper_exp * myvar1 - open, 3)'
dv.add_formula('myvar2', formula2, is_quarterly=False, add_data=True)
df2 = dv.get_ts('myvar2')
assert not df2.empty
def test_load():
dv = DataView()
dv.load_dataview(folder_path=daily_path)
assert dv.start_date == 20160601 and set(dv.symbol) == set(
'000001.SZ,600030.SH,000063.SZ'.split(','))
# test get_snapshot
snap1 = dv.get_snapshot(20170504,
symbol='600030.SH,000063.SZ',
fields='close,pb')
assert snap1.shape == (2, 2)
assert set(snap1.columns.values) == {'close', 'pb'}
assert set(snap1.index.values) == {'600030.SH', '000063.SZ'}
# test get_ts
ts1 = dv.get_ts('close',
symbol='600030.SH,000063.SZ',
start_date=20170101,
end_date=20170302)
assert ts1.shape == (38, 2)
assert set(ts1.columns.values) == {'600030.SH', '000063.SZ'}
assert ts1.index.values[-1] == 20170302
print(group2_code.tail()
) #value="industry1_code"表示以industry2_name为分类标准,不过返回的code类型(类别代号)
group3 = dp.daily_sec_industry(api,
symbols,
start,
end,
source='zz',
value="industry1_name")
print(group3.tail()) #source='zz'表示以中证为分类标准,只是分类标准不同而已
group3_code = dp.daily_sec_industry(api,
symbols,
start,
end,
source='zz',
value="industry1_code")
print(group3_code.tail()
) #source='zz'表示以中证为分类标准,只是分类标准不同而已,industry1_code返回的是类别代号(code类型)
#3_添加数据保存
dv = DataView()
dataview_folder = 'G:/data/hs300' #档案地址
dv.load_dataview(dataview_folder) #加载档案地址
dv.append_df(group, 'group') #将group列加入dv中,后面的'group'为列名
dv.save_dataview('G:/data/hs300') #保存
#获取数据
print(dv.get_ts('group').tail())
A = dv.get_ts('group') #这样看的更清楚点
def save_zz800(props):
dataview_folder = 'JAQS_Data/Mutifactor_1'
if not (os.path.isdir(dataview_folder)):
os.makedirs(dataview_folder)
DV.init_from_config(props, ds)
DV.prepare_data()
DV.save_dataview(dataview_folder)
return
save_zz800(zz800_props)
dataview_folder = 'JAQS_Data/Mutifactor_1'
dv = DataView()
dv.init_from_config(zz800_props, ds)
dv.load_dataview(dataview_folder)
import talib as ta
import numpy as np
import pandas as pd
from datetime import datetime
def change_time(df):
df.index = pd.Index(
map(lambda x: datetime.strptime(str(x), "%Y%m%d"), df.index))
return df
EMA = dv.get_ts('close').ewm(span=5, adjust=False).mean()
EMA2 = EMA.ewm(span=5, adjust=False).mean()
def test_q_get():
dv = DataView()
dv.load_dataview(folder_path=quarterly_path)
res = dv.get("", 0, 0, 'total_oper_rev')
assert set(res.index.values) == set(dv.dates[dv.dates >= dv.start_date])
#1_读取本地数据
import jaqs_fxdayu
jaqs_fxdayu.patch_all()
from jaqs_fxdayu.data import DataView
from jaqs_fxdayu.data import RemoteDataService
import os
import numpy as np
import warnings
warnings.filterwarnings("ignore")
dataview_folder = 'G:/data/hs300_2' #档案地址
dv = DataView()
dv.load_dataview(dataview_folder) #加载档案地址,结果出现Dataview loaded successfully则成功
print(dv.fields) #查看dv中取得的数据
#2_过滤停牌涨跌停（可买可卖） mask_index_member为要过滤的为True,can_enter与can_exit皆为可交易为True
from jaqs_fxdayu.util import dp
from jaqs.data.dataapi import DataApi
A = dv.get_ts('index_member') #得到一张表,1表示在指数成分里,0表示不在指数成分里
B = dv.get_ts('trade_status') #得到一张表,从中可以得出股票能否交易(或停牌)
def mask_index():
df_index_member = dv.get_ts('index_member')
mask_index_member = df_index_member == 0 #定义信号过滤条件-非指数成分,若df_index_member==0则mask_index_member=true
return mask_index_member
'end_date': 20180101,
'universe': '000300.SH',
'fields': "volume,pb,pe,ps,float_mv,sw1",
'freq': 1
}
dv.init_from_config(props, ds)
dv.prepare_data()
dv.save_dataview(dataview_folder) # 保存数据文件到指定路径,方便下次直接加载
save_dataview()
# 加载数据
dv = DataView()
dv.load_dataview(dataview_folder) #加载数据
print(dv.fields) #查看dv中取得的数据
print(dv.get_ts("pb").head()) #查看dv中取得的市净率
import numpy as np
#定义信号过滤条件-非指数成分
def mask_index_member():
df_index_member = dv.get_ts(
'index_member') #A=dv.get_ts('index_member'),1表示在沪深300中,0表示不在沪深300中
mask_index_member = df_index_member == 0 #过滤
return mask_index_member
# 定义可买卖条件——未停牌、未涨跌停