def get_stock(stockPool, date):
if stockPool == 'HS300':#用于获取沪深300股票池
stockList = jd.get_index_stocks('000300.XSHG', date)
elif stockPool == 'ZZ500':#用于获取中证500股票池
stockList = jd.get_index_stocks('399905.XSHE', date)
elif stockPool == 'ZZ800':#用于获取中证800股票池
stockList = jd.get_index_stocks('399906.XSHE', date)
elif stockPool == 'A':#用于获取全部A股股票池
stockList = jd.get_index_stocks('000002.XSHG', date) + jd.get_index_stocks('399107.XSHE', date)
else:#自定义输入股票池
stockList = stockPool
return stockList
def fetch_csi_index_component(self):
"""
抓取上证、中证指数成分股
"""
for _, index in self.all_index.iterrows():
response_df = pd.DataFrame()
index_code = index.name.split(".")[0]
stocks = get_index_stocks(index.name)
response_df['stock_code'] = stocks
response_df['stock_code'] = response_df['stock_code'].apply(
lambda x: x.split(".")[0])
index_id = f'index_cn_{index_code}'
response_df['entity_id'] = index_id
response_df['entity_type'] = 'index'
response_df['exchange'] = 'cn'
response_df['code'] = index_code
response_df['name'] = index.display_name
response_df['timestamp'] = now_pd_timestamp()
response_df['stock_id'] = response_df['stock_code'].apply(
lambda x: china_stock_code_to_id(str(x)))
response_df['id'] = response_df['stock_id'].apply(
lambda x: f'{index_id}_{x}')
df_to_db(data_schema=self.data_schema,
df=response_df,
provider=self.provider,
force_update=True)
self.logger.info(f'{index["name"]} - {index_code} 成分股抓取完成...')
self.sleep()
def set_components_data_from_joinquant(self, basic_dict, cdate):
table_name = self.get_components_table_name(cdate)
if not self.is_table_exists(table_name):
if not self.create_components_table(table_name):
logger.error("create components table failed")
return False
if self.is_date_exists(table_name, cdate):
logger.debug("existed table:%s, date:%s" % (table_name, cdate))
return True
code = "{}.XSHG".format(self.code) if self.code.startswith("6") else "{}.XSHE".format(self.code)
stocks = get_index_stocks(code, date = cdate)
df = pd.DataFrame(stocks, columns =['code'])
df['code'] = df['code'].str.split('.', expand=True)[0]
if not df[~df.code.isin(basic_dict)].empty:
logger.error("code from juquant is not in basic in data {}".format(df[~df.code.isin(basic_dict)]))
return False
df['name'] = df['code'].apply(lambda x: basic_dict[x])
df['date'] = cdate
if 'wieight' not in df.columns:
df['weight'] = 1/len(df)
if 'flag' not in df.columns:
df['flag'] = 1
df = df.reset_index(drop = True)
if is_df_has_unexpected_data(df):
logger.error("data for %s is not clear" % self.code)
return False
if self.mysql_client.set(df, table_name):
if self.redis.sadd(table_name, cdate): return True
return False
def predictBacktest(train_cfg):
UserDataApi_ = UserDataApi()
with open(train_cfg, "r") as f:
train_cfg = json.load(f)
feature_cfg = train_cfg["feature_cfg"]
model_name = train_cfg["model"]
model_cfg = train_cfg["model_cfg"]
#feature_cfg = "./config/feature_create_cfg.json"
with open(feature_cfg, "r") as f:
feature_cfg = json.load(f)
test_start = feature_cfg['test_start']
test_end = feature_cfg['test_end']
print(feature_cfg)
stock_list = jq.get_index_stocks('000300.XSHG')
stock_array = np.array(stock_list)
feature_creator = Feature(feature_cfg, UserDataApi_)
label_creator = Label(feature_cfg, UserDataApi_)
model = restore_model_from_file(model_name, model_cfg)
date_list = UserDataApi_.getTradeDays(test_start, test_end)
bt = initBackTest(date_list[0], date_list[-1], UserDataApi_)
feature_enginer = build_feature_engine("lgb", feature_cfg["engine_cfg"])
for date in date_list:
info = create_feature_daily(date, stock_list, feature_creator,
label_creator, feature_enginer,
UserDataApi_)
feature_per_day = info["feature"]
label_per_day = info["label"]
prediction = model.predict(feature_per_day)
v = UserDataApi_.validIndex(date, stock_list)
stock_list_temp = np.array(stock_list)[v]
prediction = prediction[v]
#prediction = label_per_day[0,:,0][v]
prediction = np.nan_to_num(prediction)
#v = ~(prediction == prediction_)
#prediction[v] = 0.0
#index = prediction > aver
#stock_list_temp = stock_list_temp[index]
#prediction = prediction[index]
bt.run(date, stock_list_temp, prediction, 0.9)
print("date:", date, "capital:", bt.bookkeeper.capital)
# if date == "2020-03-11":
# print (prediction)
# exit()
if not buffered_feature:
np.save("./DATA/buffer/feature_for_prediction.npy", feature_dic)
bt.analysist()
bt.vis()
def get_data_by_date(self, index_code, date):
stock_codes = jqdatasdk.get_index_stocks(index_code)
trade_day = jqdatasdk.get_trade_days(end_date=date, count=1)[0]
q = jqdatasdk.query(jqdatasdk.valuation.code, jqdatasdk.valuation.day,
jqdatasdk.valuation.pe_ratio_lyr,
jqdatasdk.valuation.pe_ratio,
jqdatasdk.valuation.pb_ratio,
jqdatasdk.valuation.market_cap,
jqdatasdk.valuation.circulating_market_cap).filter(
jqdatasdk.valuation.code.in_(stock_codes))
df = jqdatasdk.get_fundamentals(q, date=trade_day)
return df
def list_index_1_year(code, the_year):
#
yyyymmdd = "%d-05-01" % the_year
# 取当年该指数成份
members = jq.get_index_stocks(code, date=yyyymmdd)
filename = "%s/%s.%s.csv" % (data_struct.WORKING_DIR, code, the_year)
with open(filename, "w") as the_file:
writer = csv.writer(the_file, lineterminator='\n')
for val in members:
writer.writerow([val])
def get_index_stocks(code, trade_date, data_source):
# 通过深证综指399106.XSHE来观察全市场数据
stocks = []
# 主要数据从JQData进行获取
if data_source == 'jqdata':
if code == '399106.XSHE':
stocks = list(jqdatasdk.get_all_securities(date=trade_date).index)
else:
stocks = jqdatasdk.get_index_stocks(code, trade_date)
# 通过tushare提供其他不包含数据
elif data_source == 'tushare':
stocks = self_data_source.get_index_stocks(code, trade_date)
return stocks
def get_stock(stockPool,begin_date):
if stockPool=='HS300':
stockList=jq.get_index_stocks('000300.XSHG',begin_date)
elif stockPool=='ZZ500':
stockList=jq.get_index_stocks('399905.XSHE',begin_date)
elif stockPool=='ZZ800':
stockList=jq.get_index_stocks('399906.XSHE',begin_date)
elif stockPool=='CYBZ':
stockList=jq.get_index_stocks('399006.XSHE',begin_date)
elif stockPool=='ZXBZ':
stockList=jq.get_index_stocks('399005.XSHE',begin_date)
elif stockPool=='A':
stockList=jq.get_index_stocks('000002.XSHG',begin_date)+jq.get_index_stocks('399107.XSHE',begin_date)
#剔除ST股
st_data=jq.get_extras('is_st',stockList, count = 1,end_date=begin_date)
stockList = [stock for stock in stockList if not st_data[stock][0]]
#剔除停牌、新股及退市股票
stockList=delect_stop(stockList,begin_date)
#print(stockList)
return stockList
def fetch_fundamentals_1_year_300(engine, the_year):
#
# 沪深300 样本股调整实施时间分别是每年 6 月和 12 月的第二个星期五的下一交易日
# 我们假设每年五月(年报出来)换仓
print "fetch fetch_fundamentals for %d" % the_year
yyyymmdd = "%d-05-01" % the_year
# 取当年沪深300成份
hs300 = jq.get_index_stocks('000300.XSHG', date=yyyymmdd)
#print hs300
#J66 货币金融服务 1991-04-03
#J67 资本市场服务 1994-01-10
#J68 保险业 2007-01-09
#J69 其他金融业
# 排除不适用魔法公式的行业
banks = jq.get_industry_stocks('J66', date=yyyymmdd)
brokers = jq.get_industry_stocks('J67', date=yyyymmdd)
insurances = jq.get_industry_stocks('J68', date=yyyymmdd)
others = jq.get_industry_stocks('J69', date=yyyymmdd)
exclude = banks + brokers + insurances + others
#print banks
#print insurances
for code in hs300:
if (code in exclude):
print " ... skip %s ..." % code
continue
print " fetching %s" % code
fetch_target_stock_fundamentals(engine, code, the_year)
#break
print "finished fetching fetch_fundamentals for %d" % the_year
print
import jqdatasdk
from datetime import datetime
import pandas as pd
import numpy as np
jqdatasdk.auth("******", "******")
if __name__ == '__main__':
'''get threshold on certain date and stock index'''
benchmark_date = '2015-12-30'
stock_set = '000018.XSHG'
q = jqdatasdk.query(jqdatasdk.valuation.code, jqdatasdk.valuation.pe_ratio, jqdatasdk.valuation.market_cap).\
filter(jqdatasdk.valuation.pe_ratio>0, jqdatasdk.valuation.code.in_(jqdatasdk.get_index_stocks(stock_set))).\
order_by(jqdatasdk.valuation.pe_ratio.asc())
df = jqdatasdk.get_fundamentals(q, benchmark_date)
pe_mean = float(df['pe_ratio'].mean())
mc_mean = float(df['market_cap'].mean())
print df.head()
print
print 'pe_mean', pe_mean, '; mc_mean', mc_mean
print
'''get stock index we want'''
q = jqdatasdk.query(jqdatasdk.valuation.code, jqdatasdk.valuation.pe_ratio).\
filter(jqdatasdk.valuation.pe_ratio<pe_mean, jqdatasdk.valuation.market_cap<mc_mean,\
jqdatasdk.valuation.pe_ratio>0, jqdatasdk.valuation.code.in_(jqdatasdk.get_index_stocks(stock_set))).\
order_by(jqdatasdk.valuation.pe_ratio.asc())
df = jqdatasdk.get_fundamentals(q, benchmark_date)
newlist = df['code'].tolist()
print newlist
print
'''get benchmark_date data'''
def getHs300(): # 沪深300
return jq.get_index_stocks('000300.XSHG')
def record(self, entity, start, end, size, timestamps):
if start < to_pd_timestamp("2008-01-01"):
start = to_pd_timestamp("2008-01-01")
# 获取数据库中已有数据
data_schema_df = self.data_schema.query_data(entity_id=entity.id)
if not data_schema_df.empty and data_schema_df.timestamp.max(
) <= start:
from datetime import timedelta
# bdate_range_date = pd.bdate_range(start, start + timedelta(weeks=1))
bdate_range_date = pd.bdate_range(start,
start + timedelta(weeks=4 * 6))
if bdate_range_date[
bdate_range_date > start][-1] > now_pd_timestamp():
start = to_time_str(now_pd_timestamp())
else:
start = to_time_str(
bdate_range_date[bdate_range_date > start][-1])
data = get_index_stocks(to_jq_entity_id(entity), date=start)
df = pd.DataFrame(data, columns=['code'])
if pd_is_not_null(df):
df['stock_code'] = df['code'].apply(lambda x: str(x).split('.')[0])
df['stock_exchange'] = df['code'].apply(
lambda x: str(x).split('.')[1])
df['stock_exchange'] = df['stock_exchange'].replace(
'XSHG', 'sh').replace('XSHE', 'sz')
df['stock_id'] = df['stock_code'].apply(
lambda x: china_stock_code_to_id(x))
df['stock_name'] = df.stock_id.apply(
lambda x: StockDetail.query_data(entity_id=x).name[0])
df_old = df[df.stock_id.isin(data_schema_df.stock_id)].copy()
df_new = df[~df.stock_id.isin(data_schema_df.stock_id)].copy()
out_data = data_schema_df[~data_schema_df.stock_id.isin(df.stock_id
)].copy()
if df_new.empty and out_data.empty:
# 没有调入的股票 没有调出的股票
data_schema_df['timestamp'] = pd.to_datetime(start)
data_schema_df['pub_date'] = data_schema_df['timestamp']
df_to_db(df=data_schema_df,
data_schema=self.data_schema,
provider=self.provider,
force_update=self.force_update)
return None
elif df_new.empty and not out_data.empty:
# 没有调入,有调出
data_schema_df['timestamp'] = pd.to_datetime(start)
data_schema_df['pub_date'] = data_schema_df['timestamp']
df2 = pd.DataFrame()
for index, data_old in data_schema_df.iterrows():
schema_details = out_data.query(
"id == @data_old.id").copy()
if not schema_details.empty:
data_schema_df['timestamp'] = pd.to_datetime(start)
# 更新调出时间
schema_details['out_date'] = pd.to_datetime(start)
data_schema_df['pub_date'] = pd.to_datetime(start)
df2 = df2.append(schema_details)
else:
df2 = df2.append(data_old)
df_to_db(df=df2,
data_schema=self.data_schema,
provider=self.provider,
force_update=self.force_update)
return None
elif not df_new.empty and not out_data.empty:
# 有调入,有调出
data_schema_df['timestamp'] = pd.to_datetime(start)
data_schema_df['pub_date'] = data_schema_df['timestamp']
df2 = pd.DataFrame()
# 处理调出
for index, data_old in data_schema_df.iterrows():
schema_details = out_data.query(
"id == @data_old.id").copy()
if not schema_details.empty:
# 更新调出时间
schema_details['out_date'] = pd.to_datetime(start)
schema_details['pub_date'] = pd.to_datetime(start)
df2 = df2.append(schema_details)
else:
# 无调出的不变
df2 = df2.append(data_old)
# 处理调入,新增
df_new['timestamp'] = pd.to_datetime(start)
df_new['pub_date'] = df_new['timestamp']
df_new['into_date'] = start
df_new['out_date'] = pd.to_datetime("2200-01-01")
df_new['pub_date2'] = df_new['timestamp'].apply(
lambda x: to_time_str(x))
df_new['stock_code'] = df_new['code'].apply(
lambda x: str(x).split('.')[0])
df_new['entity_id'] = entity.id
df_new['id'] = df_new[['entity_id', 'stock_id']].apply(
lambda x: '_'.join(x.astype(str)), axis=1)
df_new['code'] = entity.code
df_new['entity_type'] = entity.entity_type
df_new['exchange'] = entity.exchange
df_new['name'] = entity.name
df2 = df2.append(df_new)
df_to_db(df=df2,
data_schema=self.data_schema,
provider=self.provider,
force_update=self.force_update)
return None
elif not df_new.empty and out_data.empty:
# 处理调入,新增
df_new['timestamp'] = pd.to_datetime(start)
df_new['pub_date'] = df_new['timestamp']
df_new['into_date'] = start
df_new['out_date'] = pd.to_datetime("2200-01-01")
df_new['pub_date2'] = df_new['timestamp'].apply(
lambda x: to_time_str(x))
df_new['stock_code'] = df_new['code'].apply(
lambda x: str(x).split('.')[0])
df_new['entity_id'] = entity.id
df_new['id'] = df_new[['entity_id', 'stock_id']].apply(
lambda x: '_'.join(x.astype(str)), axis=1)
df_new['code'] = entity.code
df_new['entity_type'] = entity.entity_type
df_new['exchange'] = entity.exchange
df_new['name'] = entity.name
df_to_db(df=df_new,
data_schema=self.data_schema,
provider=self.provider,
force_update=self.force_update)
return None
else:
print('1')
# self.logger.info(df.tail())
self.logger.info(f"persist etf {entity.code} portfolio success")
return None
def save_data(start_date, end_date):
jqdatasdk.auth(username=user_passwd.user_name,
password=user_passwd.passwd)
names_to_id = red.get('names_to_id')
names_to_id = json.loads(names_to_id)
indices = pd.read_csv(
r'C:\Users\never\Desktop\website-practice\indices.csv')
names = []
display_names = []
index_name_to_id = {}
for i in range(len(indices)):
if str(indices.start_date[i]) <= start_date and end_date <= str(
indices.end_date[i]):
names.append(indices.id[i])
display_names.append(indices.display_name[i])
index_name_to_id[indices.display_name[i]] = indices.id[i]
indices = display_names
names_to_id['index'] = index_name_to_id
industries_df = jqdatasdk.get_industries('sw_l1')
industry_end_date = {
'建筑建材I': '2014-02-21',
'机械设备I': '2014-02-21',
'交运设备I': '2014-02-21',
'信息设备I': '2014-02-21',
'金融服务I': '2014-02-21',
'信息服务I': '2014-02-21',
}
industries = []
industry_name_to_id = {}
for industry_name, industry_id, start_time in zip(
industries_df.name.values, industries_df.index.values,
industries_df.start_date.values):
st = str(start_time)
if industry_name in industry_end_date.keys(
) and st <= start_date and end_date < industry_end_date[
industry_name]:
industries.append(industry_name)
industry_name_to_id.update({industry_name: industry_id})
elif industry_name not in industry_end_date.keys(
) and st <= start_date:
industries.append(industry_name)
industry_name_to_id.update({industry_name: industry_id})
names_to_id['industry_index'] = industry_name_to_id
stock_pool = {}
stock_ids, stock_names = [], []
stock_name_to_id = {}
for i in range(len(display_names)):
display, pool_id = display_names[i], names[i]
p = jqdatasdk.get_all_securities(types=['stock'], date=start_date)
p = p[p.end_date >= end_date]
q = jqdatasdk.get_index_stocks(pool_id)
id_to_name = dict(zip(p.index.values, p.display_name.values))
stock_name_to_id.update(
dict(zip(p.display_name.values, p.index.values)))
stocks = []
for item in q:
try:
stocks.append(id_to_name[item])
stock_ids.append(item)
stock_names.append(id_to_name[item])
except Exception as e:
pass
stocks = sorted(stocks, key=lambda x: ' '.join(lazy_pinyin(x)))
stock_pool[display] = stocks
names_to_id['stock'] = stock_name_to_id
get_funds = jqdatasdk.get_all_securities('fund', date=start_date)
display_names = ['场内ETF', '场内LOF', '分级基金A', '分级基金B', '场内货币基金']
names = ['etf', 'lof', 'fja', 'fjb', 'money_market_fund']
exchanged_fund_pool = {}
exchanged_name_to_id = dict(zip(display_names, names))
for i in range(len(display_names)):
display, pool_id = display_names[i], names[i]
p = get_funds[get_funds['type'] == pool_id]
y1 = (p.start_date <= start_date).values
y2 = (p.end_date >= end_date).values
p = p[np.logical_and(y1, y2)]
q = p.index.values
id_to_name = dict(zip(p.index.values, p.display_name.values))
exchanged_name_to_id.update(
dict(zip(p.display_name.values, p.index.values)))
funds = []
for item in q:
try:
funds.append(id_to_name[item])
except Exception as e:
pass
funds = sorted(funds, key=lambda x: ' '.join(lazy_pinyin(x)))
exchanged_fund_pool[display] = funds
names_to_id['exchanged_fund'] = exchanged_name_to_id
get_funds = jqdatasdk.get_all_securities('open_fund', date=start_date)
get_funds = get_funds[get_funds.end_date >= end_date]
display_names = ['股票型', '混合型', '债券型', '货币型', 'ETF联接', '黄金', 'QDII']
names = [
'stock_fund', 'mixture_fund', 'bond_fund', 'money_market_fund',
'fund_fund', 'noble_metal_fund', 'QDII_fund'
]
open_fund_pool = {}
open_fund_name_to_id = dict(zip(display_names, names))
for i in range(len(display_names)):
display, pool_id = display_names[i], names[i]
if pool_id != 'QDII_fund':
p = get_funds[get_funds['type'] == pool_id]
else:
p = jqdatasdk.get_all_securities(pool_id)
q = p.index.values
id_to_name = dict(zip(p.index.values, p.display_name.values))
open_fund_name_to_id.update(
dict(zip(p.display_name.values, p.index.values)))
funds = []
for item in q:
try:
funds.append(id_to_name[item])
except Exception as e:
pass
funds = sorted(funds, key=lambda x: ' '.join(lazy_pinyin(x)))
open_fund_pool[display] = funds
names_to_id['open_fund'] = open_fund_name_to_id
data = {
'stock_pool': stock_pool,
'exchanged_fund_pool': exchanged_fund_pool,
'open_fund_pool': open_fund_pool,
'industry_pool': industries,
'index_pool': indices
}
key_name = 'data-' + start_date + '-' + end_date + '.pkl'
red[key_name] = json.dumps(data)
red['names_to_id'] = json.dumps(names_to_id)
def ma_func(pd, n):
sum_c = 0
pd_t = pd.tail(n)
# print(pd_t)
for i in range(n):
sum_c = sum_c + pd_t.iat[i]
avg = round(sum_c / n, 2)
return avg
if __name__ == '__main__':
jq.auth("13811944446", "1qazxsw2")
print(jq.get_query_count())
res_sts = []
stock_SH = jq.get_index_stocks('000001.XSHG')
stock_SZ = jq.get_index_stocks('399001.XSHE')
stock_cy = jq.get_index_stocks('399006.XSHE')
stocks = stock_SZ + stock_SH + stock_cy
stocks = list(set(stocks))
print(len(stocks))
# stocks = ['003816.XSHE']
today = dt.date.today()
k = 60
panel = jq.get_price(stocks,
count=k,
end_date=today,
frequency='daily',
fields=['close', 'volume'])
df_close = panel['close']
df_vol = panel['volume']
import pandas as pd
import numpy as np
import jqdatasdk as jq
from jqdatasdk import auth, get_query_count, get_price, opt, query, get_fundamentals, finance, get_trade_days, \
valuation, get_security_info, get_index_stocks, get_bars, bond
from tqdm import tqdm
from datetime import datetime, time, timedelta
import matplotlib.pyplot as plt
import os
from basic_funcs.basic_function import *
auth('15951961478', '961478')
get_query_count()
import warnings
warnings.filterwarnings('ignore')
dateparse = lambda x: pd.datetime.strptime(x, '%Y-%m-%d')
plt.rcParams['font.sans-serif'] = ['Songti SC']
plt.rcParams['axes.unicode_minus'] = False # 用来正常显示负号
close = pd.read_csv('/Users/caichaohong/Desktop/Zenki/price/daily/close.csv',
index_col='Unnamed: 0',
date_parser=dateparse)
z = pd.DataFrame(index=close.index)
z['hs300'] = np.nan
for tmp_date in tqdm(close.index):
hs300_list = get_index_stocks('000300.XSHG', date=tmp_date)
z['hs300'].loc[tmp_date] = hs300_list
z.to_csv('/Users/caichaohong/Desktop/Zenki/index_holdings.csv')
def getZz500(): # 中证500
return jq.get_index_stocks('000905.XSHG')
import matplotlib.pyplot as plt
import os
from basic_funcs.basic_function import *
from sklearn.linear_model import LinearRegression
from sklearn.metrics import explained_variance_score, mean_absolute_error, mean_squared_error, median_absolute_error, \
r2_score, accuracy_score
from sklearn.model_selection import train_test_split
from xgboost import XGBClassifier, plot_importance
from sklearn.preprocessing import StandardScaler
auth('15951961478', '961478')
get_query_count()
dateparse = lambda x: pd.datetime.strptime(x, '%Y-%m-%d')
hs300_list = get_index_stocks('000300.XSHG')
hs300 = pd.read_excel(
'/Users/caichaohong/Desktop/Zenki/price/510300.XSHG.xlsx',
index_col='Unnamed: 0')
hs300['rts_1'] = hs300['close'].pct_change(1)
hs300['rts_interval_1'] = transform_300_rts_to_daily_intervals(hs300['rts_1'])
hs300['rts_5'] = hs300['close'].pct_change(5)
hs300['rts_10'] = hs300['close'].pct_change(10)
hs300['net_value'] = hs300['close'] / hs300['close'][0]
close = pd.read_csv('/Users/caichaohong/Desktop/Zenki/price/daily/close.csv',
index_col='Unnamed: 0',
date_parser=dateparse)
high = pd.read_csv('/Users/caichaohong/Desktop/Zenki/price/daily/high.csv',
index_col='Unnamed: 0',
def getSh50():
return jq.get_index_stocks('000016.XSHG')
import graphviz
import pickle
from gplearn import genetic
from gplearn.functions import make_function
from gplearn.genetic import SymbolicTransformer, SymbolicRegressor
from gplearn.fitness import make_fitness
from sklearn.utils import check_random_state
from sklearn.model_selection import train_test_split
import jqdatasdk as jq
jq.auth('18903041915', 'iamaman369')
#%% 1. 数据读入
stock_list = jq.get_index_stocks('000300.XSHG')
start_date = '2020-01-01'
end_date = '2020-06-30'
fields = ['open', 'close', 'low', 'high', 'volume', 'avg', 'pre_close']
stock_price = jq.get_price(stock_list,
start_date=start_date,
end_date=end_date,
fq=None,
fields=fields)
stock_price = stock_price.sort_values(by=['code', 'time']).reset_index(
drop=True)
stock_price['time'] = stock_price['time'].apply(
lambda x: datetime.datetime.strftime(x, '%Y%m%d'))
stock_price['5'],stock_price['10'],stock_price['15'],stock_price['20'],stock_price['30'],\
stock_price['60'],stock_price['100'],stock_price['120'],stock_price['200'] = 5,10,15,20,30,60,100,120,200
'''
#start index weight part
def show_index(self, name=None):
for i in self.index_name_list:
if (name != None and name not in i[1]):
continue
print(i)
def get_index_stocks(self, indexi, date=None):
if (date == None):
date = datetime.date.today().strftime('%Y-%m-01')
tablename = '{}_index_weight_{}'.format(index, date).replace('.', '')
if (not self.ld.has_table(tablename)):
self.create_index_weight_table(index, date)
def create_index_weight_table(self, index, date):
tablename = '{}_index_weight_{}'.format(index, date).replace('.', '')
pass
if __name__ == "__main__":
k = MyDataset()
k.init()
lis = jqs.get_index_stocks('000300.XSHG')
#k.show_index()
#print("==delim==")
#k.show_index('成长指数')
#p = k.get_stock_data_start_end('600435','2020-11-01','2021-01-01')
p = k.get_multi_stock_data_start_end(lis, '2018-01-01', '2021-01-31')
print(p)
import os
root_path = os.path.dirname(os.path.realpath(__file__)) + '/../'
sys.path.append(root_path + 'scripts/')
import utils
import utils_jq
import jqdatasdk
if len(sys.argv) < 2:
print("usage: ./dump_compo.py date index")
quit()
d = sys.argv[1]
idx = sys.argv[2]
country = utils.country_map[idx]
# csi300: 000030.XSHG csi500: 000905.XSHG
compo_mapping = {'csi300': '000300.XSHG', 'csi500': '000905.XSHG'}
compo_path = root_path + 'data/compo/' + idx + '/'
if not utils.is_bday(d, country):
print('{} is {} holiday'.format(d, country))
quit()
stocks = jqdatasdk.get_index_stocks(compo_mapping[idx], date = utils.date_str(d))
t = jqdatasdk.get_index_weights(compo_mapping[idx], date = utils.date_str(d))
t = t.reset_index()
t['weight'] = t['weight'] / 100
t = t.drop(columns = ['date'])
t = t.rename(columns={'index': 'ric', 'display_name': 'ric_name'})
t = t.drop_duplicates()
t = t[['ric', 'ric_name', 'weight']]
print(compo_path + '{}.txt'.format(d))
t.to_csv(compo_path + '{}.txt'.format(d), sep='\t', index = False)
# JQ数据提取
#开始
import numpy as np
import pandas as pd
import jqdatasdk as jq
jq.auth("15102022205","steven990131")
#获取收盘行情（'open','close','low','high','volume','money','avg','pre_close'）
indexs=jq.get_index_stocks('000903.XSHG')
A=jq.get_price(indexs, start_date='2019-05-08', end_date='2019-05-08', frequency='daily', fields=['open','close','low','high','volume','money','avg','pre_close'], skip_paused=False, fq='pre')
B=A.to_frame()
B.to_csv('C:/Users/Administrator/Desktop/收盘行情20190508.csv')
#提取估值（'pe_ratio','turnover_ratio','pb_ratio','ps_ratio',market_cap','circulating_market_cap','pe_ratio_lyr'）
def maintask():
a=pd.DataFrame()
j=1
for i in indexs:
print('正在获取第%d家，股票代码%s.' % (j, i))
j+=1
q = jq.query(jq.valuation).filter(jq.valuation.code == i)
df = jq.get_fundamentals(q, '2019-05-08')
a=a.append(df)
a.to_csv("C:/Users/Administrator/Desktop/估值20190508.csv")
print(df)
#print(a)
if __name__ == '__main__':
maintask()
def get_index_stocks(index):
JqDataRetriever.authenticate()
return jqdatasdk.get_index_stocks(index)
hs300['rts_1'] = hs300['close'].pct_change(1)
hs300['rts_5'] = hs300['close'].pct_change(5)
hs300['rts_10'] = hs300['close'].pct_change(10)
hs300['net_value'] = hs300['close'] / hs300['close'][0]
close = pd.read_csv('/Users/caichaohong/Desktop/Zenki/price/daily/close.csv',
index_col='Unnamed: 0',
date_parser=dateparse)
close_rts_1 = close.pct_change(1)
margin_total_value = pd.read_csv(
'/Users/caichaohong/Desktop/Zenki/融资融券/margin_total_value.csv',
index_col='Unnamed: 0',
date_parser=dateparse)
# excess margin factor
hs300_list = get_index_stocks('000300.XSHG')
list_500 = get_index_stocks('000905.XSHG')
n = 10
z = close.pct_change(n).rank(axis=1) / margin_total_value.pct_change(n).rank(
axis=1)
zz = abs(z.sub(z.median(axis=1), axis=0))
# pp = get_params_out(top_number_list=[10,20,30,40,50], hold_time_list=[1,5,10,20], factor_df=zz[hs300_list], rts_df=close_rts_1[hs300_list])
# pp = pp.sort_values(by='annual_rts',ascending=False)
# pp.to_excel('/Users/caichaohong/Desktop/Zenki/params.xlsx')
value_rts = get_top_value_factor_rts(factor=zz[hs300_list],
rts=close_rts_1[hs300_list],
top_number=10,
hold_time=10)
plot_rts(value_rts=value_rts['daily_rts'],
def index_components(self, order_book_id, dt=None):
return get_index_stocks(index_symbol=order_book_id, date=dt)
def getSh180(): # 上证180
return jq.get_index_stocks('000010.XSHG')
import jqdatasdk as jd
jd.auth('18359196774','acginor1992CR')
date_init='2013-01-01'
stocks=jd.get_index_stocks('000300.XSHG',date=date_init)
print(len(stocks))
from ths_slj.config import JQ_CONFIG, DATA_PATH
train_start_date = '2017-01-01'
train_end_date = '2018-12-31'
padding = 150
test_start_date = '2019-01-01'
test_end_date = '2019-12-31'
if __name__ == '__main__':
if not jq.is_auth():
jq.auth(JQ_CONFIG.get('id'), JQ_CONFIG.get('password'))
codes = jq.get_index_stocks('000852.XSHG', date=test_end_date)
all_codes = jq.get_all_securities('stock')
active_codes = []
# 筛选所有的股票
for code in codes:
_temp = all_codes.loc[code]
code_start = pd.Timestamp(_temp.start_date)
code_end = pd.Timestamp(_temp.end_date)
if code_start < pd.Timestamp(train_start_date) and code_end > pd.Timestamp(test_end_date):
active_codes.append(code)
# 剩余846只
df_train = None
for code in tqdm(active_codes):
'size': 12
}
matplotlib.rc("font", **font)
from datetime import datetime
import seaborn as sns
# 指数
index_code = '000906.XSHG'
# 行业分类标准
industry_type = 'sw_l1'
# 宽度日期
market_breadth_days = 30
# 获取每个行业 以及 获取每个行业包含中证800的成分股的个数
zz800 = get_index_stocks('000906.XSHG')
zz800_industry = get_industry(zz800, date=datetime.strftime(datetime.today(), '%Y-%m-%d'))
zz800_industry = pd.DataFrame(zz800_industry).T[[industry_type]]
zz800_industry[industry_type] = zz800_industry[industry_type].apply(lambda x: x['industry_name'])
industries = zz800_industry[industry_type]
industries = list(set(industries))
industries.sort()
# 创建表格 列是日期，行是行业，最后需要转置
market_breadth = pd.DataFrame(index=(['zz800'] + industries))
# 根据交易日来计算宽度
# 最近80天交易日
# 然后倒序[::-1]
import jqdatasdk as jq
import numpy as np
import pandas as pd
import sys, os, datetime
import lib.feature_extract as fe
# set output
pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)
# fetching data
jq.auth('18611850602', 'jixian')
securities = jq.get_index_stocks('000300.XSHG')
knowledge_base = pd.DataFrame()
knowledge_base_filename = "data/knowledge_base.csv"
test_cache_filename = 'data/cache/test.csv'
i = 0
for security in securities:
start_date = datetime.date(2005, 1, 1)
end_date = datetime.date(2016, 12, 30)
frequency = 'daily'
raw_filename = 'data/cache/raw-{}-{}-{}-{}.csv'.format(
security, start_date, end_date, frequency)
# laod dataset
if os.path.isfile(raw_filename):
# print("Loading data from cache")
