def get_start_date(
self,
code_list: Union[str,
Tuple[str],
List[str]],
factor_time_range: list
):
"""
获取上市时间
"""
stock_list = utils.QA_fmt_code_list(code_list, style="jq")
df_local = jqdatasdk.get_all_securities(types="stock")
intersection = list(df_local.index.intersection(stock_list))
ss = df_local.loc[intersection]["start_date"]
ss.index = ss.index.map(lambda x: x[:6])
# 日期处理
date_range = list(map(lambda x: x.date(), factor_time_range))
multiindex = pd.MultiIndex.from_product(
[date_range,
utils.QA_fmt_code_list(intersection)]
)
values = multiindex.map(lambda x: ss.loc[x[1]]).tolist()
df_local = pd.Series(index=multiindex, data=values)
df_local.index.names = ["date", "code"]
return df_local
def train(train_cfg):
with open(train_cfg,"r") as f:
train_cfg = json.load(f)
#UserDataApi = UserDataApi()
feature_cfg = train_cfg["feature_cfg"]
model_name = train_cfg["model"]
model_cfg = train_cfg["model_cfg"]
#stock_list = jq.get_index_stocks('000300.XSHG')#jq.get_industry_stocks('I64')
stock_list = jq.get_all_securities().index.tolist()
#stock_list= ['300014.XSHE']
print (stock_list)
feature_cfg = "./config/feature_create_cfg.json"
with open(feature_cfg,"r") as f:
feature_cfg = json.load(f)
info = create_feature(feature_cfg,stock_list,if_dump = True,load_feature = False)
feature = info["feature"]
label = info["label"]
label_index = 0
clip_index = info["label_clip_index"][:,label_index]
label = label[:,label_index]
feature = feature[clip_index]
label = label[clip_index]
model = build_model(model_name,model_cfg)
model.fit(feature,label)
model.dump()
def get_all_securities():
'''获取全部股票信息'''
res = jq.get_all_securities(types=['stock'], date=None)
'''删除type字段'''
res = res.drop(['type'], axis=1)
'''清表'''
mdbconn = mysql.connector.connect(
user='******',
password='******',
database='jqdata',
use_unicode=True)
cursor = mdbconn.cursor()
cursor.execute('truncate table t_all_securities')
mdbconn.commit()
cursor.close()
# print('truncate table t_all_securities success')
'''DataFrame入库'''
pymysql.install_as_MySQLdb()
mysqlconnect = create_engine(
'mysql+mysqldb://root:12345678@localhost:3306/jqdata?charset=utf8')
res.to_sql(
name='t_all_securities',
con=mysqlconnect,
schema='jqdata',
if_exists='append',
index=True,
index_label='security',
chunksize=1000)
print('all securities saved in t_all_securities success')
return
def get_universe(d):
t = jqdatasdk.get_all_securities(types=[], date=d)
t = t[t['display_name'].str.count('ST') == 0]
t = t[t['display_name'].str.count('退') == 0]
#rics = [x for x in list(t.index) if x.startswith('60') or x.startswith('00')
rics = list(t.index)
return rics
def downloadMinuteBarByDate(start_date, end_date=datetime.today().date()):
jqdatasdk.auth(JQDATA_USER, JQDATA_PASSWORD)
"""下载所有配置中的合约的分钟线数据"""
print('-' * 50)
print(u'开始下载合约分钟线数据')
print('-' * 50)
trade_date_list = jqdatasdk.get_trade_days(start_date=start_date,
end_date=end_date)
i = 0
for trade_date in trade_date_list:
if i == 0:
i = 1
continue
symbols_df = jqdatasdk.get_all_securities(types=['futures'],
date=trade_date)
for index, row in symbols_df.iterrows():
downMinuteBarBySymbol(index, row, str(trade_date_list[i]),
str(trade_date_list[i - 1]))
i += 1
print('-' * 50)
print(u'合约分钟线数据下载完成')
print('-' * 50)
return
def QA_fetch_get_factor_start_date(factor: pd.Series) -> pd.DataFrame:
"""
获取因子池上市时间, 注意，请自行登陆聚宽本地 sdk
参数
---
:param factor: 因子值，索引为 ['日期' '资产']
返回值
---
:return: 因子数据
"""
# 因子格式化
factor = QA_fmt_factor(factor.copy())
merged_data = pd.DataFrame(factor.rename("factor"))
# 股票代码格式化
stock_list = QA_fmt_code_list(
factor.index.get_level_values("code").drop_duplicates(), style="jq")
# 上市时间获取
df_local = jqdatasdk.get_all_securities(types="stock")
intersection = df_local.index.intersection(stock_list)
ss = df_local.loc[intersection]["start_date"]
ss.index = ss.index.map(lambda x: x[:6])
# 拼接上市时间
merged_data = merged_data.loc[(slice(None), list(ss.index)), :]
merged_data["start_date"] = merged_data.index.map(
lambda x: ss.loc[x[1]]).tolist()
return merged_data
def run(self):
# 抓取etf列表
df_index = self.to_zvt_entity(get_all_securities(['etf']), entity_type='etf', category='etf')
df_to_db(df_index, data_schema=Etf, provider=self.provider, force_update=self.force_update)
# self.logger.info(df_index)
self.logger.info("persist etf list success")
logout()
def load_all_stock_code(self):
data = get_all_securities(types=['multi_factor'],
date=datetime.today())
data.index = data.index.map(self.change_code_format_from_jq)
file = os.path.join(self.primary_data_path,
r"stock_data\stock_basic_data\jq_all_code.csv")
data.to_csv(file)
def get_stock_list(self, start_date=None):
# 返回全部股票信息列表
data = None
if (self.login):
data = jqs.get_all_securities(['stock'], date=start_date)
self.log.log(
"Log : Get all stock list " + "from date {}".format(start_date)
if start_date != None else "", self.name)
return data
def get_halfhour_line_all(engine, start_d, end_d):
all_stocks = list(jq.get_all_securities(['stock']).index)
for one_code in all_stocks:
print "%s, fetching %s (%s ~ %s)" % (datetime.now(), one_code, start_d,
end_d)
get_spec_halfhour_line(engine, start_d, end_d, one_code,
[0, 7]) #要第一个‘半小时’，和最后一个‘半小时’
check_api_quote()
def download_all_securities(types=[], date=None, data_folder='../local_data/joinquant'):
# 'stock', 'fund', 'index', 'futures', 'options', 'etf', 'lof', 'fja', 'fjb', 'open_fund', 'bond_fund', 'stock_fund',
# 'QDII_fund', 'money_market_fund', 'mixture_fund'
name = 'stock'
if len(types) > 0:
name = ''.join([t + '_' for t in types])
name = name[:-1]
securities = jqdatasdk.get_all_securities(types=types, date=date) # type: pd.DataFrame
securities.to_csv(os.path.join(data_folder, name + '.csv'))
def insertInfo(self,start_date,end_date):
period_trade_date = jq.get_trade_days(start_date=start_date, end_date=end_date) # include start_date,end_date
#period_trade_date = TradeDayTable.fetch_period_trade_days(self.database,start_date=start_date, end_date=end_date)
for date in period_trade_date:
print ("inserting all_security table,date：",date)
df = jq.get_all_securities(types=[], date=date)
df = self.transform_2_jq_loc(df,date)
self.table.insert_many(json.loads(df.T.to_json()).values())
self.createIndex()
def get_large_data(self):
global data_exist
f = open("futures_list.txt", "w+")
lis = f.read()
data_exist = lis.split(" ")
# log in the JQ account
jq.auth('18224433211', 'Haohao19971003.')
all_futures = jq.get_all_securities(['futures'])
# name the index of the DataFrame
all_futures.index.name = 'id'
# convert index column to list
futures_idx = all_futures.index.values.tolist()
substring = [
'IC9999.CCFX', 'IC8888.CCFX', 'IF9999.CCFX', 'IF8888.CCFX',
'IH9999.CCFX', 'IH8888.CCFX', 'T9999.CCFX', 'T8888.CCFX',
'TF9999.CCFX', 'TF8888.CCFX', 'TS9999.CCFX', 'TS8888.CCFX'
]
empty = []
for idx in futures_idx:
for element in substring:
if idx == element:
empty.append(idx)
count = 0
for item in empty:
future_info_yearly = jq.get_price(
security=item,
start_date=datetime.strptime(self.f_start, '%Y-%m-%d'),
end_date=datetime.strptime(self.f_end, '%Y-%m-%d'),
frequency='1m')
str = item + '.csv'
future_info_yearly.index.name = 'time'
future_info_yearly.dropna(inplace=True)
if not future_info_yearly.empty:
future_info_yearly.to_csv(os.path.join(self.path, str),
mode='a')
if item not in data_exist:
data_exist.append(item)
count += 1
print(count)
f = open("futures_list.txt", "w")
f.write(" ".join(data_exist))
f.close()
for item in data_exist:
if item == "":
continue
str = item + '.csv'
df = pd.read_csv(self.path + str)
df.drop_duplicates(subset=['time'], inplace=True)
df.sort_values(by='time', inplace=True)
df = df[df.time.str.contains('time') == False]
df.to_csv(os.path.join(self.path, str),
index=False,
index_label='time')
print(data_exist)
def isPublic(date_time, stock_list):
v = np.zeros(len(stock_list), dtype=np.int)
#result = jq.get_price(list(stock_list), start_date=date_time, end_date=date_time, frequency='daily', fields=['paused'], skip_paused=False, fq='pre', count=None, panel=True, fill_paused=True)
result = jq.get_all_securities(types=[], date=date_time)
#print (result)
code_list = list(result.index)
for i, stock in enumerate(stock_list):
if stock in code_list:
v[i] = 1
return v.astype(np.bool)
def run(self):
# 抓取股票列表
df_stock = self.to_zvt_entity(get_all_securities(['stock']), entity_type='stock')
df_to_db(df_stock, data_schema=Stock, provider=self.provider, force_update=self.force_update)
# persist StockDetail too
df_to_db(df=df_stock, data_schema=StockDetail, provider=self.provider, force_update=self.force_update)
# self.logger.info(df_stock)
self.logger.info("persist stock list success")
logout()
def insertInfo(self,start_date,end_date):
#
period_trade_date = jq.get_trade_days(start_date=start_date, end_date=end_date) # include start_date,end_date
for date in period_trade_date:
print ("inserting unfq_price table,date：",date)
securities = jq.get_all_securities(types=[], date=date)
df = jq.get_price(security = list(securities.index),start_date=date, end_date=date, frequency='daily', fields=fields, skip_paused=False, fq='none', count=None, panel=False, fill_paused=False)
df = self.transform_2_jq_loc(df)
self.table.insert_many(json.loads(df.T.to_json()).values())
self.createIndex()
def get_index_stocks(code, trade_date, data_source):
# 通过深证综指399106.XSHE来观察全市场数据
stocks = []
# 主要数据从JQData进行获取
if data_source == 'jqdata':
if code == '399106.XSHE':
stocks = list(jqdatasdk.get_all_securities(date=trade_date).index)
else:
stocks = jqdatasdk.get_index_stocks(code, trade_date)
# 通过tushare提供其他不包含数据
elif data_source == 'tushare':
stocks = self_data_source.get_index_stocks(code, trade_date)
return stocks
def get_futures_list():
# log in the JQ account
jq.auth('18224433211', 'Haohao19971003.')
all_futures = jq.get_all_securities(['futures'])
# name the index of the DataFrame
all_futures.index.name = 'id'
# convert index column to list
futures_idx = all_futures.index.values.tolist()
substring = '8888'
empty = []
for idx in futures_idx:
if substring in idx:
empty.append(idx)
return empty
def get_price(begin, end):
jq.auth('', '')
get_security = jq.get_all_securities(types=['stock'], date=None)
sec_list = get_security.index.to_list()
df_price = jq.get_price(
sec_list,
start_date=begin,
end_date=end,
frequency='daily',
fields=['open', 'close', 'high', 'low', 'volume', 'pre_close'],
skip_paused=False,
fq='pre',
panel=True)
df_price.to_csv('stock_price.csv', index=False)
def p_perday_var(start_date, end_date):
tradeable_days = get_trade_days(start_date=start_date, end_date=end_date)
for trade_day in tradeable_days:
stock_code_list = jq.get_all_securities(date=trade_day)
result = jq.get_price(list(stock_code_list),
start_date=trade_day,
end_date=trade_day,
frequency='daily',
fields=['close'],
skip_paused=False,
fq='pre',
count=None,
panel=True,
fill_paused=True)
def get_all_securities(self, types=[]):
"""
获取全部股票信息,更新标的信息，没有的添加，有的看其是否已经被st,退市，进行更新。
"""
flog.FinanceLoger.logger.info('证券信息更新开始...!')
now = datetime.datetime.now()
db_session = self.db_session_factory()
list_screening = db_session.query(Setting).filter(
Setting.name == 'security.down.last').first()
list_date = datetime.datetime.strptime(list_screening.value,
'%Y-%m-%d')
day_count = (now - list_date).days
if day_count < self.app.conf.conf['Update']['SecuritiesInterval']:
return
for x in types:
res = jq.get_all_securities(types=x, date=None)
i = 0
for index, security in res.iterrows():
s = index
security_temp = db_session.query(YSecurity).filter(
YSecurity.security == s).first()
if security_temp:
security_temp.security = s
security_temp.display_name = security["display_name"]
security_temp.name = security['name']
security_temp.start_date = security['start_date']
security_temp.end_date = security["end_date"]
security_temp.update_date = now.date()
else:
security_temp = YSecurity(
security=s,
display_name=security["display_name"],
name=security["name"],
start_date=security["start_date"],
end_date=security["end_date"],
stype=security["type"],
status=0,
update_date=now.date())
db_session.add(security_temp)
db_session.commit()
i += 1
flog.FinanceLoger.logger.info('本次标[{}]的更新完成，共更新{}条!'.format(x, i))
list_screening.value = now.date().strftime('%Y-%m-%d')
db_session.commit()
db_session.close()
flog.FinanceLoger.logger.info('证券信息更新结束...!')
return
def fetch_magic_candidators(engine, t_day):
# 中证价值回报量化策略指数的样本空间由满足以下条件的沪深 A 股构成：
# （1）非 ST、*ST 股票，非暂停上市股票； （2）非金融类股票。
all_stocks = list(jq.get_all_securities(types=['stock'], date=t_day).index)
#all_stocks = all_stocks[:100]
# 排除金融类的股票
banks = jq.get_industry_stocks('J66', date=t_day)
brokers = jq.get_industry_stocks('J67', date=t_day)
insurances = jq.get_industry_stocks('J68', date=t_day)
others = jq.get_industry_stocks('J69', date=t_day)
exclude = banks + brokers + insurances + others
filtered_1 = []
for code in all_stocks:
if (code in exclude):
print " ... %s 是金融类, 排除..." % code
continue
filtered_1.append(code)
# 排除 ST
st = jq.get_extras('is_st',
filtered_1,
start_date=t_day,
end_date=t_day,
df=False)
filtered_2 = []
for code in filtered_1:
if st[code][0]:
print " ... %s 是ST，排除" % code
continue
filtered_2.append(code)
filtered_3 = []
# 排除停牌
for code in filtered_2:
if data_fetcher.is_paused(engine, code, t_day):
print " ... %s 停牌，排除" % code
continue
filtered_3.append(code)
return filtered_3
def __init__(self):
self.cursor_date = str(datetime.date.today())[:10]
# 聚宽行情，默认当前分钟 10 秒之后可以获取前一分钟行情
self.day_start = pd.Timestamp(self.cursor_date + " 09:32:00")
# 行情结束时间
self.day_end = pd.Timestamp(self.cursor_date + " 15:00:00")
# 当前时间标尺
self.cursor_time = datetime.datetime.now()
# 是否交易标志
self.on_trade_flag = True
# 股票信息
df_stock_info = (jqdatasdk.get_all_securities(
types="stock").reset_index().rename(columns={
"index": "stock_code",
"display_name": "stock_name"
}))
def load_top10_data(fun=(lambda x: x), ext="all", force=False):
path = "top10_data_%s_%s.csv" % (
ext, time.strftime("%Y-%m-%d", time.localtime()))
if not force and os.path.exists(path):
return pd.read_csv(path)
else:
jqdatasdk.auth("******", "*******")
stocks = list(jqdatasdk.get_all_securities(['stock']).index)
df_list = []
for code in stocks:
query = jqdatasdk.query(finance.STK_SHAREHOLDER_TOP10).filter(
finance.STK_SHAREHOLDER_TOP10.code == code,
finance.STK_SHAREHOLDER_TOP10.pub_date > '2019-01-01')
df_list.append(fun(finance.run_query(query)))
top10_data = pd.concat(df_list, ignore_index=True)
top10_data.to_csv(path, index=False)
return top10_data
def get_stocks():
global g
if (g['stocks'] and g['names']):
print('已存在 从本地获取')
else:
df = jqdatasdk.get_all_securities(['stock'])
# 将所有股票列表转换成数组
stocks = list(df.index)
names = list(df.display_name)
g['stocks'] = stocks
g['names'] = names
print('没有数据从远程获取')
# print(stocks)
return jsonify({'stocks': g['stocks'], 'names': g['names']})
def downloadAllMinuteBar():
jqdatasdk.auth(JQDATA_USER, JQDATA_PASSWORD)
"""下载所有配置中的合约的分钟线数据"""
print('-' * 50)
print(u'开始下载合约分钟线数据')
print('-' * 50)
today = datetime.today().date()
trade_date_list = jqdatasdk.get_trade_days(end_date=today, count=2)
symbols_df = jqdatasdk.get_all_securities(types=['futures'], date=today)
for index, row in symbols_df.iterrows():
downMinuteBarBySymbol(index, row, str(today), str(trade_date_list[-2]))
print('-' * 50)
print(u'合约分钟线数据下载完成')
print('-' * 50)
return
async def get_security_list(self) -> np.ndarray:
"""
Returns:
"""
if not self.connected:
logger.warning("not connected")
return None
types = ["stock", "fund", "index", "futures", "etf", "lof"]
securities = jq.get_all_securities(types)
securities.insert(0, "code", securities.index)
# remove client dependency of pandas
securities["start_date"] = securities["start_date"].apply(
lambda s: f"{s.year:04}-{s.month:02}-{s.day:02}")
securities["end_date"] = securities["end_date"].apply(
lambda s: f"{s.year:04}-{s.month:02}-{s.day:02}")
return securities.values
def get_all_domain():
""" stk_code, display_name """
futures_all = jq.get_all_securities(types=['futures'])
stk_all_mf = futures_all.loc[futures_all.apply(
lambda x: NlReFilter.filter_num_from_str(x['name']) in ['9999'],
axis=1), :]
# 将主力转为实际
for idx in stk_all_mf.index:
name_str = stk_all_mf.loc[idx, 'name']
if NlReFilter.filter_num_from_str(name_str) == '9999':
dmt = jq.get_dominant_future(
NlReFilter.filter_letter_from_str(name_str))
stk_all_mf.loc[
idx,
'display_name'] = stk_all_mf.loc[idx, 'display_name'] + dmt
stk_all_mf.loc[idx, 'dominant'] = dmt
stk_all_mf.loc[:, 'stk_code'] = stk_all_mf.index
return stk_all_mf
def set_JQDOTA_code_compare(method='http',
reset=False,
account=None,
password=None,
remember=False,
client=DATABASE,
ui_log=None):
if method == 'api':
JQDATA_login(account=None, password=None, remember=False)
if reset: client.drop_collection('jqdata_securities_record')
coll = client.jqdata_securities_record
coll.create_index([("jqcode", pymongo.ASCENDING),
("jqcode_simple", pymongo.ASCENDING),
("type", pymongo.ASCENDING)],
unique=True)
ref_ = coll.find()
if (ref_.count() <= 0) | (reset == True):
if method == 'api':
security_info = jqdatasdk.get_all_securities([
'stock', 'fund', 'index', 'futures', 'etf', 'lof', 'fja', 'fjb'
])
# jqdatasdk.logout()
elif method == 'http':
security_info = jqdatahttp.get_all_securities([
'stock', 'fund', 'index', 'futures', 'etf', 'lof', 'fja', 'fjb'
])
security_info = security_info.reset_index().rename(
columns={'index': 'jqcode'})
security_info['jqcode_simple'] = list(
map(lambda x: x.split('.')[0], security_info['jqcode']))
print(security_info)
coll.insert_many(QA_util_to_json_from_pandas(security_info))
QA_util_log_info('# Saving success: jqdata_securities_record', ui_log)
else:
pass
dm_old_list = []
sql = "SELECT dm ,name FROM dmb order by id "
cursor.execute(sql)
for a in cursor.fetchall():
dm_old_list.append(list(a))
#获取现有日期表
date_old_list = []
sql = "SELECT date FROM rqb order by id "
cursor.execute(sql)
for b in cursor.fetchall():
date_old_list.append(b[0])
#获取最新代码表
dm_new_list = []
df_new = jq.get_all_securities(date=d)
df_new.drop(columns=['start_date', 'end_date', 'type', 'name'],
axis=1,
inplace=True)
df_new.rename(columns={'display_name': 'name'}, inplace=True)
df_new.reset_index(inplace=True, drop=False)
df_new.rename(columns={'index': 'dm'}, inplace=True)
for i in range(0, len(df_new)):
df_new.iloc[i]['dm'] = df_new.iloc[i]['dm'][:6]
dm_new_list = df_new.values.tolist()
#更新上证指数
szzs_df = jq.get_price('000001.XSHG',
end_date=d,
frequency='daily',
fields=['open', 'close', 'high', 'low', 'volume'],