def update_margin_buy(new_start_date, new_end_date, margin_df):
future_trade_days = get_trade_days(start_date=margin_df.index[-1], end_date=new_end_date)[1:] # 第一天重复
old_trade_days = get_trade_days(start_date=new_start_date, end_date=margin_df.index[0])[:-1] # 最后一天重复
new_trade_days = list(future_trade_days) + list(old_trade_days)
if len(new_trade_days) > 0:
for date in new_trade_days:
margin_df.loc[date] = np.nan
margin_buy_value.index = pd.to_datetime(margin_buy_value.index) # 最后新加的行index不是datetime
for stock in tqdm(margin_df.columns):
if len(future_trade_days) > 0:
for d in future_trade_days:
margin_df.loc[d] = np.nan
margin_buy_value.index = pd.to_datetime(margin_buy_value.index) # 最后新加的行index不是datetime
df1 = get_mtss(stock, future_trade_days[0], future_trade_days[-1], fields=['date', 'sec_code', 'fin_buy_value'])
df1.index = pd.to_datetime(df1['date'])
margin_df[stock].loc[df1.index] = df1['fin_buy_value'].values
if len(old_trade_days) > 0:
for d in old_trade_days:
margin_df.loc[d] = np.nan
margin_buy_value.index = pd.to_datetime(margin_buy_value.index) # 最后新加的行index不是datetime
df2 = get_mtss(stock, old_trade_days[0], old_trade_days[-1], fields=['date', 'sec_code', 'fin_buy_value'])
df2.index = pd.to_datetime(df2['date'])
margin_df[stock].loc[df2.index] = df2['fin_buy_value'].values
margin_df.to_csv('/Users/caichaohong/Desktop/Zenki/融资融券/margin_buy_value.csv')
else:
print("No need to Update")
def update_st(st_df, new_end_date, new_start_date):
future_trade_days = get_trade_days(start_date=st_df.index[-1],
end_date=new_end_date)[1:] # 第一天重复
old_trade_days = get_trade_days(start_date=new_start_date,
end_date=st_df.index[0])[:-1] # 最后一天重复
new_trade_days = list(future_trade_days) + list(old_trade_days)
if len(new_trade_days) > 0:
for date in new_trade_days:
st_df.loc[date] = np.nan
get_extras(info,
security_list,
start_date='2015-01-01',
end_date='2015-12-31',
df=True,
count=None)
st_df.loc[date][df['code']] = df['circulating_market_cap'].values
else:
print("No need to Update")
st_df.index = pd.to_datetime(st_df.index)
st_df = st_df.sort_index(axis=0)
st_df = st_df.sort_index(axis=1) # 按股票代码排序
st_df = st_df.dropna(how='all', axis=0)
st_df.to_csv('/Users/caichaohong/Desktop/Zenki/price/is_st.csv')
def initDateIndexDict(self):
self.date_arr = jq.get_trade_days(self.start_date, self.end_date)
self.date_list = dateArr2List(self.date_arr)
self.trade_date_arr = jq.get_trade_days(trade_date_begin,
self.end_date)
self.trade_date_list = dateArr2List(self.trade_date_arr)
self.date_index_dict = list2Dic(self.trade_date_list)
self.inverse_date_index_dict = invert_dict(self.date_index_dict)
def update_north_data(new_end_date, new_start_date, share, ratio, value,close):
if (share.index[0] == ratio.index[0]) & (ratio.index[0] == value.index[0]):
print('all_start_time_equal{}'.format(share.index[0]))
else:
print('START time NOT equal')
if (share.index[-1] == ratio.index[-1]) & (ratio.index[-1] == value.index[-1]):
print('all_start_time_equal{}'.format(share.index[-1]))
else:
print('START time NOT equal')
future_trade_days = get_trade_days(start_date=share.index[-1], end_date=new_end_date)[1:] # 第一天重复
old_trade_days = get_trade_days(start_date=new_start_date, end_date=share.index[0])[:-1] # 最后一天重复
new_trade_days = list(future_trade_days) + list(old_trade_days)
for date in new_trade_days:
share.loc[date] = np.nan
ratio.loc[date] = np.nan
value.loc[date] = np.nan
df = finance.run_query(
query(finance.STK_HK_HOLD_INFO).filter(finance.STK_HK_HOLD_INFO.link_id.in_([310001, 310002]),
finance.STK_HK_HOLD_INFO.day.in_(new_trade_days)))
new_stocks = list(set(df['code']).difference(set(share.columns)))
for s in new_stocks:
share[s] = np.nan
ratio[s] = np.nan
value[s] = np.nan
for date in tqdm(new_trade_days):
temp = finance.run_query(
query(finance.STK_HK_HOLD_INFO).filter(finance.STK_HK_HOLD_INFO.link_id.in_([310001, 310002]),
finance.STK_HK_HOLD_INFO.day == date))
share.loc[date][temp['code']] = temp['share_number'].values # 有error???????
ratio.loc[date][temp['code']] = temp['share_ratio'].values
tmp_share = share.loc[date]
tmp_price = close.loc[date][share.columns]
value.loc[date] = tmp_price * tmp_share * 10 ** (-8)
share = share.sort_index(axis=1)
ratio = ratio.sort_index(axis=1)
value = value.sort_index(axis=1)
share.index = pd.to_datetime(share.index)
ratio.index = pd.to_datetime(ratio.index)
value.index = pd.to_datetime(value.index)
share.to_csv('/Users/caichaohong/Desktop/Zenki/南北向资金/share.csv')
ratio.to_csv('/Users/caichaohong/Desktop/Zenki/南北向资金/ratio.csv')
value.to_csv('/Users/caichaohong/Desktop/Zenki/南北向资金/value.csv')
def syncListTradeDay(sdate, edate):
listDate = jq.get_trade_days(start_date=sdate, end_date=edate, count=None)
#filePath= path.join(cfg.jqDataPath, 'trade_days.csv')
listDate = pd.Series(listDate)
#pd.Series(listDate).to_csv(filePath,header= 0, index= 0)
listDate = listDate.map(lambda x: x.strftime("%Y%m%d"))
return list(listDate)
def downloadMinuteBarByDate(start_date, end_date=datetime.today().date()):
jqdatasdk.auth(JQDATA_USER, JQDATA_PASSWORD)
"""下载所有配置中的合约的分钟线数据"""
print('-' * 50)
print(u'开始下载合约分钟线数据')
print('-' * 50)
trade_date_list = jqdatasdk.get_trade_days(start_date=start_date,
end_date=end_date)
i = 0
for trade_date in trade_date_list:
if i == 0:
i = 1
continue
symbols_df = jqdatasdk.get_all_securities(types=['futures'],
date=trade_date)
for index, row in symbols_df.iterrows():
downMinuteBarBySymbol(index, row, str(trade_date_list[i]),
str(trade_date_list[i - 1]))
i += 1
print('-' * 50)
print(u'合约分钟线数据下载完成')
print('-' * 50)
return
def dec(i):
Dec = '12-'+str(i.year)
start_date = start_rng.loc[Dec].index[0]
date = pd.to_datetime(jd.get_trade_days(start_date=str(i.year)+'-12-01', end_date=str(i.year)+'-12-30'))
d = date[date > start_date]
return d[0]
def jq_get_trade_days(start_date=None, end_date=None, count=None):
logger.info(
"HTTP GET: trade_days, with start_date={}, end_date={}, count={}".
format(start_date, end_date, count))
return get_trade_days(start_date=start_date,
end_date=end_date,
count=count)
def june(i):
June = '6-'+str(i.year)
start_date = start_rng.loc[June].index[0]
date = pd.to_datetime(jd.get_trade_days(start_date=str(i.year)+'-06-01', end_date=str(i.year)+'-06-30'))
d = date[date > start_date]
return d[0]
def init_finance_table(self, start_date=0):
self.authenticate()
self.stock = jdk.normalize_code(self.stock)
self.stock_info = jdk.get_security_info(self.stock)
if self.stock_info is None:
print('Error: Cannot get data or stock code not correct!')
'''Internal usage. To get finance table'''
q = jdk.query(jdk.valuation).filter(jdk.valuation.code == self.stock)
end_date = datetime.now().date()
if start_date == 0:
df = jdk.get_fundamentals_continuously(q, end_date=end_date, count=5000)
else:
# get the days between start date and end date
date_span = jdk.get_trade_days(start_date=start_date, end_date=end_date)
if len(date_span) == 0:
return
df = jdk.get_fundamentals_continuously(q, end_date=end_date, count=len(date_span))
if df is None:
return
self.finance_data = df.swapaxes('items', 'minor_axis')
self.finance_data = self.finance_data[self.stock]
del self.finance_data['day.1']
del self.finance_data['code.1']
del self.finance_data['id']
return
def update_financials(new_start_date, new_end_date, cir_mc, pe, ps):
# share 是持股数df，换成其他df也行，用来检测股票数量是否相等
future_trade_days = get_trade_days(start_date=pe.index[-1],
end_date=new_end_date)[1:] # 第一天重复
old_trade_days = get_trade_days(start_date=new_start_date,
end_date=pe.index[0])[:-1] # 最后一天重复
new_trade_days = list(future_trade_days) + list(old_trade_days)
if len(new_trade_days) > 0:
for date in new_trade_days:
cir_mc.loc[date] = np.nan
pe.loc[date] = np.nan
ps.loc[date] = np.nan
for date in tqdm(new_trade_days):
df = get_fundamentals(
query(valuation.code, valuation.circulating_market_cap,
valuation.pe_ratio, valuation.ps_ratio).filter(
valuation.code.in_(list(pe.columns))),
date=date)
cir_mc.loc[date][df['code']] = df['circulating_market_cap'].values
pe.loc[date][df['code']] = df['pe_ratio'].values
ps.loc[date][df['code']] = df['ps_ratio'].values
else:
print("No need to Update")
cir_mc.index = pd.to_datetime(cir_mc.index)
pe.index = pd.to_datetime(pe.index)
ps.index = pd.to_datetime(ps.index)
cir_mc = cir_mc.sort_index(axis=0) # 按index排序
pe = pe.sort_index(axis=0) # 按index排序
ps = ps.sort_index(axis=0) # 按index排序
cir_mc = cir_mc.sort_index(axis=1) # 按股票代码排序
pe = pe.sort_index(axis=1) # 按股票代码排序
ps = ps.sort_index(axis=1) # 按股票代码排序
cir_mc = cir_mc.dropna(how='all', axis=0)
pe = pe.dropna(how='all', axis=0)
ps = ps.dropna(how='all', axis=0)
cir_mc.to_csv(
'/Users/caichaohong/Desktop/Zenki/financials/circulating_market_cap.csv'
)
pe.to_csv('/Users/caichaohong/Desktop/Zenki/financials/pe_ratio.csv')
ps.to_csv('/Users/caichaohong/Desktop/Zenki/financials/ps_ratio.csv')
def record(self, entity, start, end, size, timestamps):
df = pd.DataFrame()
dates = get_trade_days(start_date=start)
df['timestamp'] = pd.to_datetime(dates)
df['id'] = [to_time_str(date) for date in dates]
df['entity_id'] = 'stock_sz_000001'
df_to_db(df=df, data_schema=self.data_schema, provider=self.provider, force_update=self.force_update)
def get_new_trade_days(self, df):
last = df.index[len(df) - 1]
dt = last
if isinstance(last, str):
dt = datetime.datetime.strptime(last, '%Y-%m-%d').date()
dt = dt + datetime.timedelta(days=1)
days = jqdatasdk.get_trade_days(start_date=dt)
return days
def insertInfo(self,start_date,end_date):
#
df = jq.get_trade_days(start_date=start_date, end_date=end_date) # include start_date,end_date
df = pd.DataFrame(df)
df = self.transform_2_jq_loc(df)
self.table.insert_many(json.loads(df.T.to_json()).values())
self.createIndex()
def getTradeDays(dateStr, num_prev):
validays = []
numpre = int(num_prev)
date = datetime.datetime.strptime(dateStr, '%Y-%m-%d')
tradays = jq.get_trade_days(end_date=date, count=numpre)
list_tradays = tradays.tolist() # datetime in list
for day in list_tradays:
validays.append(day.strftime("%Y-%m-%d"))
return validays
def update_market_cap(new_start_date, new_end_date, market_cap, close):
# share 是持股数df，换成其他df也行，用来检测股票数量是否相等
future_trade_days = get_trade_days(start_date=market_cap.index[-1],
end_date=new_end_date)[1:] # 第一天重复
old_trade_days = get_trade_days(
start_date=new_start_date, end_date=market_cap.index[0])[:-1] # 最后一天重复
new_trade_days = list(future_trade_days) + list(old_trade_days)
if len(new_trade_days) > 0:
for date in new_trade_days:
market_cap.loc[date] = np.nan
for date in tqdm(new_trade_days):
df = get_fundamentals(query(valuation.code,
valuation.market_cap).filter(
valuation.code.in_(
list(market_cap.columns))),
date=date)
market_cap.loc[date][df['code']] = df['market_cap'].values
else:
print("No need to Update")
market_cap.index = pd.to_datetime(market_cap.index)
# close 是持股数，用来检测股票数量是否相等, 新加入股票补齐
new_stocks = list(set(close.columns).difference(set(market_cap.columns)))
if len(new_stocks) > 0:
print('total number of new stocks = {}'.format(len(new_stocks)))
for s in new_stocks:
market_cap[s] = np.nan
for date in tqdm(list(market_cap.index)):
df = get_fundamentals(query(valuation.code,
valuation.market_cap).filter(
valuation.code.in_(new_stocks)),
date=datetime.date(date))
# get_fundamentals 必须是 date格式的日期
market_cap.loc[date][df['code']] = df['market_cap'].values
market_cap = market_cap.sort_index(axis=0) # 按index排序
market_cap = market_cap.sort_index(axis=1) # 按股票代码排序
market_cap = market_cap.dropna(how='all', axis=0)
market_cap.to_csv(
'/Users/caichaohong/Desktop/Zenki/financials/market_cap.csv')
def insertInfo(self,start_date,end_date):
period_trade_date = jq.get_trade_days(start_date=start_date, end_date=end_date) # include start_date,end_date
#period_trade_date = TradeDayTable.fetch_period_trade_days(self.database,start_date=start_date, end_date=end_date)
for date in period_trade_date:
print ("inserting all_security table,date：",date)
df = jq.get_all_securities(types=[], date=date)
df = self.transform_2_jq_loc(df,date)
self.table.insert_many(json.loads(df.T.to_json()).values())
self.createIndex()
def get_interval(start_year,end_year):
rng = get_rng(start_year=start_year,end_year=end_year)
ind = npr.randint(0,len(rng))
start_date = rng[ind].strftime('%Y-%m-%d')
#适当处理得到结束日期
end_date = jd.get_trade_days(end_date=rng[ind+1],count=2)[0]
return (start_date,end_date)
def get_trade_list(start, end):
"""
return:
trade_list: 时间区间内的交易日
month_end: 月末时间
month_start: 月初时间
"""
trade_list = jq.get_trade_days(start, end)
return trade_list
def get_trade_days(self):
"""
获取最近10天交易日
:return:
"""
if self.cache.get('trade_days') is None:
self.cache.set(
'trade_days',
jq.get_trade_days(end_date=datetime.datetime.now(), count=10),
21600)
return self.cache.get('trade_days')
def isOpenFromJqdata(jqDataAccount='13268108673',
jqDataPassword='******'):
jqdatasdk.auth(jqDataAccount, jqDataPassword)
array = jqdatasdk.get_trade_days(start_date='2018-11-01',
end_date='2019-12-31')
today = time.strftime('%Y-%m-%d', time.localtime(time.time()))
for date in array:
datestr = str(date)
if today == datestr:
return True
return False
def insertInfo(self,start_date,end_date):
#
period_trade_date = jq.get_trade_days(start_date=start_date, end_date=end_date) # include start_date,end_date
for date in period_trade_date:
print ("inserting unfq_price table,date：",date)
securities = jq.get_all_securities(types=[], date=date)
df = jq.get_price(security = list(securities.index),start_date=date, end_date=date, frequency='daily', fields=fields, skip_paused=False, fq='none', count=None, panel=False, fill_paused=False)
df = self.transform_2_jq_loc(df)
self.table.insert_many(json.loads(df.T.to_json()).values())
self.createIndex()
def get_data_by_date(self, index_code, date):
stock_codes = jqdatasdk.get_index_stocks(index_code)
trade_day = jqdatasdk.get_trade_days(end_date=date, count=1)[0]
q = jqdatasdk.query(jqdatasdk.valuation.code, jqdatasdk.valuation.day,
jqdatasdk.valuation.pe_ratio_lyr,
jqdatasdk.valuation.pe_ratio,
jqdatasdk.valuation.pb_ratio,
jqdatasdk.valuation.market_cap,
jqdatasdk.valuation.circulating_market_cap).filter(
jqdatasdk.valuation.code.in_(stock_codes))
df = jqdatasdk.get_fundamentals(q, date=trade_day)
return df
def get_index_data(self, index_code, trade_days):
open_date = jqdatasdk.get_security_info(index_code).start_date
if (open_date > trade_days[0]):
trade_days = jqdatasdk.get_trade_days(start_date=open_date)
df = pd.DataFrame()
for day in trade_days:
dic = self._calc_pe_pb(index_code, day)
if dic:
df = df.append(dic, ignore_index=True)
if not df.empty:
df.set_index('day', inplace=True)
return df
def get_trade_days(self, start_date=None, end_date=None, count=None):
data = None
if (start_date != None and count != None):
self.print_line(
"Error : trade day info can't work with start_date and count")
if (self.login):
data = jqs.get_trade_days(start_date=start_date,
end_date=end_date,
count=count)
self.log.log(
"Log : Get all trade days " + "from date {}".format(start_date)
if start_date != None else "" +
"to date {}".format(end_date) if end_date != None else "" +
"count {}".format(count) if count != None else "", self.name)
return data
def get_sw1_valuation(start_date=None, end_date=None):
#2014-02-22之后申万一级行业28个
code = jd.get_industries(name='sw_l1',date='2014-02-22').index.tolist()
days = jd.get_trade_days(start_date,end_date)
index = jd.finance.run_query(jd.query(jd.finance.SW1_DAILY_VALUATION).filter(
jd.finance.SW1_DAILY_VALUATION.date=='2014-02-22'
).limit(1)).columns.tolist()
data = pd.DataFrame(columns = index)
for day in days:
df=jd.finance.run_query(jd.query(jd.finance.SW1_DAILY_VALUATION).filter(
jd.finance.SW1_DAILY_VALUATION.code.in_(code),
jd.finance.SW1_DAILY_VALUATION.date==day
))
name1 = set(list(map(lambda x:x[:-1],jd.get_industries(name='sw_l1',date='2014-02-22').name.tolist())))
name2 = set(df.name.tolist())
if not name1-name2:
data = pd.concat([data, df], axis = 0, sort=False)
return data
def downloadAllMinuteBar():
jqdatasdk.auth(JQDATA_USER, JQDATA_PASSWORD)
"""下载所有配置中的合约的分钟线数据"""
print('-' * 50)
print(u'开始下载合约分钟线数据')
print('-' * 50)
today = datetime.today().date()
trade_date_list = jqdatasdk.get_trade_days(end_date=today, count=2)
symbols_df = jqdatasdk.get_all_securities(types=['futures'], date=today)
for index, row in symbols_df.iterrows():
downMinuteBarBySymbol(index, row, str(today), str(trade_date_list[-2]))
print('-' * 50)
print(u'合约分钟线数据下载完成')
print('-' * 50)
return
def downloadDonmainMinuteBar():
jqdatasdk.auth(JQDATA_USER, JQDATA_PASSWORD)
"""下载所有配置中的合约的分钟线数据"""
print('-' * 50)
print(u'开始下载合约分钟线数据')
print('-' * 50)
today = datetime.today().date()
trade_date_list = jqdatasdk.get_trade_days(end_date=today, count=3)
#symbols_df = jqdatasdk.get_all_securities(types=['futures'], date=today)
symbols_df = jqdatasdk.get_dominant_future('AG', '2018-09-30') #jinsong
downMinuteBarBySymbol("RB9999.XSGE", {'name': 'RB9999.XSGE'.split('.')[0]},
str(today), str(trade_date_list[-2]))
print('-' * 50)
print(u'合约分钟线数据下载完成')
print('-' * 50)
return
def hist_data(code, start_date=None, end_date=None):
jqdatasdk.auth("13695683829", "ssk741212")
# 参数检查
today_str = (datetime.now().date() - timedelta(1)).strftime('%Y-%m-%d')
if start_date is None or start_date < _start_date:
start_date = _start_date
if end_date is None or end_date < _start_date or end_date > today_str:
end_date = today_str
# 交易日
trade_days = get_trade_days(start_date=start_date, end_date=end_date)
col = _codes[code].get('position')
name = _codes[code].get('name')
def data_list():
for day in trade_days:
datas = []
for page in [2, 3, 4]:
# 链接
url = '%stype=zy%s&date=%s' % (_url_base, page, day)
# 页面请求
content = requests.get(url).text
soup = BeautifulSoup(content, 'html.parser')
data = soup.select('tbody[class="tc"] td')
if len(data) == 0:
break
datas.append(data[col].get_text())
print
'\r数据更新：板块%s %s %s ' % (code, name, day),
yield [day] + datas
# 组织数据（迭代数据生成器）
df = pd.DataFrame(data=[item for item in data_list()],
columns=['date', 'pe', 'pb', 'dyr'])
df.set_index('date', inplace=True)
df.index.name = None
df.index = pd.to_datetime(df.index, format='%Y-%m-%d')
return df.astype('float')
zz800_industry = get_industry(zz800, date=datetime.strftime(datetime.today(), '%Y-%m-%d'))
zz800_industry = pd.DataFrame(zz800_industry).T[[industry_type]]
zz800_industry[industry_type] = zz800_industry[industry_type].apply(lambda x: x['industry_name'])
industries = zz800_industry[industry_type]
industries = list(set(industries))
industries.sort()
# 创建表格 列是日期，行是行业，最后需要转置
market_breadth = pd.DataFrame(index=(['zz800'] + industries))
# 根据交易日来计算宽度
# 最近80天交易日
# 然后倒序[::-1]
# 最后取出80天[:80]
trade_days = get_trade_days(start_date='2015-01-01', end_date=None)[::-1][:market_breadth_days]
for day in tqdm(trade_days):
# BIAS:(CLOSE-MA(CLOSE,N))/MA(CLOSE,N)*100
# 负的说明 当前收盘价在MAN日线之下
BIAS1, BIAS2, BIAS3 = BIAS(zz800, check_date=day, N1=20, N2=60, N3=120) # 乖离率
zz800_industry['BIAS20'] = pd.Series(BIAS1)
day_market_breadth1 = pd.DataFrame()
# 统计zz800中站在20日线上的有多少占比
day_market_breadth1.loc['zz800', 'BIAS20'] = round(
(len(zz800_industry[zz800_industry['BIAS20'] > 0]) / len(zz800_industry)) * 100)
# 统计每个行业有多少只股票站在20日线上
