def get_cs_index_stock(code, timestamp, name=None):
entity_type = 'index'
exchange = 'sh'
entity_id = f'{entity_type}_{exchange}_{code}'
response = requests.get(url.format(code), headers=DEFAULT_HEADER)
response.raise_for_status()
df = pd.read_excel(io.BytesIO(response.content))
df = df[['日期Date', '成分券代码Constituent Code', '成分券名称Constituent Name']].rename(
columns={'日期Date': 'timestamp', '成分券代码Constituent Code': 'stock_code',
'成分券名称Constituent Name': 'stock_name'})
df['entity_id'] = entity_id
df['entity_type'] = 'index'
df['exchange'] = 'sh'
df['code'] = code
df['name'] = name
df['stock_id'] = df['stock_code'].apply(lambda x: china_stock_code_to_id(str(x)))
# id format: {entity_id}_{timestamp}_{stock_id}
df['id'] = df[['entity_id', 'timestamp', 'stock_id']].apply(lambda x: '_'.join(x.astype(str)), axis=1)
df['timestamp'] = pd.to_datetime(df['timestamp'])
return df
def get_cs_index_stock(code, timestamp, name=None):
entity_type = "index"
exchange = "sh"
entity_id = f"{entity_type}_{exchange}_{code}"
response = requests.get(url.format(code), headers=DEFAULT_HEADER)
response.raise_for_status()
df = pd.read_excel(io.BytesIO(response.content))
df = df[["日期Date", "成分券代码Constituent Code",
"成分券名称Constituent Name"]].rename(
columns={
"日期Date": "timestamp",
"成分券代码Constituent Code": "stock_code",
"成分券名称Constituent Name": "stock_name"
})
df["entity_id"] = entity_id
df["entity_type"] = "index"
df["exchange"] = "sh"
df["code"] = code
df["name"] = name
df["stock_id"] = df["stock_code"].apply(
lambda x: china_stock_code_to_id(str(x)))
# id format: {entity_id}_{timestamp}_{stock_id}
df["id"] = df[["entity_id", "timestamp",
"stock_id"]].apply(lambda x: "_".join(x.astype(str)),
axis=1)
df["timestamp"] = pd.to_datetime(df["timestamp"])
return df
def get_trade_dates(self, conn, start, end, order="ESC"):
"""
获取start到end(包括start、end)之间的交易日期列表
start所表示的日期在end之前
:param end:
:param order:
:return:
"""
pass
__all__ = ['TlChinaStockKdataRecorder']
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--level', help='trading level', default='1d', choices=[item.value for item in IntervalLevel])
parser.add_argument('--codes', help='codes', default=['600118'], nargs='+')
args = parser.parse_args()
level = IntervalLevel(args.level)
codes = args.codes
init_log('tl_china_stock_{}_kdata.log'.format(args.level))
TlChinaStockKdataRecorder(level=level, sleeping_time=0, codes=codes, real_time=False,
adjust_type=AdjustType.hfq).run()
print(get_kdata(entity_id=china_stock_code_to_id(args.codes[0]), limit=10, order=Stock1dHfqKdata.timestamp.desc(),adjust_type=AdjustType.hfq))
def record(self, entity, start, end, size, timestamps):
# 忽略退市的
if entity.end_date:
return None
redundant_times = 1
while redundant_times > 0:
df = run_query(
table='finance.FUND_PORTFOLIO_STOCK',
conditions=
f'pub_date#>=#{to_time_str(start)}&code#=#{entity.code}',
parse_dates=None)
df = df.dropna()
if pd_is_not_null(df):
# data format
# id code period_start period_end pub_date report_type_id report_type rank symbol name shares market_cap proportion
# 0 8640569 159919 2018-07-01 2018-09-30 2018-10-26 403003 第三季度 1 601318 中国平安 19869239.0 1.361043e+09 7.09
# 1 8640570 159919 2018-07-01 2018-09-30 2018-10-26 403003 第三季度 2 600519 贵州茅台 921670.0 6.728191e+08 3.50
# 2 8640571 159919 2018-07-01 2018-09-30 2018-10-26 403003 第三季度 3 600036 招商银行 18918815.0 5.806184e+08 3.02
# 3 8640572 159919 2018-07-01 2018-09-30 2018-10-26 403003 第三季度 4 601166 兴业银行 22862332.0 3.646542e+08 1.90
df['timestamp'] = pd.to_datetime(df['pub_date'])
df.rename(columns={
'symbol': 'stock_code',
'name': 'stock_name'
},
inplace=True)
df['proportion'] = df['proportion'] * 0.01
df = portfolio_relate_stock(df, entity)
df['stock_id'] = df['stock_code'].apply(
lambda x: china_stock_code_to_id(x))
df['id'] = df[['entity_id', 'stock_id', 'pub_date',
'id']].apply(lambda x: '_'.join(x.astype(str)),
axis=1)
df['report_date'] = pd.to_datetime(df['period_end'])
df['report_period'] = df['report_type'].apply(
lambda x: jq_to_report_period(x))
saved = df_to_db(df=df,
data_schema=self.data_schema,
provider=self.provider,
force_update=self.force_update)
# 取不到非重复的数据
if saved == 0:
return None
# self.logger.info(df.tail())
self.logger.info(
f"persist fund {entity.code}({entity.name}) portfolio success {df.iloc[-1]['pub_date']}"
)
latest = df['timestamp'].max()
# 取到了最近两年的数据，再请求一次,确保取完最新的数据
if latest.year >= now_pd_timestamp().year - 1:
redundant_times = redundant_times - 1
start = latest
else:
return None
return None