def persist_index(self, df) -> None:
df['timestamp'] = df['timestamp'].apply(lambda x: to_pd_timestamp(x))
df['list_date'] = df['list_date'].apply(lambda x: to_pd_timestamp(x))
df['id'] = df['code'].apply(lambda code: f'index_cn_{code}')
df['entity_id'] = df['id']
df['exchange'] = 'cn'
df['entity_type'] = 'index'
df['is_delisted'] = False
df = df.dropna(axis=0, how='any')
df = df.drop_duplicates(subset='id', keep='last')
persist_entities(df, entity_type='index', provider=self.provider)
def run(self):
for exchange_str in self.exchanges:
exchange = CCXTAccount.get_ccxt_exchange(exchange_str)
try:
markets = exchange.fetch_markets()
df = pd.DataFrame()
# markets有些为key=symbol的dict,有些为list
markets_type = type(markets)
if markets_type != dict and markets_type != list:
self.logger.exception(
"unknown return markets type {}".format(markets_type))
return
aa = []
for market in markets:
if markets_type == dict:
name = market
code = market
if markets_type == list:
code = market['symbol']
name = market['symbol']
if name not in COIN_PAIRS:
continue
aa.append(market)
security_item = {
'id': '{}_{}_{}'.format('coin', exchange_str, code),
'entity_id': '{}_{}_{}'.format('coin', exchange_str,
code),
'exchange': exchange_str,
'entity_type': 'coin',
'code': code,
'name': name
}
df = df.append(security_item, ignore_index=True)
# 存储该交易所的数字货币列表
if not df.empty:
persist_entities(df=df,
entity_type='coin',
provider=self.provider)
self.logger.info(
"init_markets for {} success".format(exchange_str))
except Exception as e:
self.logger.exception(
"init_markets for {} failed".format(exchange_str), e)
def persist_etf_list(self, df: pd.DataFrame, exchange: str):
if df is None:
return
df = df.copy()
if exchange == 'sh':
df = df[['FUND_ID', 'FUND_NAME']]
elif exchange == 'sz':
df = df[['证券代码', '证券简称']]
df.columns = ['code', 'name']
df['id'] = df['code'].apply(lambda code: f'index_{exchange}_{code}')
df['entity_id'] = df['id']
df['exchange'] = exchange
df['entity_type'] = 'index'
df['category'] = StockCategory.etf.value
df = df.dropna(axis=0, how='any')
df = df.drop_duplicates(subset='id', keep='last')
persist_entities(df, entity_type='index', provider=self.provider)
def download_stock_list(self, response, exchange):
df = None
if exchange == 'sh':
df = pd.read_csv(io.BytesIO(response.content),
sep='\s+',
encoding='GB2312',
dtype=str,
parse_dates=['上市日期'])
if df is not None:
df = df.loc[:, ['公司代码', '公司简称', '上市日期']]
elif exchange == 'sz':
df = pd.read_excel(io.BytesIO(response.content),
sheet_name='A股列表',
dtype=str,
parse_dates=['A股上市日期'])
if df is not None:
df = df.loc[:, ['A股代码', 'A股简称', 'A股上市日期']]
if df is not None:
df.columns = ['code', 'name', 'list_date']
df = df.dropna(subset=['code'])
# handle the dirty data
# 600996,贵广网络,2016-12-26,2016-12-26,sh,stock,stock_sh_600996,,次新股,贵州,,
df.loc[df['code'] == '600996', 'list_date'] = '2016-12-26'
print(df[df['list_date'] == '-'])
df['list_date'] = df['list_date'].apply(
lambda x: to_pd_timestamp(x))
df['exchange'] = exchange
df['entity_type'] = 'stock'
df['id'] = df[['entity_type', 'exchange',
'code']].apply(lambda x: '_'.join(x.astype(str)),
axis=1)
df['entity_id'] = df['id']
df['timestamp'] = df['list_date']
df = df.dropna(axis=0, how='any')
df = df.drop_duplicates(subset=('id'), keep='last')
persist_entities(df, provider=self.provider)
def init_main_index(provider='exchange'):
for item in CHINA_STOCK_MAIN_INDEX:
item['timestamp'] = to_pd_timestamp(item['timestamp'])
df = pd.DataFrame(CHINA_STOCK_MAIN_INDEX)
# print(df)
persist_entities(df, entity_type='index', provider=provider)