def get_a_share_list(self):
database_name = DatabaseName.A_SHARE_KLINE_DAILY.value
a_share_list = self.connect.get_list_collection_names(database_name)
a_share_list = [
i for i in a_share_list if is_security_type(i, 'EXTRA_STOCK_A')
]
self.connect.disconnect()
return a_share_list
def insert_security_code(self, market, file_name, path):
with MongoConnect(self.database):
print(path + file_name + '\n')
kline_daily_data = pd.read_csv(path + file_name, encoding='unicode_escape')
security_code = file_name.split('.')[0] + '.' + market
if is_security_type(security_code, 'EXTRA_STOCK_A'):
kline_daily_data = kline_daily_data.reindex(columns=['date', 'open', 'high', 'low', 'close', 'volumw',
'turover', 'match_items', 'interest'])
kline_daily_data.rename(columns={'volumw': 'volume', 'turover': 'amount'}, inplace=True)
kline_daily_data = kline_daily_data[kline_daily_data.date >= 20020104]
with switch_collection(Kline, security_code) as KlineDaily_security_code:
doc_list = []
security_code_data = pd.DataFrame()
if security_code in self.data_dict.keys():
security_code_data = self.data_dict[security_code].set_index(["TRADE_DT"])
security_code_data = security_code_data.fillna(0)
for index, row in kline_daily_data.iterrows():
date_int = int(row['date'])
if not np.isnan(date_int):
try:
pre_close = int(10000 * security_code_data.loc[date_int, 'S_DQ_PRECLOSE'])
except KeyError:
pre_close = None
date_int = str(date_int)
time_tag = datetime.strptime(date_int, "%Y%m%d")
doc = KlineDaily_security_code(time_tag=time_tag, pre_close=pre_close,
open=int(row['open']), high=int(row['high']),
low=int(row['low']), close=int(row['close']),
volume=int(row['volume']), amount=int(row['amount']),
match_items=int(row['match_items']), interest=int(row['interest']))
doc_list.append(doc)
# 用csv全表补充20020104之前的日线数据，match_items为0
security_code_data = security_code_data[security_code_data.index < 20020104]
for index, row in security_code_data.iterrows():
if row['S_DQ_AMOUNT'] > 0:
date_int = int(index)
date_int = str(date_int)
time_tag = datetime.strptime(date_int, "%Y%m%d")
try:
pre_close = int(row['S_DQ_PRECLOSE'] * 10000)
except KeyError:
pre_close = None
doc = KlineDaily_security_code(time_tag=time_tag, pre_close=pre_close,
open=int(row['S_DQ_OPEN'] * 10000),
high=int(row['S_DQ_HIGH'] * 10000),
low=int(row['S_DQ_LOW'] * 10000),
close=int(row['S_DQ_CLOSE'] * 10000),
volume=int(row['S_DQ_VOLUME'] * 100),
amount=int(row['S_DQ_AMOUNT'] * 1000),
match_items=0, interest=0)
doc_list.append(doc)
KlineDaily_security_code.objects.insert(doc_list)
def update_all_market_data(self, end=datetime.now()):
get_collection_list = GetCollectionList()
a_share_list = get_collection_list.get_a_share_list()
a_share_list = [i for i in a_share_list if is_security_type(i, 'EXTRA_STOCK_A')]
all_market_data = self.get_all_market_data(security_list=a_share_list, end=end)
folder_name = LocalDataFolderName.MARKET_DATA.value
sub_folder_name = LocalDataFolderName.KLINE_DAILY.value
sub_sub_folder_name = LocalDataFolderName.A_SHARE.value
for field in self.field:
if field not in ['time_tag', 'interest']:
path = LocalDataPath.path + folder_name + '/' + sub_folder_name + '/' + sub_sub_folder_name + '/'
data_name = field
save_data_to_hdf5(path, data_name, pd.DataFrame(all_market_data[field]))