def update_index_class(self, industry_class_name, industry_class_dict):
with MongoConnect(self.database):
index_members_data = AShareIndexMembers.objects(
index_code__in=industry_class_dict.keys()).as_pymongo()
field_list = ['index_code', 'security_code', 'in_date', 'out_date']
self.index_members_df = pd.DataFrame(
list(index_members_data)).reindex(columns=field_list)
self.index_members_df = self.index_members_df.fillna(
datetime.now()).reset_index(drop=True)
get_collection_list = GetCollectionList()
a_share_list = get_collection_list.get_a_share_list()
calendar_obj = GetCalendar()
calendar_SH = calendar_obj.get_calendar('SH')
self.index_class = pd.DataFrame(columns=a_share_list,
index=calendar_SH)
def industry_history(x, index_members_df):
industry_in_out_date = index_members_df[
index_members_df.security_code == x.name]
for index, row in industry_in_out_date.iterrows():
x[row['in_date']:row['out_date']] = row['index_code']
return x
self.index_class = self.index_class.apply(
industry_history, args=(self.index_members_df, ), axis=0)
self.index_class = self.index_class.fillna(method='pad').fillna(
method='backfill')
folder_name = LocalDataFolderName.INDUSTRY_CLASS.value
path = LocalDataPath.path + folder_name + '/'
data_name = industry_class_name
save_data_to_hdf5(path, data_name, self.index_class)
def update_index_data(self, end=datetime.now()):
"""
:param end:
:return:
"""
get_collection_list = GetCollectionList()
index_list = get_collection_list.get_index_list()
self.end = end
database = DatabaseName.INDEX_KLINE_DAILY.value
with MongoConnect(database):
index_data_dict = {}
for index_code in index_list:
with switch_collection(Kline, index_code) as KlineDaily_index_code:
security_code_data = KlineDaily_index_code.objects(time_tag__lte=self.end).as_pymongo()
security_code_data_df = pd.DataFrame(list(security_code_data)).reindex(columns=self.field)
security_code_data_df.set_index(["time_tag"], inplace=True)
index_data_dict[index_code] = security_code_data_df
field_data_dict = {}
for i in self.field:
if i != 'time_tag':
field_data_pd = pd.DataFrame({key: value[i] for key, value in index_data_dict.items()})
# 原始数据的开高低收除以10000
if i in ['open', 'high', 'low', 'close']:
field_data_dict[i] = field_data_pd.div(10000)
else:
field_data_dict[i] = field_data_pd
folder_name = LocalDataFolderName.MARKET_DATA.value
sub_folder_name = LocalDataFolderName.KLINE_DAILY.value
sub_sub_folder_name = LocalDataFolderName.INDEX.value
for field in self.field:
if field not in ['time_tag', 'interest']:
path = LocalDataPath.path + folder_name + '/' + sub_folder_name + '/' + sub_sub_folder_name + '/'
data_name = field
save_data_to_hdf5(path, data_name, pd.DataFrame(field_data_dict[field]))
def update_all_market_data(self, end=datetime.now()):
get_collection_list = GetCollectionList()
a_share_list = get_collection_list.get_a_share_list()
a_share_list = [i for i in a_share_list if is_security_type(i, 'EXTRA_STOCK_A')]
all_market_data = self.get_all_market_data(security_list=a_share_list, end=end)
folder_name = LocalDataFolderName.MARKET_DATA.value
sub_folder_name = LocalDataFolderName.KLINE_DAILY.value
sub_sub_folder_name = LocalDataFolderName.A_SHARE.value
for field in self.field:
if field not in ['time_tag', 'interest']:
path = LocalDataPath.path + folder_name + '/' + sub_folder_name + '/' + sub_sub_folder_name + '/'
data_name = field
save_data_to_hdf5(path, data_name, pd.DataFrame(all_market_data[field]))