def save_a_share_cash_flow(self): database = 'stock_base_data' with MongoConnect(database): doc_list = [] for index, row in self.data_df.iterrows(): row_dict = dict(row) row_dict['security_code'] = row_dict['S_INFO_WINDCODE'] row_dict.pop('WIND_CODE') row_dict.pop('OBJECT_ID') row_dict.pop('S_INFO_WINDCODE') doc = AShareCashFlow() for key, value in row_dict.items(): if key.lower() in self.field_is_str_list: if key.lower() in ['ann_dt', 'report_period', 'statement_type', 'actual_ann_dt']: if np.isnan(value): setattr(doc, key.lower(), -1) else: setattr(doc, key.lower(), int(value)) else: setattr(doc, key.lower(), str(value)) else: setattr(doc, key.lower(), value) doc_list.append(doc) if len(doc_list) > 999: AShareCashFlow.objects.insert(doc_list) doc_list = [] else: AShareCashFlow.objects.insert(doc_list)
def save_a_share_ex_right_dividend(self): database = 'stock_base_data' with MongoConnect(database): doc_list = [] for index, row in self.data_df.iterrows(): row_dict = dict(row) row_dict['security_code'] = row_dict['S_INFO_WINDCODE'] row_dict.pop('OBJECT_ID') row_dict.pop('S_INFO_WINDCODE') doc = AShareExRightDividend() for key, value in row_dict.items(): if key.lower() in self.collection_property_list: property_name = AShareExRightDividend.__dict__[key.lower()] if isinstance(property_name, StringField): setattr(doc, key.lower(), str(value)) elif isinstance(property_name, DateTimeField): if np.isnan(value): setattr(doc, key.lower(), None) else: setattr(doc, key.lower(), datetime.strptime(str(int(value)), "%Y%m%d")) else: setattr(doc, key.lower(), value) doc_list.append(doc) if len(doc_list) > 999: AShareExRightDividend.objects.insert(doc_list) doc_list = [] else: AShareExRightDividend.objects.insert(doc_list)
def save_a_share_profit_notice(self): database = 'stock_base_data' with MongoConnect(database): doc_list = [] for index, row in self.data_df.iterrows(): row_dict = dict(row) row_dict['security_code'] = row_dict['S_INFO_WINDCODE'] row_dict.pop('OBJECT_ID') row_dict.pop('S_INFO_WINDCODE') doc = AShareProfitNotice() for key, value in row_dict.items(): if key.lower() in self.field_is_str_list: if key.lower() in ['s_profitnotice_date', 's_profitnotice_period', 's_profitnotice_firstanndate']: if np.isnan(value): setattr(doc, key.lower(), -1) else: setattr(doc, key.lower(), int(value)) else: setattr(doc, key.lower(), str(value)) else: setattr(doc, key.lower(), value) doc_list.append(doc) if len(doc_list) > 999: AShareProfitNotice.objects.insert(doc_list) doc_list = [] else: AShareProfitNotice.objects.insert(doc_list)
def save_a_share_adj_factor(self): """ 取当日收盘价,作为转、送的股价, 再计算复权因子更新到AShareExRightDividend, 复权因子adj_factor :return: """ database = 'stock_base_data' with MongoConnect(database): pass
def insert_security_code_list(self): stock_code_list = [] for market in self.market_list: path = self.data_path + market + '/MultDate/' file_list = os.listdir(path) stock_code_list += [ i.split('.')[0] + '.' + market for i in file_list ] file_num = 0 p = Pool(8) for file_name in file_list: file_num += 1 print('完成数量:', file_num) p.apply_async(self.insert_security_code, args=(market, file_name, path)) p.close() p.join() delist = list( set(self.data_dict.keys()).difference(set(stock_code_list))) with MongoConnect(self.database): for security_code in delist: with switch_collection( Kline, security_code) as KlineDaily_security_code: doc_list = [] security_code_data = self.data_dict[ security_code].set_index(["TRADE_DT"]) for index, row in security_code_data.iterrows(): if row['S_DQ_AMOUNT'] > 0: date_int = int(index) date_int = str(date_int) time_tag = datetime.strptime(date_int, "%Y%m%d") try: pre_close = int(row['S_DQ_PRECLOSE'] * 10000) except KeyError: pre_close = None doc = KlineDaily_security_code( time_tag=time_tag, pre_close=pre_close, open=int(row['S_DQ_OPEN'] * 10000), high=int(row['S_DQ_HIGH'] * 10000), low=int(row['S_DQ_LOW'] * 10000), close=int(row['S_DQ_CLOSE'] * 10000), volume=int(row['S_DQ_VOLUME'] * 100), amount=int(row['S_DQ_AMOUNT'] * 1000), match_items=0, interest=0) doc_list.append(doc) KlineDaily_security_code.objects.insert(doc_list)
def save_a_share_calendar(self): database = DatabaseName.STOCK_BASE_DATA.value with MongoConnect(database): doc_list = [] data_grouped = self.data_df.groupby("S_INFO_EXCHMARKET") data_dict = {i[0]: list(i[1]['TRADE_DAYS']) for i in data_grouped} for market, trade_days in data_dict.items(): if market == 'SSE': market = 'SH' elif market == 'SZSE': market = 'SZ' trade_days = [date_to_datetime(str(i)) for i in sorted(trade_days)] doc = AShareCalendar(market=market, trade_days=trade_days) doc_list.append(doc) AShareCalendar.objects.insert(doc_list)
def insert_security_code(self, market, file_name, path): database = 'a_share_kline_daily' with MongoConnect(database): print(path + file_name + '\n') kline_daily_data = pd.read_csv(path + file_name, encoding='unicode_escape') security_code = file_name.split('.')[0] + '.' + market kline_daily_data = kline_daily_data.reindex(columns=[ 'date', 'open', 'high', 'low', 'close', 'volumw', 'turover', 'match_items', 'interest' ]) kline_daily_data.rename(columns={ 'volumw': 'volume', 'turover': 'amount' }, inplace=True) with switch_collection(Kline, security_code) as KlineDaily_security_code: doc_list = [] security_code_data = pd.DataFrame() if security_code in self.data_dict.keys(): security_code_data = self.data_dict[ security_code].set_index(["TRADE_DT"]) for index, row in kline_daily_data.iterrows(): date_int = int(row['date']) if not np.isnan(date_int): try: pre_close = int( 10000 * security_code_data.loc[date_int, 'S_DQ_PRECLOSE']) except KeyError: pre_close = None date_int = str(date_int) time_tag = datetime.strptime(date_int, "%Y%m%d") doc = KlineDaily_security_code( time_tag=time_tag, pre_close=pre_close, open=int(row['open']), high=int(row['high']), low=int(row['low']), close=int(row['close']), volume=int(row['volume']), amount=int(row['amount']), match_items=int(row['match_items']), interest=int(row['interest'])) doc_list.append(doc) KlineDaily_security_code.objects.insert(doc_list)
def save_a_share_index_members(self): database = 'stock_base_data' with MongoConnect(database): doc_list = [] for index, row in self.data_df.iterrows(): row_dict = dict(row) row_dict['index_code'] = row_dict['S_INFO_WINDCODE'] row_dict['security_code'] = row_dict['S_CON_WINDCODE'] row_dict['in_date'] = row_dict['S_CON_INDATE'] row_dict['out_date'] = row_dict['S_CON_OUTDATE'] row_dict['current_sign'] = row_dict['CUR_SIGN'] row_dict.pop('OBJECT_ID') row_dict.pop('S_INFO_WINDCODE') row_dict.pop('S_CON_WINDCODE') row_dict.pop('S_CON_INDATE') row_dict.pop('S_CON_OUTDATE') row_dict.pop('CUR_SIGN') self.field_is_str_list = self.field_is_str_list + [ 'in_date', 'out_date' ] doc = AShareIndexMembers() for key, value in row_dict.items(): if key.lower() in self.collection_property_list: property_name = AShareIndexMembers.__dict__[ key.lower()] if isinstance(property_name, StringField): setattr(doc, key.lower(), str(value)) elif isinstance(property_name, DateTimeField): if np.isnan(value): setattr(doc, key.lower(), None) else: setattr( doc, key.lower(), datetime.strptime(str(int(value)), "%Y%m%d")) else: setattr(doc, key.lower(), value) doc_list.append(doc) if len(doc_list) > 999: AShareIndexMembers.objects.insert(doc_list) doc_list = [] else: AShareIndexMembers.objects.insert(doc_list)
def save_a_index_members(self): database = 'stock_base_data' with MongoConnect(database): doc_list = [] for index, row in self.data_df.iterrows(): row_dict = dict(row) row_dict['index_code'] = row_dict['S_INFO_WINDCODE'] row_dict['security_code'] = row_dict['S_CON_WINDCODE'] row_dict['in_date'] = row_dict['S_CON_INDATE'] row_dict['out_date'] = row_dict['S_CON_OUTDATE'] row_dict['current_sign'] = row_dict['CUR_SIGN'] row_dict.pop('OBJECT_ID') row_dict.pop('S_INFO_WINDCODE') row_dict.pop('S_CON_WINDCODE') row_dict.pop('S_CON_INDATE') row_dict.pop('S_CON_OUTDATE') row_dict.pop('CUR_SIGN') doc = AIndexMembers() for key, value in row_dict.items(): if key.lower() in self.field_is_str_list: if key.lower() in [ 's_con_indate', 's_con_outdate', 'current_sign' ]: if np.isnan(value): setattr(doc, key.lower(), -1) else: setattr(doc, key.lower(), int(value)) else: setattr(doc, key.lower(), str(value)) else: setattr(doc, key.lower(), value) doc_list.append(doc) if len(doc_list) > 999: AIndexMembers.objects.insert(doc_list) doc_list = [] else: AIndexMembers.objects.insert(doc_list)
def insert_security_code(self, market, file_name, path): database = 'kline' with MongoConnect(database): print(path + file_name + '\n') kline_daily_data = pd.read_csv(path + file_name, encoding='unicode_escape') date = kline_daily_data['date'][0] if not np.isnan(date): security_code = file_name.split('.')[0] + '.' + market kline_daily_data = kline_daily_data.reindex(columns=[ 'open', 'high', 'low', 'close', 'volumw', 'turover', 'match_items', 'interest' ]) kline_daily_data.rename(columns={ 'volumw': 'volume', 'turover': 'amount' }, inplace=True) with switch_collection( KlineDaily, security_code) as KlineDaily_security_code: doc_list = [] for index, row in kline_daily_data.iterrows(): data = [int(value) for value in dict(row).values()] data = [ int(row['open']), int(row['high']), int(row['low']), int(row['close']), int(row['volume']), int(row['amount']), int(row['match_items']), int(row['interest']), ] doc = KlineDaily_security_code( security_code=security_code, date=int(date), data=data) doc_list.append(doc) KlineDaily_security_code.objects.insert(doc_list)
def insert_security_code(self, market, file_name, path): database = DatabaseName.INDEX_KLINE_DAILY.value with MongoConnect(database): print(path + file_name + '\n') kline_daily_data = pd.read_csv(path + file_name, encoding='unicode_escape') code = file_name.split('.')[0] code_transfer_dict = { '999999': '000001', '999998': '000002', '999997': '000003', '999996': '000004', '999995': '000005', '999994': '000006', '999993': '000007', '999992': '000008', '999991': '000010', '999990': '000011', '999989': '000012', '999988': '000013', '999987': '000016', '999986': '000015', '000300': '000300' } if market == 'SH': if code in code_transfer_dict.keys(): code = code_transfer_dict[code] else: code = '00' + code[2:] security_code = code + '.' + market kline_daily_data = kline_daily_data.reindex(columns=[ 'date', 'open', 'high', 'low', 'close', 'volumw', 'turover', 'match_items', 'interest' ]) kline_daily_data.rename(columns={ 'volumw': 'volume', 'turover': 'amount' }, inplace=True) with switch_collection(Kline, security_code) as KlineDaily_security_code: doc_list = [] for index, row in kline_daily_data.iterrows(): date_int = int(row['date']) if not np.isnan(date_int): date_int = str(date_int) time_tag = datetime.strptime(date_int, "%Y%m%d") doc = KlineDaily_security_code( time_tag=time_tag, pre_close=None, open=int(row['open']), high=int(row['high']), low=int(row['low']), close=int(row['close']), volume=int(row['volume']), amount=int(row['amount']), match_items=None, interest=None) doc_list.append(doc) KlineDaily_security_code.objects.insert(doc_list)
# -*- coding: utf-8 -*- # ------------------------------ # @Time : 2019/11/21 # @Author : gao # @File : update_finance_data.py # @Project : AmazingQuant # ------------------------------ from AmazingQuant.data_center.mongo_connection import MongoConnect from AmazingQuant.data_center.database_field.field_a_share_finance_data import AShareIncome, AShareCashFlow from AmazingQuant.utils.performance_test import Timer if __name__ == '__main__': database = 'stock_base_data' with MongoConnect(database): with Timer(True): security_code_list = AShareIncome.objects.distinct('security_code') data = AShareIncome.objects(security_code__in=security_code_list, statement_type=408009000) for i in data: print(i.security_code)
def insert_security_code(self, market, file_name, path): with MongoConnect(self.database): print(path + file_name + '\n') kline_daily_data = pd.read_csv(path + file_name, encoding='unicode_escape') security_code = file_name.split('.')[0] + '.' + market if is_security_type(security_code, 'EXTRA_STOCK_A'): kline_daily_data = kline_daily_data.reindex(columns=[ 'date', 'open', 'high', 'low', 'close', 'volumw', 'turover', 'match_items', 'interest' ]) kline_daily_data.rename(columns={ 'volumw': 'volume', 'turover': 'amount' }, inplace=True) kline_daily_data = kline_daily_data[ kline_daily_data.date >= 20020104] with switch_collection( Kline, security_code) as KlineDaily_security_code: doc_list = [] security_code_data = pd.DataFrame() if security_code in self.data_dict.keys(): security_code_data = self.data_dict[ security_code].set_index(["TRADE_DT"]) security_code_data = security_code_data.fillna(0) for index, row in kline_daily_data.iterrows(): date_int = int(row['date']) if not np.isnan(date_int): try: pre_close = int( 10000 * security_code_data.loc[date_int, 'S_DQ_PRECLOSE']) except KeyError: pre_close = None date_int = str(date_int) time_tag = datetime.strptime(date_int, "%Y%m%d") doc = KlineDaily_security_code( time_tag=time_tag, pre_close=pre_close, open=int(row['open']), high=int(row['high']), low=int(row['low']), close=int(row['close']), volume=int(row['volume']), amount=int(row['amount']), match_items=int(row['match_items']), interest=int(row['interest'])) doc_list.append(doc) # 用csv全表补充20020104之前的日线数据,match_items为0 security_code_data = security_code_data[ security_code_data.index < 20020104] for index, row in security_code_data.iterrows(): if row['S_DQ_AMOUNT'] > 0: date_int = int(index) date_int = str(date_int) time_tag = datetime.strptime(date_int, "%Y%m%d") try: pre_close = int(row['S_DQ_PRECLOSE'] * 10000) except KeyError: pre_close = None doc = KlineDaily_security_code( time_tag=time_tag, pre_close=pre_close, open=int(row['S_DQ_OPEN'] * 10000), high=int(row['S_DQ_HIGH'] * 10000), low=int(row['S_DQ_LOW'] * 10000), close=int(row['S_DQ_CLOSE'] * 10000), volume=int(row['S_DQ_VOLUME'] * 100), amount=int(row['S_DQ_AMOUNT'] * 1000), match_items=0, interest=0) doc_list.append(doc) KlineDaily_security_code.objects.insert(doc_list)