def save_a_share_cash_flow(self):
database = 'stock_base_data'
with MongoConnect(database):
doc_list = []
for index, row in self.data_df.iterrows():
row_dict = dict(row)
row_dict['security_code'] = row_dict['S_INFO_WINDCODE']
row_dict.pop('WIND_CODE')
row_dict.pop('OBJECT_ID')
row_dict.pop('S_INFO_WINDCODE')
doc = AShareCashFlow()
for key, value in row_dict.items():
if key.lower() in self.field_is_str_list:
if key.lower() in ['ann_dt', 'report_period', 'statement_type', 'actual_ann_dt']:
if np.isnan(value):
setattr(doc, key.lower(), -1)
else:
setattr(doc, key.lower(), int(value))
else:
setattr(doc, key.lower(), str(value))
else:
setattr(doc, key.lower(), value)
doc_list.append(doc)
if len(doc_list) > 999:
AShareCashFlow.objects.insert(doc_list)
doc_list = []
else:
AShareCashFlow.objects.insert(doc_list)
def save_a_share_ex_right_dividend(self):
database = 'stock_base_data'
with MongoConnect(database):
doc_list = []
for index, row in self.data_df.iterrows():
row_dict = dict(row)
row_dict['security_code'] = row_dict['S_INFO_WINDCODE']
row_dict.pop('OBJECT_ID')
row_dict.pop('S_INFO_WINDCODE')
doc = AShareExRightDividend()
for key, value in row_dict.items():
if key.lower() in self.collection_property_list:
property_name = AShareExRightDividend.__dict__[key.lower()]
if isinstance(property_name, StringField):
setattr(doc, key.lower(), str(value))
elif isinstance(property_name, DateTimeField):
if np.isnan(value):
setattr(doc, key.lower(), None)
else:
setattr(doc, key.lower(), datetime.strptime(str(int(value)), "%Y%m%d"))
else:
setattr(doc, key.lower(), value)
doc_list.append(doc)
if len(doc_list) > 999:
AShareExRightDividend.objects.insert(doc_list)
doc_list = []
else:
AShareExRightDividend.objects.insert(doc_list)
def save_a_share_profit_notice(self):
database = 'stock_base_data'
with MongoConnect(database):
doc_list = []
for index, row in self.data_df.iterrows():
row_dict = dict(row)
row_dict['security_code'] = row_dict['S_INFO_WINDCODE']
row_dict.pop('OBJECT_ID')
row_dict.pop('S_INFO_WINDCODE')
doc = AShareProfitNotice()
for key, value in row_dict.items():
if key.lower() in self.field_is_str_list:
if key.lower() in ['s_profitnotice_date', 's_profitnotice_period', 's_profitnotice_firstanndate']:
if np.isnan(value):
setattr(doc, key.lower(), -1)
else:
setattr(doc, key.lower(), int(value))
else:
setattr(doc, key.lower(), str(value))
else:
setattr(doc, key.lower(), value)
doc_list.append(doc)
if len(doc_list) > 999:
AShareProfitNotice.objects.insert(doc_list)
doc_list = []
else:
AShareProfitNotice.objects.insert(doc_list)
def save_a_share_adj_factor(self):
"""
取当日收盘价，作为转、送的股价，
再计算复权因子更新到AShareExRightDividend, 复权因子adj_factor
:return:
"""
database = 'stock_base_data'
with MongoConnect(database):
pass
def insert_security_code_list(self):
stock_code_list = []
for market in self.market_list:
path = self.data_path + market + '/MultDate/'
file_list = os.listdir(path)
stock_code_list += [
i.split('.')[0] + '.' + market for i in file_list
]
file_num = 0
p = Pool(8)
for file_name in file_list:
file_num += 1
print('完成数量：', file_num)
p.apply_async(self.insert_security_code,
args=(market, file_name, path))
p.close()
p.join()
delist = list(
set(self.data_dict.keys()).difference(set(stock_code_list)))
with MongoConnect(self.database):
for security_code in delist:
with switch_collection(
Kline, security_code) as KlineDaily_security_code:
doc_list = []
security_code_data = self.data_dict[
security_code].set_index(["TRADE_DT"])
for index, row in security_code_data.iterrows():
if row['S_DQ_AMOUNT'] > 0:
date_int = int(index)
date_int = str(date_int)
time_tag = datetime.strptime(date_int, "%Y%m%d")
try:
pre_close = int(row['S_DQ_PRECLOSE'] * 10000)
except KeyError:
pre_close = None
doc = KlineDaily_security_code(
time_tag=time_tag,
pre_close=pre_close,
open=int(row['S_DQ_OPEN'] * 10000),
high=int(row['S_DQ_HIGH'] * 10000),
low=int(row['S_DQ_LOW'] * 10000),
close=int(row['S_DQ_CLOSE'] * 10000),
volume=int(row['S_DQ_VOLUME'] * 100),
amount=int(row['S_DQ_AMOUNT'] * 1000),
match_items=0,
interest=0)
doc_list.append(doc)
KlineDaily_security_code.objects.insert(doc_list)
def save_a_share_calendar(self):
database = DatabaseName.STOCK_BASE_DATA.value
with MongoConnect(database):
doc_list = []
data_grouped = self.data_df.groupby("S_INFO_EXCHMARKET")
data_dict = {i[0]: list(i[1]['TRADE_DAYS']) for i in data_grouped}
for market, trade_days in data_dict.items():
if market == 'SSE':
market = 'SH'
elif market == 'SZSE':
market = 'SZ'
trade_days = [date_to_datetime(str(i)) for i in sorted(trade_days)]
doc = AShareCalendar(market=market, trade_days=trade_days)
doc_list.append(doc)
AShareCalendar.objects.insert(doc_list)
def insert_security_code(self, market, file_name, path):
database = 'a_share_kline_daily'
with MongoConnect(database):
print(path + file_name + '\n')
kline_daily_data = pd.read_csv(path + file_name,
encoding='unicode_escape')
security_code = file_name.split('.')[0] + '.' + market
kline_daily_data = kline_daily_data.reindex(columns=[
'date', 'open', 'high', 'low', 'close', 'volumw', 'turover',
'match_items', 'interest'
])
kline_daily_data.rename(columns={
'volumw': 'volume',
'turover': 'amount'
},
inplace=True)
with switch_collection(Kline,
security_code) as KlineDaily_security_code:
doc_list = []
security_code_data = pd.DataFrame()
if security_code in self.data_dict.keys():
security_code_data = self.data_dict[
security_code].set_index(["TRADE_DT"])
for index, row in kline_daily_data.iterrows():
date_int = int(row['date'])
if not np.isnan(date_int):
try:
pre_close = int(
10000 *
security_code_data.loc[date_int,
'S_DQ_PRECLOSE'])
except KeyError:
pre_close = None
date_int = str(date_int)
time_tag = datetime.strptime(date_int, "%Y%m%d")
doc = KlineDaily_security_code(
time_tag=time_tag,
pre_close=pre_close,
open=int(row['open']),
high=int(row['high']),
low=int(row['low']),
close=int(row['close']),
volume=int(row['volume']),
amount=int(row['amount']),
match_items=int(row['match_items']),
interest=int(row['interest']))
doc_list.append(doc)
KlineDaily_security_code.objects.insert(doc_list)
def save_a_share_index_members(self):
database = 'stock_base_data'
with MongoConnect(database):
doc_list = []
for index, row in self.data_df.iterrows():
row_dict = dict(row)
row_dict['index_code'] = row_dict['S_INFO_WINDCODE']
row_dict['security_code'] = row_dict['S_CON_WINDCODE']
row_dict['in_date'] = row_dict['S_CON_INDATE']
row_dict['out_date'] = row_dict['S_CON_OUTDATE']
row_dict['current_sign'] = row_dict['CUR_SIGN']
row_dict.pop('OBJECT_ID')
row_dict.pop('S_INFO_WINDCODE')
row_dict.pop('S_CON_WINDCODE')
row_dict.pop('S_CON_INDATE')
row_dict.pop('S_CON_OUTDATE')
row_dict.pop('CUR_SIGN')
self.field_is_str_list = self.field_is_str_list + [
'in_date', 'out_date'
]
doc = AShareIndexMembers()
for key, value in row_dict.items():
if key.lower() in self.collection_property_list:
property_name = AShareIndexMembers.__dict__[
key.lower()]
if isinstance(property_name, StringField):
setattr(doc, key.lower(), str(value))
elif isinstance(property_name, DateTimeField):
if np.isnan(value):
setattr(doc, key.lower(), None)
else:
setattr(
doc, key.lower(),
datetime.strptime(str(int(value)),
"%Y%m%d"))
else:
setattr(doc, key.lower(), value)
doc_list.append(doc)
if len(doc_list) > 999:
AShareIndexMembers.objects.insert(doc_list)
doc_list = []
else:
AShareIndexMembers.objects.insert(doc_list)
def save_a_index_members(self):
database = 'stock_base_data'
with MongoConnect(database):
doc_list = []
for index, row in self.data_df.iterrows():
row_dict = dict(row)
row_dict['index_code'] = row_dict['S_INFO_WINDCODE']
row_dict['security_code'] = row_dict['S_CON_WINDCODE']
row_dict['in_date'] = row_dict['S_CON_INDATE']
row_dict['out_date'] = row_dict['S_CON_OUTDATE']
row_dict['current_sign'] = row_dict['CUR_SIGN']
row_dict.pop('OBJECT_ID')
row_dict.pop('S_INFO_WINDCODE')
row_dict.pop('S_CON_WINDCODE')
row_dict.pop('S_CON_INDATE')
row_dict.pop('S_CON_OUTDATE')
row_dict.pop('CUR_SIGN')
doc = AIndexMembers()
for key, value in row_dict.items():
if key.lower() in self.field_is_str_list:
if key.lower() in [
's_con_indate', 's_con_outdate', 'current_sign'
]:
if np.isnan(value):
setattr(doc, key.lower(), -1)
else:
setattr(doc, key.lower(), int(value))
else:
setattr(doc, key.lower(), str(value))
else:
setattr(doc, key.lower(), value)
doc_list.append(doc)
if len(doc_list) > 999:
AIndexMembers.objects.insert(doc_list)
doc_list = []
else:
AIndexMembers.objects.insert(doc_list)
def insert_security_code(self, market, file_name, path):
database = 'kline'
with MongoConnect(database):
print(path + file_name + '\n')
kline_daily_data = pd.read_csv(path + file_name,
encoding='unicode_escape')
date = kline_daily_data['date'][0]
if not np.isnan(date):
security_code = file_name.split('.')[0] + '.' + market
kline_daily_data = kline_daily_data.reindex(columns=[
'open', 'high', 'low', 'close', 'volumw', 'turover',
'match_items', 'interest'
])
kline_daily_data.rename(columns={
'volumw': 'volume',
'turover': 'amount'
},
inplace=True)
with switch_collection(
KlineDaily, security_code) as KlineDaily_security_code:
doc_list = []
for index, row in kline_daily_data.iterrows():
data = [int(value) for value in dict(row).values()]
data = [
int(row['open']),
int(row['high']),
int(row['low']),
int(row['close']),
int(row['volume']),
int(row['amount']),
int(row['match_items']),
int(row['interest']),
]
doc = KlineDaily_security_code(
security_code=security_code,
date=int(date),
data=data)
doc_list.append(doc)
KlineDaily_security_code.objects.insert(doc_list)
def insert_security_code(self, market, file_name, path):
database = DatabaseName.INDEX_KLINE_DAILY.value
with MongoConnect(database):
print(path + file_name + '\n')
kline_daily_data = pd.read_csv(path + file_name,
encoding='unicode_escape')
code = file_name.split('.')[0]
code_transfer_dict = {
'999999': '000001',
'999998': '000002',
'999997': '000003',
'999996': '000004',
'999995': '000005',
'999994': '000006',
'999993': '000007',
'999992': '000008',
'999991': '000010',
'999990': '000011',
'999989': '000012',
'999988': '000013',
'999987': '000016',
'999986': '000015',
'000300': '000300'
}
if market == 'SH':
if code in code_transfer_dict.keys():
code = code_transfer_dict[code]
else:
code = '00' + code[2:]
security_code = code + '.' + market
kline_daily_data = kline_daily_data.reindex(columns=[
'date', 'open', 'high', 'low', 'close', 'volumw', 'turover',
'match_items', 'interest'
])
kline_daily_data.rename(columns={
'volumw': 'volume',
'turover': 'amount'
},
inplace=True)
with switch_collection(Kline,
security_code) as KlineDaily_security_code:
doc_list = []
for index, row in kline_daily_data.iterrows():
date_int = int(row['date'])
if not np.isnan(date_int):
date_int = str(date_int)
time_tag = datetime.strptime(date_int, "%Y%m%d")
doc = KlineDaily_security_code(
time_tag=time_tag,
pre_close=None,
open=int(row['open']),
high=int(row['high']),
low=int(row['low']),
close=int(row['close']),
volume=int(row['volume']),
amount=int(row['amount']),
match_items=None,
interest=None)
doc_list.append(doc)
KlineDaily_security_code.objects.insert(doc_list)
# -*- coding: utf-8 -*-
# ------------------------------
# @Time : 2019/11/21
# @Author : gao
# @File : update_finance_data.py
# @Project : AmazingQuant
# ------------------------------
from AmazingQuant.data_center.mongo_connection import MongoConnect
from AmazingQuant.data_center.database_field.field_a_share_finance_data import AShareIncome, AShareCashFlow
from AmazingQuant.utils.performance_test import Timer
if __name__ == '__main__':
database = 'stock_base_data'
with MongoConnect(database):
with Timer(True):
security_code_list = AShareIncome.objects.distinct('security_code')
data = AShareIncome.objects(security_code__in=security_code_list, statement_type=408009000)
for i in data:
print(i.security_code)
def insert_security_code(self, market, file_name, path):
with MongoConnect(self.database):
print(path + file_name + '\n')
kline_daily_data = pd.read_csv(path + file_name,
encoding='unicode_escape')
security_code = file_name.split('.')[0] + '.' + market
if is_security_type(security_code, 'EXTRA_STOCK_A'):
kline_daily_data = kline_daily_data.reindex(columns=[
'date', 'open', 'high', 'low', 'close', 'volumw',
'turover', 'match_items', 'interest'
])
kline_daily_data.rename(columns={
'volumw': 'volume',
'turover': 'amount'
},
inplace=True)
kline_daily_data = kline_daily_data[
kline_daily_data.date >= 20020104]
with switch_collection(
Kline, security_code) as KlineDaily_security_code:
doc_list = []
security_code_data = pd.DataFrame()
if security_code in self.data_dict.keys():
security_code_data = self.data_dict[
security_code].set_index(["TRADE_DT"])
security_code_data = security_code_data.fillna(0)
for index, row in kline_daily_data.iterrows():
date_int = int(row['date'])
if not np.isnan(date_int):
try:
pre_close = int(
10000 *
security_code_data.loc[date_int,
'S_DQ_PRECLOSE'])
except KeyError:
pre_close = None
date_int = str(date_int)
time_tag = datetime.strptime(date_int, "%Y%m%d")
doc = KlineDaily_security_code(
time_tag=time_tag,
pre_close=pre_close,
open=int(row['open']),
high=int(row['high']),
low=int(row['low']),
close=int(row['close']),
volume=int(row['volume']),
amount=int(row['amount']),
match_items=int(row['match_items']),
interest=int(row['interest']))
doc_list.append(doc)
# 用csv全表补充20020104之前的日线数据，match_items为0
security_code_data = security_code_data[
security_code_data.index < 20020104]
for index, row in security_code_data.iterrows():
if row['S_DQ_AMOUNT'] > 0:
date_int = int(index)
date_int = str(date_int)
time_tag = datetime.strptime(date_int, "%Y%m%d")
try:
pre_close = int(row['S_DQ_PRECLOSE'] * 10000)
except KeyError:
pre_close = None
doc = KlineDaily_security_code(
time_tag=time_tag,
pre_close=pre_close,
open=int(row['S_DQ_OPEN'] * 10000),
high=int(row['S_DQ_HIGH'] * 10000),
low=int(row['S_DQ_LOW'] * 10000),
close=int(row['S_DQ_CLOSE'] * 10000),
volume=int(row['S_DQ_VOLUME'] * 100),
amount=int(row['S_DQ_AMOUNT'] * 1000),
match_items=0,
interest=0)
doc_list.append(doc)
KlineDaily_security_code.objects.insert(doc_list)