Ejemplo n.º 1
0
    def save_a_share_cash_flow(self):
        database = 'stock_base_data'
        with MongoConnect(database):
            doc_list = []
            for index, row in self.data_df.iterrows():
                row_dict = dict(row)
                row_dict['security_code'] = row_dict['S_INFO_WINDCODE']
                row_dict.pop('WIND_CODE')
                row_dict.pop('OBJECT_ID')
                row_dict.pop('S_INFO_WINDCODE')

                doc = AShareCashFlow()
                for key, value in row_dict.items():
                    if key.lower() in self.field_is_str_list:
                        if key.lower() in ['ann_dt', 'report_period', 'statement_type', 'actual_ann_dt']:
                            if np.isnan(value):
                                setattr(doc, key.lower(), -1)
                            else:
                                setattr(doc, key.lower(), int(value))
                        else:
                            setattr(doc, key.lower(), str(value))
                    else:
                        setattr(doc, key.lower(), value)
                doc_list.append(doc)
                if len(doc_list) > 999:
                    AShareCashFlow.objects.insert(doc_list)
                    doc_list = []
            else:
                AShareCashFlow.objects.insert(doc_list)
Ejemplo n.º 2
0
    def save_a_share_ex_right_dividend(self):
        database = 'stock_base_data'
        with MongoConnect(database):
            doc_list = []
            for index, row in self.data_df.iterrows():
                row_dict = dict(row)

                row_dict['security_code'] = row_dict['S_INFO_WINDCODE']
                row_dict.pop('OBJECT_ID')
                row_dict.pop('S_INFO_WINDCODE')

                doc = AShareExRightDividend()

                for key, value in row_dict.items():
                    if key.lower() in self.collection_property_list:
                        property_name = AShareExRightDividend.__dict__[key.lower()]
                        if isinstance(property_name, StringField):
                            setattr(doc, key.lower(), str(value))
                        elif isinstance(property_name, DateTimeField):
                            if np.isnan(value):
                                setattr(doc, key.lower(), None)
                            else:
                                setattr(doc, key.lower(), datetime.strptime(str(int(value)), "%Y%m%d"))
                        else:
                            setattr(doc, key.lower(), value)
                doc_list.append(doc)
                if len(doc_list) > 999:
                    AShareExRightDividend.objects.insert(doc_list)
                    doc_list = []
            else:
                AShareExRightDividend.objects.insert(doc_list)
Ejemplo n.º 3
0
    def save_a_share_profit_notice(self):
        database = 'stock_base_data'
        with MongoConnect(database):
            doc_list = []
            for index, row in self.data_df.iterrows():
                row_dict = dict(row)
                row_dict['security_code'] = row_dict['S_INFO_WINDCODE']
                row_dict.pop('OBJECT_ID')
                row_dict.pop('S_INFO_WINDCODE')

                doc = AShareProfitNotice()
                for key, value in row_dict.items():
                    if key.lower() in self.field_is_str_list:
                        if key.lower() in ['s_profitnotice_date', 's_profitnotice_period', 's_profitnotice_firstanndate']:
                            if np.isnan(value):
                                setattr(doc, key.lower(), -1)
                            else:
                                setattr(doc, key.lower(), int(value))
                        else:
                            setattr(doc, key.lower(), str(value))
                    else:
                        setattr(doc, key.lower(), value)
                doc_list.append(doc)
                if len(doc_list) > 999:
                    AShareProfitNotice.objects.insert(doc_list)
                    doc_list = []
            else:
                AShareProfitNotice.objects.insert(doc_list)
Ejemplo n.º 4
0
 def save_a_share_adj_factor(self):
     """
     取当日收盘价,作为转、送的股价,
     再计算复权因子更新到AShareExRightDividend, 复权因子adj_factor
     :return:
     """
     database = 'stock_base_data'
     with MongoConnect(database):
         pass
Ejemplo n.º 5
0
    def insert_security_code_list(self):
        stock_code_list = []
        for market in self.market_list:
            path = self.data_path + market + '/MultDate/'
            file_list = os.listdir(path)
            stock_code_list += [
                i.split('.')[0] + '.' + market for i in file_list
            ]
            file_num = 0
            p = Pool(8)
            for file_name in file_list:
                file_num += 1
                print('完成数量:', file_num)
                p.apply_async(self.insert_security_code,
                              args=(market, file_name, path))
            p.close()
            p.join()

        delist = list(
            set(self.data_dict.keys()).difference(set(stock_code_list)))
        with MongoConnect(self.database):
            for security_code in delist:
                with switch_collection(
                        Kline, security_code) as KlineDaily_security_code:
                    doc_list = []
                    security_code_data = self.data_dict[
                        security_code].set_index(["TRADE_DT"])
                    for index, row in security_code_data.iterrows():
                        if row['S_DQ_AMOUNT'] > 0:
                            date_int = int(index)
                            date_int = str(date_int)
                            time_tag = datetime.strptime(date_int, "%Y%m%d")
                            try:
                                pre_close = int(row['S_DQ_PRECLOSE'] * 10000)
                            except KeyError:
                                pre_close = None
                            doc = KlineDaily_security_code(
                                time_tag=time_tag,
                                pre_close=pre_close,
                                open=int(row['S_DQ_OPEN'] * 10000),
                                high=int(row['S_DQ_HIGH'] * 10000),
                                low=int(row['S_DQ_LOW'] * 10000),
                                close=int(row['S_DQ_CLOSE'] * 10000),
                                volume=int(row['S_DQ_VOLUME'] * 100),
                                amount=int(row['S_DQ_AMOUNT'] * 1000),
                                match_items=0,
                                interest=0)
                            doc_list.append(doc)
                    KlineDaily_security_code.objects.insert(doc_list)
 def save_a_share_calendar(self):
     database = DatabaseName.STOCK_BASE_DATA.value
     with MongoConnect(database):
         doc_list = []
         data_grouped = self.data_df.groupby("S_INFO_EXCHMARKET")
         data_dict = {i[0]: list(i[1]['TRADE_DAYS']) for i in data_grouped}
         for market, trade_days in data_dict.items():
             if market == 'SSE':
                 market = 'SH'
             elif market == 'SZSE':
                 market = 'SZ'
             trade_days = [date_to_datetime(str(i)) for i in sorted(trade_days)]
             doc = AShareCalendar(market=market, trade_days=trade_days)
             doc_list.append(doc)
         AShareCalendar.objects.insert(doc_list)
Ejemplo n.º 7
0
 def insert_security_code(self, market, file_name, path):
     database = 'a_share_kline_daily'
     with MongoConnect(database):
         print(path + file_name + '\n')
         kline_daily_data = pd.read_csv(path + file_name,
                                        encoding='unicode_escape')
         security_code = file_name.split('.')[0] + '.' + market
         kline_daily_data = kline_daily_data.reindex(columns=[
             'date', 'open', 'high', 'low', 'close', 'volumw', 'turover',
             'match_items', 'interest'
         ])
         kline_daily_data.rename(columns={
             'volumw': 'volume',
             'turover': 'amount'
         },
                                 inplace=True)
         with switch_collection(Kline,
                                security_code) as KlineDaily_security_code:
             doc_list = []
             security_code_data = pd.DataFrame()
             if security_code in self.data_dict.keys():
                 security_code_data = self.data_dict[
                     security_code].set_index(["TRADE_DT"])
             for index, row in kline_daily_data.iterrows():
                 date_int = int(row['date'])
                 if not np.isnan(date_int):
                     try:
                         pre_close = int(
                             10000 *
                             security_code_data.loc[date_int,
                                                    'S_DQ_PRECLOSE'])
                     except KeyError:
                         pre_close = None
                     date_int = str(date_int)
                     time_tag = datetime.strptime(date_int, "%Y%m%d")
                     doc = KlineDaily_security_code(
                         time_tag=time_tag,
                         pre_close=pre_close,
                         open=int(row['open']),
                         high=int(row['high']),
                         low=int(row['low']),
                         close=int(row['close']),
                         volume=int(row['volume']),
                         amount=int(row['amount']),
                         match_items=int(row['match_items']),
                         interest=int(row['interest']))
                     doc_list.append(doc)
             KlineDaily_security_code.objects.insert(doc_list)
Ejemplo n.º 8
0
    def save_a_share_index_members(self):
        database = 'stock_base_data'
        with MongoConnect(database):
            doc_list = []
            for index, row in self.data_df.iterrows():
                row_dict = dict(row)

                row_dict['index_code'] = row_dict['S_INFO_WINDCODE']
                row_dict['security_code'] = row_dict['S_CON_WINDCODE']
                row_dict['in_date'] = row_dict['S_CON_INDATE']
                row_dict['out_date'] = row_dict['S_CON_OUTDATE']
                row_dict['current_sign'] = row_dict['CUR_SIGN']

                row_dict.pop('OBJECT_ID')
                row_dict.pop('S_INFO_WINDCODE')
                row_dict.pop('S_CON_WINDCODE')
                row_dict.pop('S_CON_INDATE')
                row_dict.pop('S_CON_OUTDATE')
                row_dict.pop('CUR_SIGN')
                self.field_is_str_list = self.field_is_str_list + [
                    'in_date', 'out_date'
                ]
                doc = AShareIndexMembers()

                for key, value in row_dict.items():
                    if key.lower() in self.collection_property_list:
                        property_name = AShareIndexMembers.__dict__[
                            key.lower()]
                        if isinstance(property_name, StringField):
                            setattr(doc, key.lower(), str(value))
                        elif isinstance(property_name, DateTimeField):
                            if np.isnan(value):
                                setattr(doc, key.lower(), None)
                            else:
                                setattr(
                                    doc, key.lower(),
                                    datetime.strptime(str(int(value)),
                                                      "%Y%m%d"))
                        else:
                            setattr(doc, key.lower(), value)
                doc_list.append(doc)
                if len(doc_list) > 999:
                    AShareIndexMembers.objects.insert(doc_list)
                    doc_list = []
            else:
                AShareIndexMembers.objects.insert(doc_list)
Ejemplo n.º 9
0
    def save_a_index_members(self):
        database = 'stock_base_data'
        with MongoConnect(database):
            doc_list = []
            for index, row in self.data_df.iterrows():
                row_dict = dict(row)

                row_dict['index_code'] = row_dict['S_INFO_WINDCODE']
                row_dict['security_code'] = row_dict['S_CON_WINDCODE']
                row_dict['in_date'] = row_dict['S_CON_INDATE']
                row_dict['out_date'] = row_dict['S_CON_OUTDATE']
                row_dict['current_sign'] = row_dict['CUR_SIGN']

                row_dict.pop('OBJECT_ID')
                row_dict.pop('S_INFO_WINDCODE')
                row_dict.pop('S_CON_WINDCODE')
                row_dict.pop('S_CON_INDATE')
                row_dict.pop('S_CON_OUTDATE')
                row_dict.pop('CUR_SIGN')

                doc = AIndexMembers()
                for key, value in row_dict.items():
                    if key.lower() in self.field_is_str_list:
                        if key.lower() in [
                                's_con_indate', 's_con_outdate', 'current_sign'
                        ]:
                            if np.isnan(value):
                                setattr(doc, key.lower(), -1)
                            else:
                                setattr(doc, key.lower(), int(value))
                        else:
                            setattr(doc, key.lower(), str(value))
                    else:
                        setattr(doc, key.lower(), value)
                doc_list.append(doc)
                if len(doc_list) > 999:
                    AIndexMembers.objects.insert(doc_list)
                    doc_list = []
            else:
                AIndexMembers.objects.insert(doc_list)
Ejemplo n.º 10
0
    def insert_security_code(self, market, file_name, path):
        database = 'kline'
        with MongoConnect(database):
            print(path + file_name + '\n')
            kline_daily_data = pd.read_csv(path + file_name,
                                           encoding='unicode_escape')
            date = kline_daily_data['date'][0]
            if not np.isnan(date):
                security_code = file_name.split('.')[0] + '.' + market
                kline_daily_data = kline_daily_data.reindex(columns=[
                    'open', 'high', 'low', 'close', 'volumw', 'turover',
                    'match_items', 'interest'
                ])
                kline_daily_data.rename(columns={
                    'volumw': 'volume',
                    'turover': 'amount'
                },
                                        inplace=True)
                with switch_collection(
                        KlineDaily, security_code) as KlineDaily_security_code:
                    doc_list = []
                    for index, row in kline_daily_data.iterrows():
                        data = [int(value) for value in dict(row).values()]
                        data = [
                            int(row['open']),
                            int(row['high']),
                            int(row['low']),
                            int(row['close']),
                            int(row['volume']),
                            int(row['amount']),
                            int(row['match_items']),
                            int(row['interest']),
                        ]
                        doc = KlineDaily_security_code(
                            security_code=security_code,
                            date=int(date),
                            data=data)
                        doc_list.append(doc)

                    KlineDaily_security_code.objects.insert(doc_list)
    def insert_security_code(self, market, file_name, path):
        database = DatabaseName.INDEX_KLINE_DAILY.value
        with MongoConnect(database):
            print(path + file_name + '\n')
            kline_daily_data = pd.read_csv(path + file_name,
                                           encoding='unicode_escape')
            code = file_name.split('.')[0]
            code_transfer_dict = {
                '999999': '000001',
                '999998': '000002',
                '999997': '000003',
                '999996': '000004',
                '999995': '000005',
                '999994': '000006',
                '999993': '000007',
                '999992': '000008',
                '999991': '000010',
                '999990': '000011',
                '999989': '000012',
                '999988': '000013',
                '999987': '000016',
                '999986': '000015',
                '000300': '000300'
            }
            if market == 'SH':
                if code in code_transfer_dict.keys():
                    code = code_transfer_dict[code]
                else:
                    code = '00' + code[2:]
            security_code = code + '.' + market
            kline_daily_data = kline_daily_data.reindex(columns=[
                'date', 'open', 'high', 'low', 'close', 'volumw', 'turover',
                'match_items', 'interest'
            ])
            kline_daily_data.rename(columns={
                'volumw': 'volume',
                'turover': 'amount'
            },
                                    inplace=True)

            with switch_collection(Kline,
                                   security_code) as KlineDaily_security_code:
                doc_list = []
                for index, row in kline_daily_data.iterrows():
                    date_int = int(row['date'])
                    if not np.isnan(date_int):
                        date_int = str(date_int)
                        time_tag = datetime.strptime(date_int, "%Y%m%d")
                        doc = KlineDaily_security_code(
                            time_tag=time_tag,
                            pre_close=None,
                            open=int(row['open']),
                            high=int(row['high']),
                            low=int(row['low']),
                            close=int(row['close']),
                            volume=int(row['volume']),
                            amount=int(row['amount']),
                            match_items=None,
                            interest=None)
                        doc_list.append(doc)

                KlineDaily_security_code.objects.insert(doc_list)
Ejemplo n.º 12
0
# -*- coding: utf-8 -*-

# ------------------------------
# @Time    : 2019/11/21
# @Author  : gao
# @File    : update_finance_data.py
# @Project : AmazingQuant
# ------------------------------

from AmazingQuant.data_center.mongo_connection import MongoConnect
from AmazingQuant.data_center.database_field.field_a_share_finance_data import AShareIncome, AShareCashFlow
from AmazingQuant.utils.performance_test import Timer


if __name__ == '__main__':
    database = 'stock_base_data'
    with MongoConnect(database):
        with Timer(True):
            security_code_list = AShareIncome.objects.distinct('security_code')
            data = AShareIncome.objects(security_code__in=security_code_list, statement_type=408009000)
            for i in data:
                print(i.security_code)
Ejemplo n.º 13
0
    def insert_security_code(self, market, file_name, path):
        with MongoConnect(self.database):
            print(path + file_name + '\n')
            kline_daily_data = pd.read_csv(path + file_name,
                                           encoding='unicode_escape')
            security_code = file_name.split('.')[0] + '.' + market
            if is_security_type(security_code, 'EXTRA_STOCK_A'):
                kline_daily_data = kline_daily_data.reindex(columns=[
                    'date', 'open', 'high', 'low', 'close', 'volumw',
                    'turover', 'match_items', 'interest'
                ])
                kline_daily_data.rename(columns={
                    'volumw': 'volume',
                    'turover': 'amount'
                },
                                        inplace=True)
                kline_daily_data = kline_daily_data[
                    kline_daily_data.date >= 20020104]
                with switch_collection(
                        Kline, security_code) as KlineDaily_security_code:
                    doc_list = []
                    security_code_data = pd.DataFrame()
                    if security_code in self.data_dict.keys():
                        security_code_data = self.data_dict[
                            security_code].set_index(["TRADE_DT"])
                        security_code_data = security_code_data.fillna(0)
                    for index, row in kline_daily_data.iterrows():
                        date_int = int(row['date'])
                        if not np.isnan(date_int):
                            try:
                                pre_close = int(
                                    10000 *
                                    security_code_data.loc[date_int,
                                                           'S_DQ_PRECLOSE'])
                            except KeyError:
                                pre_close = None
                            date_int = str(date_int)
                            time_tag = datetime.strptime(date_int, "%Y%m%d")
                            doc = KlineDaily_security_code(
                                time_tag=time_tag,
                                pre_close=pre_close,
                                open=int(row['open']),
                                high=int(row['high']),
                                low=int(row['low']),
                                close=int(row['close']),
                                volume=int(row['volume']),
                                amount=int(row['amount']),
                                match_items=int(row['match_items']),
                                interest=int(row['interest']))
                            doc_list.append(doc)

                    # 用csv全表补充20020104之前的日线数据,match_items为0
                    security_code_data = security_code_data[
                        security_code_data.index < 20020104]
                    for index, row in security_code_data.iterrows():
                        if row['S_DQ_AMOUNT'] > 0:
                            date_int = int(index)
                            date_int = str(date_int)
                            time_tag = datetime.strptime(date_int, "%Y%m%d")
                            try:
                                pre_close = int(row['S_DQ_PRECLOSE'] * 10000)
                            except KeyError:
                                pre_close = None
                            doc = KlineDaily_security_code(
                                time_tag=time_tag,
                                pre_close=pre_close,
                                open=int(row['S_DQ_OPEN'] * 10000),
                                high=int(row['S_DQ_HIGH'] * 10000),
                                low=int(row['S_DQ_LOW'] * 10000),
                                close=int(row['S_DQ_CLOSE'] * 10000),
                                volume=int(row['S_DQ_VOLUME'] * 100),
                                amount=int(row['S_DQ_AMOUNT'] * 1000),
                                match_items=0,
                                interest=0)
                            doc_list.append(doc)
                    KlineDaily_security_code.objects.insert(doc_list)