def record(self, entity, start, end, size, timestamps):
        for timestamp in timestamps:
            df = run_query(
                table='finance.STK_HK_HOLD_INFO',
                conditions=
                f'link_id#=#{entity.code}&day#=#{to_time_str(timestamp)}')
            print(df)

            if pd_is_not_null(df):
                df.rename(columns={
                    'day': 'timestamp',
                    'link_id': 'holder_code',
                    'link_name': 'holder_name'
                },
                          inplace=True)
                df['timestamp'] = pd.to_datetime(df['timestamp'])

                df['entity_id'] = df['code'].apply(
                    lambda x: to_entity_id(entity_type='stock', jq_code=x))
                df['code'] = df['code'].apply(lambda x: x.split('.')[0])

                # id格式为:{holder_name}_{entity_id}_{timestamp}
                df['id'] = df[['holder_name', 'entity_id', 'timestamp']].apply(
                    lambda se: "{}_{}_{}".format(
                        se['holder_name'], se['entity_id'],
                        to_time_str(se['timestamp'], fmt=TIME_FORMAT_DAY)),
                    axis=1)

                df_to_db(df=df,
                         data_schema=self.data_schema,
                         provider=self.provider,
                         force_update=self.force_update)
Esempio n. 2
0
    def record(self, entity, start, end, size, timestamps):
        jq_code = code_map_jq.get(entity.code)

        q = query(finance.STK_MT_TOTAL).filter(
            finance.STK_MT_TOTAL.exchange_code == jq_code,
            finance.STK_MT_TOTAL.date >= to_time_str(start)).limit(2000)

        df = finance.run_query(q)
        print(df)

        json_results = []

        for item in df.to_dict(orient='records'):
            result = {
                'provider': self.provider,
                'timestamp': item['date'],
                'name': entity.name,
                'margin_value': item['fin_value'],
                'margin_buy': item['fin_buy_value'],
                'short_value': item['sec_value'],
                'short_volume': item['sec_sell_volume'],
                'total_value': item['fin_sec_value']
            }

            json_results.append(result)

        if len(json_results) < 100:
            self.one_shot = True

        return json_results
Esempio n. 3
0
    def record(self, entity, start, end, size, timestamps):
        q = query(
            valuation
        ).filter(
            valuation.code == to_jq_entity_id(entity)
        )
        count: pd.Timedelta = now_pd_timestamp() - start
        df = get_fundamentals_continuously(q, end_date=now_time_str(), count=count.days + 1, panel=False)
        df['entity_id'] = entity.id
        df['timestamp'] = pd.to_datetime(df['day'])
        df['code'] = entity.code
        df['name'] = entity.name
        df['id'] = df['timestamp'].apply(lambda x: "{}_{}".format(entity.id, to_time_str(x)))
        df = df.rename({'pe_ratio_lyr': 'pe',
                        'pe_ratio': 'pe_ttm',
                        'pb_ratio': 'pb',
                        'ps_ratio': 'ps',
                        'pcf_ratio': 'pcf'},
                       axis='columns')

        df['market_cap'] = df['market_cap'] * 100000000
        df['circulating_market_cap'] = df['circulating_market_cap'] * 100000000
        df['capitalization'] = df['capitalization'] * 10000
        df['circulating_cap'] = df['circulating_cap'] * 10000
        df['turnover_ratio'] = df['turnover_ratio'] * 0.01
        df_to_db(df=df, data_schema=self.data_schema, provider=self.provider, force_update=self.force_update)

        return None
Esempio n. 4
0
    def generate_request_param(self, security_item, start, end, size, timestamps):
        if len(timestamps) <= 10:
            param = {
                "color": "w",
                "fc": get_fc(security_item),
                "corpType": company_type_flag(security_item),
                # 0 means get all types
                "reportDateType": 0,
                "endDate": '',
                "latestCount": size
            }
        else:
            param = {
                "color": "w",
                "fc": get_fc(security_item),
                "corpType": company_type_flag(security_item),
                # 0 means get all types
                "reportDateType": 0,
                "endDate": to_time_str(timestamps[10]),
                "latestCount": 10
            }

        if self.finance_report_type == 'LiRunBiaoList' or self.finance_report_type == 'XianJinLiuLiangBiaoList':
            param['reportType'] = 1

        return param
Esempio n. 5
0
def common_draw(df_list: List[pd.DataFrame], chart_type=Line, columns=[], name_field='security_id', render='html',
                file_name=None):
    if len(df_list) > 1:
        df_list = fill_with_same_index(df_list=df_list)

    chart = None

    if chart_type == Line:
        chart = get_default_line()
        assert len(columns) == 1

    xdata = [to_time_str(timestamp) for timestamp in df_list[0].index]

    chart.add_xaxis(xdata)

    for df in df_list:
        series_name = df[df[name_field].notna()][name_field][0]

        if len(columns) == 1:
            ydata = df.loc[:, columns[0]].values.tolist()

        chart.add_yaxis(series_name, ydata, is_smooth=True,
                        markpoint_opts=opts.MarkPointOpts(
                            data=[opts.MarkPointItem(type_="min"), opts.MarkPointItem(type_="max")]))

        if render == 'html':
            chart.render(get_ui_path(file_name))
        elif render == 'notebook':
            chart.render_notebook()

    return chart
Esempio n. 6
0
    def on_trading_open(self, timestamp):
        self.logger.info('on_trading_open:{}'.format(timestamp))
        if is_same_date(timestamp, self.start_timestamp):
            return
        # get the account for trading at the date
        accounts = get_account_stats(session=self.session,
                                     trader_name=self.trader_name,
                                     return_type='domain',
                                     end_timestamp=to_time_str(timestamp),
                                     limit=1,
                                     order=AccountStats.timestamp.desc())
        if accounts:
            account = accounts[0]
        else:
            return

        positions = []
        # FIXME:dump all directly
        for position_domain in account.positions:
            position_dict = position_schema.dump(position_domain)
            self.logger.info('current position:{}'.format(position_dict))
            del position_dict['account_stats']
            positions.append(position_dict)

        self.latest_account = account_stats_schema.dump(account)
        self.latest_account['positions'] = positions
        self.logger.info('on_trading_open:{},latest_account:{}'.format(
            timestamp, self.latest_account))
Esempio n. 7
0
    def on_finish(self, security_item):
        kdatas = get_kdata(provider=self.provider, security_id=security_item.id, level=self.level.value,
                           order=self.data_schema.timestamp.asc(),
                           return_type='domain',
                           session=self.session,
                           filters=[self.data_schema.hfq_close.is_(None),
                                    self.data_schema.timestamp >= to_pd_timestamp('2005-01-01')])
        if kdatas:
            start = kdatas[0].timestamp
            end = kdatas[-1].timestamp

            # get hfq from joinquant
            df = get_price(to_jq_security_id(security_item), start_date=to_time_str(start), end_date=now_time_str(),
                           frequency='daily',
                           fields=['factor', 'open', 'close', 'low', 'high'],
                           skip_paused=True, fq='post')
            if df is not None and not df.empty:
                # fill hfq data
                for kdata in kdatas:
                    time_str = to_time_str(kdata.timestamp)
                    if time_str in df.index:
                        kdata.hfq_open = df.loc[time_str, 'open']
                        kdata.hfq_close = df.loc[time_str, 'close']
                        kdata.hfq_high = df.loc[time_str, 'high']
                        kdata.hfq_low = df.loc[time_str, 'low']
                        kdata.factor = df.loc[time_str, 'factor']
                self.session.commit()

                latest_factor = df.factor[-1]
                # factor not change yet, no need to reset the qfq past
                if latest_factor == self.current_factors.get(security_item.id):
                    sql = 'UPDATE {} SET qfq_close=hfq_close/{},qfq_high=hfq_high/{}, qfq_open= hfq_open/{}, qfq_low= hfq_low/{} where ' \
                          'security_id=\'{}\' and level=\'{}\' and (qfq_close isnull or qfq_high isnull or qfq_low isnull or qfq_open isnull)'.format(
                        self.data_schema.__table__, latest_factor, latest_factor, latest_factor, latest_factor,
                        security_item.id, self.level.value)
                else:
                    sql = 'UPDATE {} SET qfq_close=hfq_close/{},qfq_high=hfq_high/{}, qfq_open= hfq_open/{}, qfq_low= hfq_low/{} where ' \
                          'security_id=\'{}\' and level=\'{}\''.format(self.data_schema.__table__, latest_factor,
                                                                       latest_factor, latest_factor, latest_factor,
                                                                       security_item.id,
                                                                       self.level.value)
                self.logger.info(sql)
                self.session.execute(sql)
                self.session.commit()

        # TODO:use netease provider to get turnover_rate
        self.logger.info('use netease provider to get turnover_rate')
Esempio n. 8
0
def test_000778_cash_flow_statement():
    correct_timestamps = ['2018-09-30', '2018-06-30', '2018-03-31', '2017-12-31', '2017-09-30', '2017-06-30',
                          '2017-03-31', '2016-12-31', '2016-09-30', '2016-06-30', '2016-03-31', '2015-12-31',
                          '2015-09-30', '2015-06-30', '2015-03-31', '2014-12-31', '2014-09-30', '2014-06-30',
                          '2014-03-31', '2013-12-31', '2013-09-30', '2013-06-30', '2013-03-31', '2012-12-31',
                          '2012-09-30', '2012-06-30', '2012-03-31', '2011-12-31', '2011-09-30', '2011-06-30',
                          '2011-03-31', '2010-12-31', '2010-09-30', '2010-06-30', '2010-03-31', '2009-12-31',
                          '2009-09-30', '2009-06-30', '2009-03-31', '2008-12-31', '2008-09-30', '2008-06-30',
                          '2008-03-31', '2007-12-31', '2007-09-30', '2007-06-30', '2007-03-31', '2006-12-31',
                          '2006-09-30', '2006-06-30', '2006-03-31', '2005-12-31', '2005-09-30', '2005-06-30',
                          '2005-03-31', '2004-12-31', '2004-09-30', '2004-06-30', '2004-03-31', '2003-12-31',
                          '2003-09-30', '2003-06-30', '2003-03-31', '2002-12-31', '2002-06-30', '2001-12-31',
                          '2001-06-30', '2000-12-31', '2000-06-30', '1999-12-31', '1998-12-31', '1998-06-30']
    result = fundamental.get_cash_flow_statement(session=session, provider=Provider.EASTMONEY, return_type='domain',
                                                 codes=['000778'], end_timestamp='2018-12-30',
                                                 order=CashFlowStatement.timestamp.desc())
    assert len(correct_timestamps) == len(result)
    timestamps = [to_time_str(item.report_date) for item in result]
    assert set(correct_timestamps) == set(timestamps)
    latest: CashFlowStatement = result[0]

    assert latest.cash_from_selling == 27784000000
    assert latest.tax_refund == 60700000
    assert latest.cash_from_other_op == 1463000000
    assert latest.total_op_cash_inflows == 29310000000
    assert latest.cash_to_goods_services == 21210000000
    assert latest.cash_to_employees == 1460000000
    assert latest.taxes_and_surcharges == 2016000000
    assert latest.cash_to_other_related_op == 573700000
    assert latest.total_op_cash_outflows == 25260000000
    assert latest.net_op_cash_flows == 4050000000

    assert latest.cash_from_disposal_of_investments == 556500000
    assert latest.cash_from_returns_on_investments == 44180000
    assert latest.cash_from_disposal_fixed_intangible_assets == 457200
    assert latest.cash_from_disposal_subsidiaries == 1046000000
    assert latest.cash_from_other_investing == 553000000
    assert latest.total_investing_cash_inflows == 2201000000
    assert latest.cash_to_acquire_fixed_intangible_assets == 2521000000
    assert latest.cash_to_investments == 1808000000
    assert latest.total_investing_cash_outflows == 4329000000
    assert latest.net_investing_cash_flows == -2128000000

    assert latest.cash_from_accepting_investment == 24500000
    assert latest.cash_from_subsidiaries_accepting_minority_interest == 24500000
    assert latest.cash_from_borrowings == 10080000000
    assert latest.cash_from_issuing_bonds == 997000000
    assert latest.cash_from_other_financing == 200000000
    assert latest.total_financing_cash_inflows == 11300000000
    assert latest.cash_to_repay_borrowings == 11940000000
    assert latest.cash_to_pay_interest_dividend == 892100000
    assert latest.cash_to_other_financing == 328500000
    assert latest.total_financing_cash_outflows == 13160000000
    assert latest.net_financing_cash_flows == -1862000000

    assert latest.foreign_exchange_rate_effect == 21350000
    assert latest.net_cash_increase == 81240000
    assert latest.cash_at_beginning == 5078000000
    assert latest.cash == 5159000000
 def generate_request_param(self, security_item, start, end, size,
                            timestamp):
     return {
         'security_item': security_item,
         'start': to_time_str(start, fmt=TIME_FORMAT_DAY1),
         'end': now_time_str(fmt=TIME_FORMAT_DAY1),
         'level': self.level.value
     }
 def generate_id(se):
     holdname = se['holder_name']
     if len(holdname) > 20:
         holdname = str(holdname).split('、')[0]
     return "{}_{}_{}".format(
         se['entity_id'],
         to_time_str(se['timestamp'], fmt=TIME_FORMAT_DAY),
         holdname)
Esempio n. 11
0
    def record(self, entity, start, end, size, timestamps):
        if not end:
            end = to_time_str(now_pd_timestamp())
        if (pd.to_datetime(end) - start).days >= 800:
            from datetime import timedelta
            end = to_time_str(start + timedelta(days=800))
        start = to_time_str(start)
        exchange = 'SH' if 'sh' in entity.id else 'SZ'
        em_code = entity.code + '.' + exchange
        if em_code == '000022.SH':
            return None
        columns_list = {
            'MV': 'market_cap',  #总市值
            'LIQMV': 'circulating_market_cap',  #流通市值
            'TURN': 'turnover_ratio',  #换手率
            'PELYR': 'pe',  # 静态pe
            'PETTM': 'pe_ttm',  # 动态pe
            'PBLYR': 'pb',  # 市净率PB(最新年报)
            'PBMRQ': 'pb_mrq',  # 市净率PB(MRQ)
            'PSTTM': 'ps_ttm',  #市销率PS(TTM)
            'PCFTTM': 'pcf_ttm',  #市现率PCF(最新年报,经营性现金流)
            'DIVIDENDYIELD': 'div_yield',  #股息率
        }
        df = c.csd(em_code, [i for i in columns_list.keys()], start, end,
                   "ispandas=1,DelType=1")
        if not isinstance(df, pd.DataFrame):
            return None
        # 该方法会获取到未来数据,总市值为Nan时删除该条
        df = df.dropna(subset=['MV'])
        if df.empty:
            return None
        df = df.rename(columns=columns_list)

        df['entity_id'] = entity.id[:15]
        df['timestamp'] = pd.to_datetime(df['DATES'])
        df['code'] = entity.code
        df['name'] = entity.name
        df['id'] = df.apply(
            lambda x: "{}_{}".format(x.entity_id, to_time_str(x.timestamp)),
            axis=1)

        df_to_db(df=df,
                 data_schema=self.data_schema,
                 provider=self.provider,
                 force_update=self.force_update)
        return None
Esempio n. 12
0
def marshal_object_for_ui(object):
    if isinstance(object, Enum):
        return object.value

    if isinstance(object, pd.Timestamp):
        return to_time_str(object)

    return object
Esempio n. 13
0
def report_core_company():
    while True:
        error_count = 0
        email_action = EmailInformer()

        try:
            # StockTradeDay.record_data(provider='joinquant')
            # Stock.record_data(provider='joinquant')
            # FinanceFactor.record_data(provider='eastmoney')
            # BalanceSheet.record_data(provider='eastmoney')

            target_date = to_time_str(now_pd_timestamp())

            my_selector: TargetSelector = FundamentalSelector(
                start_timestamp='2015-01-01', end_timestamp=target_date)
            my_selector.run()

            long_targets = my_selector.get_open_long_targets(
                timestamp=target_date)
            if long_targets:
                stocks = get_entities(provider='joinquant',
                                      entity_schema=Stock,
                                      entity_ids=long_targets,
                                      return_type='domain')

                # add them to eastmoney
                try:
                    try:
                        eastmoneypy.del_group('core')
                    except:
                        pass
                    eastmoneypy.create_group('core')
                    for stock in stocks:
                        eastmoneypy.add_to_group(stock.code, group_name='core')
                except Exception as e:
                    email_action.send_message(
                        "*****@*****.**", f'report_core_company error',
                        'report_core_company error:{}'.format(e))

                info = [f'{stock.name}({stock.code})' for stock in stocks]
                msg = ' '.join(info)
            else:
                msg = 'no targets'

            logger.info(msg)

            email_action.send_message(get_subscriber_emails(),
                                      f'{to_time_str(target_date)} 核心资产选股结果',
                                      msg)
            break
        except Exception as e:
            logger.exception('report_core_company error:{}'.format(e))
            time.sleep(60 * 3)
            error_count = error_count + 1
            if error_count == 10:
                email_action.send_message(
                    "*****@*****.**", f'report_core_company error',
                    'report_core_company error:{}'.format(e))
Esempio n. 14
0
    def record(self, entity, start, end, size, timestamps):
        now_date = to_time_str(now_pd_timestamp())
        sql_bars = 'select a.TRADE_DATE,a.TICKER_SYMBOL,b.SEC_SHORT_NAME,a.PRE_CLOSE_PRICE_1,' \
                   ' a.OPEN_PRICE_1,a.CLOSE_PRICE_1,a.HIGHEST_PRICE_1,a.LOWEST_PRICE_1,c.TURNOVER_VOL,' \
                   'c.TURNOVER_VALUE,c.DEAL_AMOUNT,c.CHG_PCT,c.NEG_MARKET_VALUE,c.MARKET_VALUE, c.TURNOVER_RATE ' \
                   'from mkt_equd_adj a,md_security b,mkt_equd c ' \
                   'where a.SECURITY_ID=b.SECURITY_ID ' \
                   'and a.TICKER_SYMBOL=c.TICKER_SYMBOL ' \
                   'and a.TRADE_DATE=c.TRADE_DATE ' \
                   'and a.TRADE_DATE>=%s ' \
                   'and a.TRADE_DATE<=%s and a.HIGHEST_PRICE_1!=0 and a.TICKER_SYMBOL=%s'

        if not self.end_timestamp:
            prev_trade_date = self.get_prev_trade_date(self.tonglian_conn, now_date)
            df = pd.read_sql(sql_bars, self.tonglian_conn, params=(prev_trade_date, now_date,entity.code))

        else:
            end_timestamp = to_time_str(self.end_timestamp)
            prev_end_timestamp = self.get_prev_trade_date(self.tonglian_conn, end_timestamp)
            df = pd.read_sql(sql_bars, self.tonglian_conn, params=(prev_end_timestamp, end_timestamp,entity.code))
        if pd_is_not_null(df):
            df.rename(columns={'TICKER_SYMBOL': 'code',
                               'TRADE_DATE': 'timestamp',
                               'OPEN_PRICE_1': 'open',    #开盘价
                               'CLOSE_PRICE_1': 'close',   #收盘价
                               'HIGHEST_PRICE_1': 'high', #最高价
                               'LOWEST_PRICE_1': 'low',  #最低价
                               'TURNOVER_VOL': 'volume',    #成交量
                               'TURNOVER_VALUE': 'money',    #成交额
                               'SEC_SHORT_NAME': 'name',    #证券名称
                               }, inplace=True)
            df = df[['timestamp', 'code','name','open', 'close', 'low', 'high', 'volume', 'money']]
            df['entity_id'] = entity.id
            df['timestamp'] = pd.to_datetime(df['timestamp'])
            df['provider'] = 'tonglian'
            df['level'] = self.level.value
            def generate_kdata_id(se):
                if self.level >= IntervalLevel.LEVEL_1DAY:
                    return "{}_{}".format(se['entity_id'], to_time_str(se['timestamp'], fmt=TIME_FORMAT_DAY))
                else:
                    return "{}_{}".format(se['entity_id'], to_time_str(se['timestamp'], fmt=TIME_FORMAT_ISO8601))
            df['id'] = df[['entity_id', 'timestamp']].apply(generate_kdata_id, axis=1)
            df_to_db(df=df, data_schema=self.data_schema, provider=self.provider, force_update=self.force_update)

        return None
Esempio n. 15
0
def test_000001_income_statement():
    correct_timestamps = [
        '2018-09-30', '2018-06-30', '2018-03-31', '2017-12-31', '2017-09-30',
        '2017-06-30', '2017-03-31', '2016-12-31', '2016-09-30', '2016-06-30',
        '2016-03-31', '2015-12-31', '2015-09-30', '2015-06-30', '2015-03-31',
        '2014-12-31', '2014-09-30', '2014-06-30', '2014-03-31', '2013-12-31',
        '2013-09-30', '2013-06-30', '2013-03-31', '2012-12-31', '2012-09-30',
        '2012-06-30', '2012-03-31', '2011-12-31', '2011-09-30', '2011-06-30',
        '2011-03-31', '2010-12-31', '2010-09-30', '2010-06-30', '2010-03-31',
        '2009-12-31', '2009-09-30', '2009-06-30', '2009-03-31', '2008-12-31',
        '2008-09-30', '2008-06-30', '2008-03-31', '2007-12-31', '2007-09-30',
        '2007-06-30', '2007-03-31', '2006-12-31', '2006-09-30', '2006-06-30',
        '2006-03-31', '2005-12-31', '2005-09-30', '2005-06-30', '2005-03-31',
        '2004-12-31', '2004-09-30', '2004-06-30', '2004-03-31', '2003-12-31',
        '2003-09-30', '2003-06-30', '2003-03-31', '2002-12-31', '2002-09-30',
        '2002-06-30', '2002-03-31', '2001-12-31', '2001-09-30', '2001-06-30',
        '2001-03-31', '2000-12-31', '2000-06-30', '1999-12-31', '1999-06-30',
        '1998-12-31', '1998-06-30', '1997-12-31', '1997-06-30', '1996-12-31',
        '1996-06-30', '1995-12-31', '1995-06-30', '1994-12-31', '1994-06-30',
        '1993-12-31', '1993-06-30', '1992-12-31', '1991-12-31', '1990-12-31',
        '1989-12-31'
    ]
    result = IncomeStatement.query_data(
        region=Region.CHN,
        session=session,
        provider=Provider.EastMoney,
        return_type='domain',
        codes=['000001'],
        end_timestamp='2018-12-30',
        order=IncomeStatement.report_date.desc(),
        time_field='report_date')
    assert len(correct_timestamps) == len(result)
    timestamps = [to_time_str(item.report_date) for item in result]
    assert set(correct_timestamps) == set(timestamps)
    latest: IncomeStatement = result[0]

    assert latest.operating_income == 86660000000
    assert latest.fi_net_interest_income == 54530000000
    assert latest.fi_interest_income == 121700000000
    assert latest.fi_interest_expenses == 67130000000
    assert latest.fi_net_incomes_from_fees_and_commissions == 23710000000
    assert latest.fi_incomes_from_fees_and_commissions == 28920000000
    assert latest.fi_expenses_for_fees_and_commissions == 5218000000
    assert latest.investment_income == 7099000000
    assert latest.fi_income_from_fair_value_change == 1047000000
    assert latest.fi_income_from_exchange == -40000000
    assert latest.fi_other_income == 144000000

    assert latest.operating_costs == 26430000000
    assert latest.business_taxes_and_surcharges == 847000000
    assert latest.fi_operate_and_manage_expenses == 25580000000

    assert latest.operating_profit == 26610000000
    assert latest.non_operating_income == 14000000
    assert latest.non_operating_costs == 62000000

    assert latest.total_profits == 26570000000
Esempio n. 16
0
    def record(self, entity, start, end, size, timestamps):
        df = pd.DataFrame()
        dates = get_trade_days(start_date=start)
        self.logger.info(f'add dates:{dates}')
        df['timestamp'] = pd.to_datetime(dates)
        df['id'] = [to_time_str(date) for date in dates]
        df['entity_id'] = 'stock_sz_000001'

        df_to_db(df=df, data_schema=self.data_schema, provider=self.provider, force_update=self.force_update)
Esempio n. 17
0
def is_in_finished_timestamps(security_type, exchange, timestamp, level: TradingLevel):
    if type(security_type) == str:
        security_type = SecurityType(security_type)

    if security_type == SecurityType.stock and exchange in ('sh', 'sz'):
        if to_time_str(timestamp, fmt=TIME_FORMAT_MINUTE1) in china_stock_level_map_finished_timestamps.get(
                level.value):
            return True
    return False
Esempio n. 18
0
def test_000778_income_statement():
    correct_timestamps = ['2018-09-30', '2018-06-30', '2018-03-31', '2017-12-31', '2017-09-30', '2017-06-30',
                          '2017-03-31', '2016-12-31', '2016-09-30', '2016-06-30', '2016-03-31', '2015-12-31',
                          '2015-09-30', '2015-06-30', '2015-03-31', '2014-12-31', '2014-09-30', '2014-06-30',
                          '2014-03-31', '2013-12-31', '2013-09-30', '2013-06-30', '2013-03-31', '2012-12-31',
                          '2012-09-30', '2012-06-30', '2012-03-31', '2011-12-31', '2011-09-30', '2011-06-30',
                          '2011-03-31', '2010-12-31', '2010-09-30', '2010-06-30', '2010-03-31', '2009-12-31',
                          '2009-09-30', '2009-06-30', '2009-03-31', '2008-12-31', '2008-09-30', '2008-06-30',
                          '2008-03-31', '2007-12-31', '2007-09-30', '2007-06-30', '2007-03-31', '2006-12-31',
                          '2006-09-30', '2006-06-30', '2006-03-31', '2005-12-31', '2005-09-30', '2005-06-30',
                          '2005-03-31', '2004-12-31', '2004-09-30', '2004-06-30', '2004-03-31', '2003-12-31',
                          '2003-09-30', '2003-06-30', '2003-03-31', '2002-12-31', '2002-09-30', '2002-06-30',
                          '2002-03-31', '2001-12-31', '2001-06-30', '2000-12-31', '2000-06-30', '1999-12-31',
                          '1999-06-30', '1998-12-31', '1998-06-30', '1997-12-31', '1997-06-30', '1996-12-31',
                          '1995-12-31', '1994-12-31']
    result = fundamental.get_income_statement(session=session, provider=Provider.EASTMONEY, return_type='domain',
                                              codes=['000778'], end_timestamp='2018-12-30',
                                              order=IncomeStatement.timestamp.desc())
    assert len(correct_timestamps) == len(result)
    timestamps = [to_time_str(item.report_date) for item in result]
    assert set(correct_timestamps) == set(timestamps)
    latest: IncomeStatement = result[0]

    assert latest.operating_income == 31710000000
    assert latest.total_operating_costs == 29230000000
    assert latest.operating_costs == 26220000000
    assert latest.rd_costs == 185500000
    assert latest.net_change_in_insurance_contract_reserves == 0
    assert latest.business_taxes_and_surcharges == 359700000
    assert latest.sales_costs == 771400000
    assert latest.managing_costs == 472900000
    assert latest.financing_costs == 397500000
    assert latest.assets_devaluation == 824400000
    assert latest.investment_income == 104100000
    assert latest.investment_income_from_related_enterprise == 61290000

    assert latest.operating_profit == 2637000000
    assert latest.non_operating_income == 38340000
    assert latest.non_operating_costs == 221700000

    assert latest.total_profits == 2454000000
    assert latest.tax_expense == 579600000
    assert latest.net_profit == 1874000000
    assert latest.net_profit_as_parent == 1811000000
    assert latest.net_profit_as_minority_interest == 63570000
    assert latest.deducted_net_profit == 1897000000

    assert latest.eps == 0.4537
    assert latest.diluted_eps == 0.4537

    assert latest.other_comprehensive_income == -521000000
    assert latest.other_comprehensive_income_as_parent == -522400000
    assert latest.other_comprehensive_income_as_minority_interest == 1403000
    assert latest.total_comprehensive_income == 1353000000
    assert latest.total_comprehensive_income_as_parent == 1288000000
    assert latest.total_comprehensive_income_as_minority_interest == 64980000
Esempio n. 19
0
def test_000001_finance_factor():
    correct_timestamps = ['2018-09-30', '2018-06-30', '2018-03-31', '2017-12-31', '2017-09-30', '2017-06-30',
                          '2017-03-31', '2016-12-31', '2016-09-30', '2016-06-30', '2016-03-31', '2015-12-31',
                          '2015-09-30', '2015-06-30', '2015-03-31', '2014-12-31', '2014-09-30', '2014-06-30',
                          '2014-03-31', '2013-12-31', '2013-09-30', '2013-06-30', '2013-03-31', '2012-12-31',
                          '2012-09-30', '2012-06-30', '2012-03-31', '2011-12-31', '2011-09-30', '2011-06-30',
                          '2011-03-31', '2010-12-31', '2010-09-30', '2010-06-30', '2010-03-31', '2009-12-31',
                          '2009-09-30', '2009-06-30', '2009-03-31', '2008-12-31', '2008-09-30', '2008-06-30',
                          '2008-03-31', '2007-12-31', '2007-09-30', '2007-06-30', '2007-03-31', '2006-12-31',
                          '2006-09-30', '2006-06-30', '2006-03-31', '2005-12-31', '2005-09-30', '2005-06-30',
                          '2005-03-31', '2004-12-31', '2004-09-30', '2004-06-30', '2004-03-31', '2003-12-31',
                          '2003-09-30', '2003-06-30', '2003-03-31', '2002-12-31', '2002-09-30', '2002-06-30',
                          '2002-03-31', '2001-12-31', '2001-09-30', '2001-06-30', '2001-03-31', '2000-12-31',
                          '2000-06-30', '1999-12-31', '1999-06-30', '1998-12-31', '1998-06-30', '1997-12-31',
                          '1997-06-30', '1996-12-31', '1996-06-30', '1995-12-31', '1995-06-30', '1994-12-31',
                          '1994-06-30', '1993-12-31', '1993-06-30', '1992-12-31', '1991-12-31', '1990-12-31',
                          '1989-12-31']
    result = fundamental.get_finance_factor(session=session, provider=Provider.EASTMONEY, return_type='domain',
                                            codes=['000001'], end_timestamp='2018-12-30',
                                            order=FinanceFactor.timestamp.desc())
    assert len(correct_timestamps) == len(result)
    timestamps = [to_time_str(item.report_date) for item in result]
    assert set(correct_timestamps) == set(timestamps)
    latest: FinanceFactor = result[0]
    assert latest.basic_eps == 1.14
    assert latest.deducted_eps == 1.13
    assert latest.diluted_eps == 1.14
    assert latest.bps == 12.538
    assert latest.capital_reserve_ps == 3.2886
    assert latest.undistributed_profit_ps == 5.3566
    assert latest.op_cash_flow_ps == -0.6587

    assert latest.total_op_income == 86660000000
    assert latest.net_profit == 20460000000
    assert latest.deducted_net_profit == 20350000000
    assert latest.op_income_growth_yoy == 0.0856
    assert latest.net_profit_growth_yoy == 0.068
    assert latest.deducted_net_profit_growth_yoy == 0.0636
    assert latest.op_income_growth_qoq == 0.0336
    assert latest.net_profit_growth_qoq == 0.0202
    assert latest.deducted_net_profit_growth_qoq == 0.0168

    assert latest.roe == 0.0948
    assert latest.deducted_roe == 0.0943
    assert latest.rota == 0.0062
    assert latest.net_margin == 0.2360

    assert latest.debt_asset_ratio == 0.9298
    assert latest.em == 14.25
    assert latest.equity_ratio == 13.25

    assert latest.fi_total_deposit == 2130000000000
    assert latest.fi_total_loan == 1920000000000
    assert latest.fi_loan_deposit_ratio == 0.9004
    assert latest.fi_npl_ratio == 0.0168
    assert latest.fi_npl_provision_coverage == 1.6914
Esempio n. 20
0
def china_stock_finished_timestamp(timestamp: pd.Timestamp,
                                   level: TradingLevel):
    timestamp = to_pd_timestamp(timestamp)

    if timestamp.microsecond != 0:
        return False

    return to_time_str(timestamp, fmt=TIME_FORMAT_MINUTE1
                       ) in china_stock_level_map_finished_timestamps.get(
                           level.value)
Esempio n. 21
0
    def request(self, url=None, method='get', param=None, path_fields=None):
        security_item = param['security_item']
        start_timestamp = param['start_timestamp']
        size = param['size']
        ccxt_level = param['ccxt_level']
        level = param['level']
        ccxt_account: CCXTAccount = param['ccxt_account']

        ccxt_exchange = ccxt_account.get_ccxt_exchange(security_item.exchange)

        if ccxt_exchange.has['fetchOHLCV']:
            limit = ccxt_account.get_kdata_limit(security_item.exchange)

            limit = min(size, limit)

            kdata_list = []

            try:
                if ccxt_account.exchange_conf[
                        security_item.exchange]['support_since']:
                    kdatas = ccxt_exchange.fetch_ohlcv(security_item.code,
                                                       timeframe=ccxt_level,
                                                       since=start_timestamp)
                else:
                    kdatas = ccxt_exchange.fetch_ohlcv(security_item.code,
                                                       timeframe=ccxt_level,
                                                       limit=limit)

                # always ignore the latest one,because it's not finished
                for kdata in kdatas[0:-1]:
                    current_timestamp = kdata[0]
                    if level == TradingLevel.LEVEL_1DAY:
                        current_timestamp = to_time_str(current_timestamp)

                    kdata_json = {
                        'timestamp': to_pd_timestamp(current_timestamp),
                        'open': kdata[1],
                        'high': kdata[2],
                        'low': kdata[3],
                        'close': kdata[4],
                        'volume': kdata[5],
                        'name': security_item.name,
                        'provider': 'ccxt',
                        'level': level.value
                    }
                    kdata_list.append(kdata_json)

                return kdata_list
            except Exception as e:
                logger.exception("record_kdata for security:{} failed".format(
                    security_item.id))
        else:
            logger.warning("exchange:{} not support fetchOHLCV".format(
                security_item.exchange))
    def record(self, entity, start, end, size, timestamps, http_session):
        if self.end_timestamp:
            end_timestamp = to_time_str(self.end_timestamp)
            df = yh_get_bars(code=entity.code, interval=self.yahoo_trading_level, start=start, end=end_timestamp, actions=False)
        else:
            end_timestamp = None
            df = yh_get_bars(code=entity.code, interval=self.yahoo_trading_level, start=start, actions=False)

        if pd_is_not_null(df):
            return df
        return None
Esempio n. 23
0
 def default(self, object):
     if isinstance(object, pd.Series):
         return object.to_dict()
     elif isinstance(object, pd.Timestamp):
         return to_time_str(object, fmt=TIME_FORMAT_ISO8601)
     elif isinstance(object, Enum):
         return object.value
     elif isinstance(object, Bean):
         return object.dict()
     else:
         return super().default(object)
Esempio n. 24
0
def generate_finished_timestamps(security_type, exchange, level):
    return [
        to_time_str(timestamp, fmt=TIME_FORMAT_MINUTE1)
        for timestamp in iterate_timestamps(security_type=security_type,
                                            exchange=exchange,
                                            start_timestamp='1999-01-01',
                                            end_timestamp='1999-01-01',
                                            level=level,
                                            contain_all_timestamp=False,
                                            kdata_use_begin_time=False)
    ]
Esempio n. 25
0
    def run(self):
        from zvt.api import get_kdata
        bond_data = get_kdata(entity_id='bond_cn_EMM00166466')
        now_date = to_time_str(now_pd_timestamp())
        if bond_data.empty:
            # 初始时间定在2007年
            start = '2007-01-01'
        else:
            start = to_time_str(bond_data.timestamp.max())
        # EMM00166466 中债国债到期收益率:10年
        df = c.edb(
            "EMM00166466",
            f"IsLatest=0,StartDate={start},EndDate={now_date},ispandas=1")

        if pd_is_not_null(df):
            df['name'] = "中债国债到期收益率:10年"
            df.rename(columns={
                'RESULT': 'data_value',
                'DATES': 'timestamp'
            },
                      inplace=True)

            df['entity_id'] = 'bond_cn_EMM00166466'
            df['timestamp'] = pd.to_datetime(df['timestamp'])
            df['provider'] = 'emquantapi'
            df['exchange'] = 'cn'
            df['level'] = '1d'
            df['code'] = "EMM00166466"

            def generate_kdata_id(se):
                return "{}_{}".format(
                    se['entity_id'],
                    to_time_str(se['timestamp'], fmt=TIME_FORMAT_DAY))

            df['id'] = df[['entity_id', 'timestamp']].apply(generate_kdata_id,
                                                            axis=1)

            df_to_db(df=df,
                     data_schema=self.data_schema,
                     provider=self.provider,
                     force_update=self.force_update)
    def record(self, entity, start, end, size, timestamps):
        # if entity.exchange == "swl1":
        #     return None
        if not end:
            if (now_pd_timestamp() - start).days > 365:
                from datetime import timedelta
                end = to_time_str(start + timedelta(days=365))
            else:
                end = to_time_str(now_pd_timestamp())
        start = to_time_str(start)

        df = c.csd(f"{entity.code}.SWI", "OPEN,CLOSE,HIGH,LOW,VOLUME,AMOUNT", start, end,
                   "period=1,adjustflag=1,curtype=1,order=1,ispandas=1")
        if type(df) != pd.DataFrame:
            return None
        df.rename(columns={
            'DATES': 'timestamp',
            'OPEN': 'open',
            'CLOSE': 'close',
            'HIGH': 'high',
            'LOW': 'low',
            'VOLUME': 'volume',
            'AMOUNT': 'turnover',
        }, inplace=True)

        if pd_is_not_null(df):
            df['name'] = entity.name
            df['entity_id'] = entity.id
            df['timestamp'] = pd.to_datetime(df['timestamp'])
            df['provider'] = 'joinquant'
            df['level'] = '1d'
            df['code'] = entity.code

            def generate_kdata_id(se):
                return "{}_{}".format(se['entity_id'], to_time_str(se['timestamp'], fmt=TIME_FORMAT_DAY))

            df['id'] = df[['entity_id', 'timestamp']].apply(generate_kdata_id, axis=1)

            df_to_db(df=df, data_schema=self.data_schema, provider=self.provider, force_update=self.force_update)

        return None
Esempio n. 27
0
    def fetch_cumulative_net_value(self, security_item, start, end,
                                   http_session) -> pd.DataFrame:
        query_url = 'http://api.fund.eastmoney.com/f10/lsjz?' \
                    'fundCode={}&pageIndex={}&pageSize=200&startDate={}&endDate={}'

        page = 1
        df = pd.DataFrame()
        while True:
            url = query_url.format(security_item.code, page,
                                   to_time_str(start), to_time_str(end))
            text = sync_get(http_session,
                            url,
                            headers=EASTMONEY_ETF_NET_VALUE_HEADER,
                            return_type='text')
            if text is None:
                break

            try:
                response_json = demjson.decode(text)
                response_df = pd.DataFrame(response_json['Data']['LSJZList'])
            except:
                break

            # 最后一页
            if not pd_is_not_null(response_df):
                break

            response_df['FSRQ'] = pd.to_datetime(response_df['FSRQ'])
            response_df['JZZZL'] = pd.to_numeric(response_df['JZZZL'],
                                                 errors='coerce')
            response_df['LJJZ'] = pd.to_numeric(response_df['LJJZ'],
                                                errors='coerce')
            response_df = response_df.fillna(0)
            response_df.set_index('FSRQ', inplace=True, drop=True)

            df = pd.concat([df, response_df])
            page += 1

            self.sleep()

        return df
Esempio n. 28
0
    def persist(self, ref_record, df_record, entity_item, http_session):
        saved_counts = 0
        entity_finished = False

        if pd_is_not_null(df_record):
            assert 'id' in df_record.columns
            saved_counts = df_to_db(df=df_record,
                                    ref_df=ref_record,
                                    region=self.region,
                                    data_schema=self.data_schema,
                                    provider=self.provider,
                                    fix_duplicate_way=self.fix_duplicate_way)
            if saved_counts == 0:
                entity_finished = True

        # could not get more data
        else:
            # not realtime
            if not self.real_time:
                entity_finished = True

            # realtime and to the close time
            elif (self.close_hour is not None) and (self.close_minute is not None):
                now = now_pd_timestamp(self.region)
                if now.hour >= self.close_hour:
                    if now.minute - self.close_minute >= 5:
                        self.logger.info(f'{entity_item.id} now is the close time: {now}')
                        entity_finished = True

        if isinstance(self, FixedCycleDataRecorder):
            entity_finished = True

        start_timestamp = to_time_str(df_record['timestamp'].min(axis=0))
        end_timestamp = to_time_str(df_record['timestamp'].max(axis=0))

        self.result = [saved_counts, start_timestamp, end_timestamp]

        # add finished entity to finished_items
        if entity_finished:
            self.on_finish_entity(entity_item, http_session)
        return entity_finished, saved_counts
Esempio n. 29
0
    def generate_request_param(self, security_item, start, end, size,
                               timestamp):
        if self.start_timestamp:
            start = max(self.start_timestamp, to_pd_timestamp(start))

        return {
            'security_item': security_item,
            'start_timestamp': to_time_str(start),
            'size': size,
            'level': self.level,
            'ccxt_level': self.ccxt_trading_level
        }
Esempio n. 30
0
    def generate_request_param(self, security_item, start, end, size,
                               timestamp):
        if self.start_timestamp:
            start = max(self.start_timestamp, to_pd_timestamp(start))
        if size < 20:
            size = 20

        return {
            'security_item': security_item,
            'start_timestamp': to_time_str(start),
            'size': size
        }