def cash_flow_statement_to_es(force=False): es_index_mapping('cash_flow_statement', CashFlowStatement) for _, security_item in get_security_list().iterrows(): try: start_date = None if not force: query = {"term": {"securityId": ""}} query["term"]["securityId"] = security_item["id"] latest_record = es_get_latest_record( index='cash_flow_statement', time_field='reportDate', query=query) logger.info("latest_record:{}".format(latest_record)) if latest_record: start_date = latest_record['reportDate'] for json_object in get_cash_flow_statement_items( security_item, start_date=start_date): if start_date and is_same_date(start_date, json_object['reportDate']): continue cash_flow_statement = CashFlowStatement( meta={'id': json_object['id']}) fill_doc_type(cash_flow_statement, json_object) cash_flow_statement.save() except Exception as e: logger.warn("wrong CashFlowStatement:{},error:{}", security_item, e)
def cash_flow_statement_to_es(force=False): es_index_mapping('cash_flow_statement', CashFlowStatement) for _, security_item in get_security_list().iterrows(): try: start_date = None if not force: query = { "term": {"securityId": ""} } query["term"]["securityId"] = security_item["id"] latest_record = es_get_latest_record(index='cash_flow_statement', time_field='reportDate', query=query) logger.info("latest_record:{}".format(latest_record)) if latest_record: start_date = latest_record['reportDate'] actions = [] for json_object in get_cash_flow_statement_items(security_item, start_date=start_date): if start_date and is_same_date(start_date, json_object['reportDate']): continue cash_flow_statement = CashFlowStatement(meta={'id': json_object['id']}) fill_doc_type(cash_flow_statement, json_object) # cash_flow_statement.save() actions.append(cash_flow_statement.to_dict(include_meta=True)) if actions: resp = elasticsearch.helpers.bulk(es, actions) logger.info(resp) except Exception as e: logger.warn("wrong CashFlowStatement:{},error:{}", security_item, e)
def crawl_finance_data(start_code=STOCK_START_CODE, end_code=STOCK_END_CODE): for _, security_item in get_security_list(start=start_code, end=end_code).iterrows(): try: # 先抓事件,有些后续抓取依赖事件 process_crawl(StockFinanceReportEventSpider, {"security_item": security_item}) current_report_date = get_report_date() # 资产负债表 path = get_balance_sheet_path(security_item) if not os.path.exists(path): process_crawl(StockFinanceSpider, {"security_item": security_item, "report_type": "balance_sheet"}) else: for balance_sheet_item in get_balance_sheet_items(security_item): # 当前报告期还没抓取 if balance_sheet_item['reportDate'] != current_report_date: # 报告出来了 df = event.get_finance_report_event(security_item, index='reportDate') if current_report_date in df.index: process_crawl(StockFinanceSpider, {"security_item": security_item, "report_type": "balance_sheet"}) break # 利润表 path = get_income_statement_path(security_item) if not os.path.exists(path): process_crawl(StockFinanceSpider, {"security_item": security_item, "report_type": "income_statement"}) else: for balance_sheet_item in get_income_statement_items(security_item): if balance_sheet_item['reportDate'] != current_report_date: # 报告出来了 df = event.get_finance_report_event(security_item, index='reportDate') if current_report_date in df.index: process_crawl(StockFinanceSpider, {"security_item": security_item, "report_type": "income_statement"}) break # 现金流量表 path = get_cash_flow_statement_path(security_item) if not os.path.exists(path): process_crawl(StockFinanceSpider, {"security_item": security_item, "report_type": "cash_flow"}) else: for balance_sheet_item in get_cash_flow_statement_items(security_item): if balance_sheet_item['reportDate'] != current_report_date: # 报告出来了 df = event.get_finance_report_event(security_item, index='reportDate') if current_report_date in df.index: process_crawl(StockFinanceSpider, {"security_item": security_item, "report_type": "cash_flow"}) break except Exception as e: logger.error(e)
def finance_sheet_to_es(sheet_type='balance_sheet', force=False): if sheet_type == 'balance_sheet': doc_type = BalanceSheet elif sheet_type == 'income_statement': doc_type = IncomeStatement elif sheet_type == 'cash_flow_statement': doc_type = CashFlowStatement es_index_mapping(sheet_type, doc_type) for _, security_item in get_security_list().iterrows(): try: start_date = None if not force: query = {"term": {"securityId": ""}} query["term"]["securityId"] = security_item["id"] latest_record = es_get_latest_record(index=sheet_type, time_field='reportDate', query=query) logger.info("latest_record:{}".format(latest_record)) if latest_record: start_date = latest_record['reportDate'] actions = [] items = [] if sheet_type == 'balance_sheet': items = get_balance_sheet_items(security_item, start_date=start_date) elif sheet_type == 'income_statement': items = get_income_statement_items(security_item, start_date=start_date) elif sheet_type == 'cash_flow_statement': items = get_cash_flow_statement_items(security_item, start_date=start_date) for json_object in items: if start_date and is_same_date(start_date, json_object['reportDate']): continue the_doc = doc_type(meta={'id': json_object['id']}) fill_doc_type(the_doc, json_object) # balance_sheet.save() actions.append(the_doc.to_dict(include_meta=True)) if actions: resp = elasticsearch.helpers.bulk(es_client, actions) logger.info(resp) except Exception as e: logger.warning("{} wrong {},error:{}", security_item, sheet_type, e)
def test_get_cash_flow_statement_items(): cash_flow_statements = finance.get_cash_flow_statement_items('600977') assert len(cash_flow_statements) > 0 for item in cash_flow_statements: assert item['netCashFlowsFromOperatingActivities'] > 0 assert item['reportEventDate'] > item['reportPeriod']