def cash_flow_statement_to_es(force=False): es_index_mapping('cash_flow_statement', CashFlowStatement) for _, security_item in get_security_list().iterrows(): try: start_date = None if not force: query = { "term": {"securityId": ""} } query["term"]["securityId"] = security_item["id"] latest_record = es_get_latest_record(index='cash_flow_statement', time_field='reportDate', query=query) logger.info("latest_record:{}".format(latest_record)) if latest_record: start_date = latest_record['reportDate'] actions = [] for json_object in get_cash_flow_statement_items(security_item, start_date=start_date): if start_date and is_same_date(start_date, json_object['reportDate']): continue cash_flow_statement = CashFlowStatement(meta={'id': json_object['id']}) fill_doc_type(cash_flow_statement, json_object) # cash_flow_statement.save() actions.append(cash_flow_statement.to_dict(include_meta=True)) if actions: resp = elasticsearch.helpers.bulk(es, actions) logger.info(resp) except Exception as e: logger.warn("wrong CashFlowStatement:{},error:{}", security_item, e)
def cash_flow_statement_to_es(force=False): es_index_mapping('cash_flow_statement', CashFlowStatement) for _, security_item in get_security_list().iterrows(): try: start_date = None if not force: query = {"term": {"securityId": ""}} query["term"]["securityId"] = security_item["id"] latest_record = es_get_latest_record( index='cash_flow_statement', time_field='reportDate', query=query) logger.info("latest_record:{}".format(latest_record)) if latest_record: start_date = latest_record['reportDate'] for json_object in get_cash_flow_statement_items( security_item, start_date=start_date): if start_date and is_same_date(start_date, json_object['reportDate']): continue cash_flow_statement = CashFlowStatement( meta={'id': json_object['id']}) fill_doc_type(cash_flow_statement, json_object) cash_flow_statement.save() except Exception as e: logger.warn("wrong CashFlowStatement:{},error:{}", security_item, e)
def get_cash_flow_statement_items(security_item, start_date=None, report_period=None, report_event_date=None, return_type='json'): """ get cash flow statement items. Parameters ---------- security_item : SecurityItem or str the security item,id or code start_date : TimeStamp str or TimeStamp start date report_period : TimeStamp str or TimeStamp the finance report period,eg.'20170331' report_event_date : TimeStamp str or TimeStamp the finance report published date return_type : str {'json','doc'},default: 'json' Returns ------- list of CashFlowStatement list of json """ security_item = to_security_item(security_item) path = get_cash_flow_statement_path(security_item) if not os.path.exists(path): return [] encoding = 'GB2312' with open(path, encoding=encoding) as fr: lines = fr.readlines() # for idx, line in enumerate(lines): # yield idx, line.split() reportDate = lines[0].split()[1:-1] # /*一、经营活动产生的现金流量*/ # 销售商品、提供劳务收到的现金 cashFromSellingCommoditiesOrOfferingLabor = lines[3].split()[1:-1] # 收到的税费返还 refundOfTaxAndFeeReceived = lines[4].split()[1:-1] # 收到的其他与经营活动有关的现金 cashReceivedRelatingToOtherOperatingActivities = lines[5].split()[1:-1] # 经营活动现金流入小计 subTotalOfCashInflowsFromOperatingActivities = lines[6].split()[1:-1] # 购买商品、接受劳务支付的现金 cashPaidForGoodsAndServices = lines[7].split()[1:-1] # 支付给职工以及为职工支付的现金 cashPaidToAndOnBehalfOfemployees = lines[8].split()[1:-1] # 支付的各项税费 paymentsOfTaxesAndSurcharges = lines[9].split()[1:-1] # 支付的其他与经营活动有关的现金 cashPaidRelatingToOtherOperatingActivities = lines[10].split()[1:-1] # 经营活动现金流出小计 subTotalOfCashOutflowsFromOperatingActivities = lines[11].split()[1:-1] # 经营活动产生的现金流量净额 netCashFlowsFromOperatingActivities = lines[12].split()[1:-1] # /*二、投资活动产生的现金流量*/ # 收回投资所收到的现金 cashReceivedFromDisposalOfInvestments = lines[14].split()[1:-1] # 取得投资收益所收到的现金 cashReceivedFromReturnsOnIvestments = lines[15].split()[1:-1] # 处置固定资产、无形资产和其他长期资产所收回的现金净额 netCashReceivedFromDisposalAssets = lines[16].split()[1:-1] # 处置子公司及其他营业单位收到的现金净额 netCashReceivedFromDisposalSubsidiaries = lines[17].split()[1:-1] # 收到的其他与投资活动有关的现金 cashReceivedFromOtherInvesting = lines[18].split()[1:-1] # 投资活动现金流入小计 subTotalOfCashInflowsFromInvesting = lines[19].split()[1:-1] # 购建固定资产、无形资产和其他长期资产所支付的现金 cashPaidToAcquireFixedAssets = lines[20].split()[1:-1] # 投资所支付的现金 cashPaidToAcquireInvestments = lines[21].split()[1:-1] # 取得子公司及其他营业单位支付的现金净额 netCashPaidToAcquireSubsidiaries = lines[22].split()[1:-1] # 支付的其他与投资活动有关的现金 cashPaidRelatingToOtherInvesting = lines[23].split()[1:-1] # 投资活动现金流出小计 subTotalOfCashOutflowsFromInvesting = lines[24].split()[1:-1] # 投资活动产生的现金流量净额 netCashFlowsFromInvesting = lines[25].split()[1:-1] # /*三、筹资活动产生的现金流量*/ # 吸收投资收到的现金 cashReceivedFromCapitalContributions = lines[27].split()[1:-1] # 其中:子公司吸收少数股东投资收到的现金 cashReceivedFromMinorityShareholdersOfSubsidiaries = lines[28].split( )[1:-1] # 取得借款收到的现金 cashReceivedFromBorrowings = lines[29].split()[1:-1] # 发行债券收到的现金 cashReceivedFromIssuingBonds = lines[30].split()[1:-1] # 收到其他与筹资活动有关的现金 cashReceivedRelatingToOtherFinancingActivities = lines[31].split( )[1:-1] # 筹资活动现金流入小计 subTotalOfCashInflowsFromFinancingActivities = lines[32].split()[1:-1] # 偿还债务支付的现金 cashRepaymentsOfBorrowings = lines[33].split()[1:-1] # 分配股利、利润或偿付利息所支付的现金 cashPaymentsForInterestExpensesAndDistributionOfDividendsOrProfits = lines[ 34].split()[1:-1] # 其中:子公司支付给少数股东的股利、利润 cashPaymentsForDividendsOrProfitToMinorityShareholders = lines[ 35].split()[1:-1] # 支付其他与筹资活动有关的现金 cashPaymentsRelatingToOtherFinancingActivities = lines[36].split( )[1:-1] # 筹资活动现金流出小计 subTotalOfCashOutflowsFromFinancingActivities = lines[37].split()[1:-1] # 筹资活动产生的现金流量净额 netCashFlowsFromFinancingActivities = lines[38].split()[1:-1] # /*四、汇率变动对现金及现金等价物的影响*/ effectOfForeignExchangeRate = lines[39].split()[1:-1] # /*五、现金及现金等价物净增加额*/ netIncreaseInCash = lines[40].split()[1:-1] # 加:期初现金及现金等价物余额 cashAtBeginningOfyear = lines[41].split()[1:-1] # /*六、期末现金及现金等价物余额*/ cashAtEndOfyear = lines[42].split()[1:-1] # /*附注*/ # 净利润 netProfit = lines[44].split()[1:-1] # 少数股东权益 minorityBookValue = lines[45].split()[1:-1] # 未确认的投资损失 unrealisedInvestmentLosses = lines[46].split()[1:-1] # 资产减值准备 allowanceForAssetDevaluation = lines[47].split()[1:-1] # 固定资产折旧、油气资产折耗、生产性物资折旧 depreciationOfFixedAssets = lines[48].split()[1:-1] # 无形资产摊销 amorizationOfIntangibleAssets = lines[49].split()[1:-1] # 长期待摊费用摊销 longTermDeferredExpenses = lines[50].split()[1:-1] # 待摊费用的减少 decreaseOfDeferredExpenses = lines[51].split()[1:-1] # 预提费用的增加 IncreaseOfwithholdingExpenses = lines[52].split()[1:-1] # 处置固定资产、无形资产和其他长期资产的损失 lossOnDisposalOfFixedAssets = lines[53].split()[1:-1] # 固定资产报废损失 lossOnFixedAssetsDamaged = lines[54].split()[1:-1] # 公允价值变动损失 lossOnFairValueChange = lines[55].split()[1:-1] # 递延收益增加(减:减少) changeOnDeferredRevenue = lines[56].split()[1:-1] # 预计负债 estimatedLiabilities = lines[57].split()[1:-1] # 财务费用 financingExpenses = lines[58].split()[1:-1] # 投资损失 investmentLoss = lines[59].split()[1:-1] # 递延所得税资产减少 decreaseOnDeferredIncomeTaxAssets = lines[60].split()[1:-1] # 递延所得税负债增加 increaseOnDeferredIncomeTaxLiabilities = lines[61].split()[1:-1] # 存货的减少 decreaseInInventories = lines[62].split()[1:-1] # 经营性应收项目的减少 decreaseInReceivablesUnderOperatingActivities = lines[63].split()[1:-1] # 经营性应付项目的增加 increaseInReceivablesUnderOperatingActivities = lines[64].split()[1:-1] # 已完工尚未结算款的减少(减:增加) decreaseOnAmountDue = lines[65].split()[1:-1] # 已结算尚未完工款的增加(减:减少) increaseOnSettlementNotYetCompleted = lines[66].split()[1:-1] # 其他 other = lines[67].split()[1:-1] # 经营活动产生现金流量净额 netCashFlowFromOperatingActivities = lines[68].split()[1:-1] # 债务转为资本 debtsTransferToCapital = lines[69].split()[1:-1] # 一年内到期的可转换公司债券 oneYearDueConvertibleBonds = lines[70].split()[1:-1] # 融资租入固定资产 financingRentToFixedAsset = lines[71].split()[1:-1] # 现金的期末余额 cashAtTheEndOfPeriod = lines[72].split()[1:-1] # 现金的期初余额 cashAtTheBeginningOfPeriod = lines[73].split()[1:-1] # 现金等价物的期末余额 cashEquivalentsAtTheEndOfPeriod = lines[74].split()[1:-1] # 现金等价物的期初余额 cashEquivalentsAtTheBeginningOfPeriod = lines[75].split()[1:-1] # 现金及现金等价物的净增加额 netIncreaseInCashAndCashEquivalents = lines[76].split()[1:-1] result_list = [] for idx, _ in enumerate(reportDate): if start_date: if pd.Timestamp(reportDate[idx]) < pd.Timestamp(start_date): continue if report_period and not is_same_date(report_period, reportDate[idx]): continue reportEventDate = get_report_event_date( security_item, report_date=reportDate[idx]) # use report_event_date to filter the reportEventDate before it for not getting future data if report_event_date and pd.Timestamp( report_event_date) < pd.Timestamp(reportEventDate): continue the_json = { "id": '{}_{}'.format(security_item["id"], reportDate[idx]), "reportDate": to_time_str(reportDate[idx]), "reportEventDate": reportEventDate, "securityId": security_item["id"], "code": security_item["code"], # /*一、经营活动产生的现金流量*/ # 销售商品、提供劳务收到的现金 "cashFromSellingCommoditiesOrOfferingLabor": to_float(cashFromSellingCommoditiesOrOfferingLabor[idx]), # 收到的税费返还 "refundOfTaxAndFeeReceived": to_float(refundOfTaxAndFeeReceived[idx]), # 收到的其他与经营活动有关的现金 "cashReceivedRelatingToOtherOperatingActivities": to_float(cashReceivedRelatingToOtherOperatingActivities[idx]), # 经营活动现金流入小计 "subTotalOfCashInflowsFromOperatingActivities": to_float(subTotalOfCashInflowsFromOperatingActivities[idx]), # 购买商品、接受劳务支付的现金 "cashPaidForGoodsAndServices": to_float(cashPaidForGoodsAndServices[idx]), # 支付给职工以及为职工支付的现金 "cashPaidToAndOnBehalfOfemployees": to_float(cashPaidToAndOnBehalfOfemployees[idx]), # 支付的各项税费 "paymentsOfTaxesAndSurcharges": to_float(paymentsOfTaxesAndSurcharges[idx]), # 支付的其他与经营活动有关的现金 "cashPaidRelatingToOtherOperatingActivities": to_float(cashPaidRelatingToOtherOperatingActivities[idx]), # 经营活动现金流出小计 "subTotalOfCashOutflowsFromOperatingActivities": to_float(subTotalOfCashOutflowsFromOperatingActivities[idx]), # 经营活动产生的现金流量净额 "netCashFlowsFromOperatingActivities": to_float(netCashFlowsFromOperatingActivities[idx]), # /*二、投资活动产生的现金流量*/ # 收回投资所收到的现金 "cashReceivedFromDisposalOfInvestments": to_float(cashReceivedFromDisposalOfInvestments[idx]), # 取得投资收益所收到的现金 "cashReceivedFromReturnsOnIvestments": to_float(cashReceivedFromReturnsOnIvestments[idx]), # 处置固定资产、无形资产和其他长期资产所收回的现金净额 "netCashReceivedFromDisposalAssets": to_float(netCashReceivedFromDisposalAssets[idx]), # 处置子公司及其他营业单位收到的现金净额 "netCashReceivedFromDisposalSubsidiaries": to_float(netCashReceivedFromDisposalSubsidiaries[idx]), # 收到的其他与投资活动有关的现金 "cashReceivedFromOtherInvesting": to_float(cashReceivedFromOtherInvesting[idx]), # 投资活动现金流入小计 "subTotalOfCashInflowsFromInvesting": to_float(subTotalOfCashInflowsFromInvesting[idx]), # 购建固定资产、无形资产和其他长期资产所支付的现金 "cashPaidToAcquireFixedAssets": to_float(cashPaidToAcquireFixedAssets[idx]), # 投资所支付的现金 "cashPaidToAcquireInvestments": to_float(cashPaidToAcquireInvestments[idx]), # 取得子公司及其他营业单位支付的现金净额 "netCashPaidToAcquireSubsidiaries": to_float(netCashPaidToAcquireSubsidiaries[idx]), # 支付的其他与投资活动有关的现金 "cashPaidRelatingToOtherInvesting": to_float(cashPaidRelatingToOtherInvesting[idx]), # 投资活动现金流出小计 "subTotalOfCashOutflowsFromInvesting": to_float(subTotalOfCashOutflowsFromInvesting[idx]), # 投资活动产生的现金流量净额 "netCashFlowsFromInvesting": to_float(netCashFlowsFromInvesting[idx]), # /*三、筹资活动产生的现金流量*/ # 吸收投资收到的现金 "cashReceivedFromCapitalContributions": to_float(cashReceivedFromCapitalContributions[idx]), # 其中:子公司吸收少数股东投资收到的现金 "cashReceivedFromMinorityShareholdersOfSubsidiaries": cashReceivedFromMinorityShareholdersOfSubsidiaries[idx], # 取得借款收到的现金 "cashReceivedFromBorrowings": to_float(cashReceivedFromBorrowings[idx]), # 发行债券收到的现金 "cashReceivedFromIssuingBonds": to_float(cashReceivedFromIssuingBonds[idx]), # 收到其他与筹资活动有关的现金 "cashReceivedRelatingToOtherFinancingActivities": to_float(cashReceivedRelatingToOtherFinancingActivities[idx]), # 筹资活动现金流入小计 "subTotalOfCashInflowsFromFinancingActivities": to_float(subTotalOfCashInflowsFromFinancingActivities[idx]), # 偿还债务支付的现金 "cashRepaymentsOfBorrowings": to_float(cashRepaymentsOfBorrowings[idx]), # 分配股利、利润或偿付利息所支付的现金 "cashPaymentsForInterestExpensesAndDistributionOfDividendsOrProfits": cashPaymentsForInterestExpensesAndDistributionOfDividendsOrProfits[ idx], # 其中:子公司支付给少数股东的股利、利润 "cashPaymentsForDividendsOrProfitToMinorityShareholders": cashPaymentsForDividendsOrProfitToMinorityShareholders[idx], # 支付其他与筹资活动有关的现金 "cashPaymentsRelatingToOtherFinancingActivities": to_float(cashPaymentsRelatingToOtherFinancingActivities[idx]), # 筹资活动现金流出小计 "subTotalOfCashOutflowsFromFinancingActivities": to_float(subTotalOfCashOutflowsFromFinancingActivities[idx]), # 筹资活动产生的现金流量净额 "netCashFlowsFromFinancingActivities": to_float(netCashFlowsFromFinancingActivities[idx]), # /*四、汇率变动对现金及现金等价物的影响*/ "effectOfForeignExchangeRate": to_float(effectOfForeignExchangeRate[idx]), # /*五、现金及现金等价物净增加额*/ "netIncreaseInCash": to_float(netIncreaseInCash[idx]), # 加:期初现金及现金等价物余额 "cashAtBeginningOfyear": to_float(cashAtBeginningOfyear[idx]), # /*六、期末现金及现金等价物余额*/ "cashAtEndOfyear": to_float(cashAtEndOfyear[idx]), # /*附注*/ # 净利润 "netProfit": to_float(netProfit[idx]), # 少数股东权益 "minorityBookValue": to_float(minorityBookValue[idx]), # 未确认的投资损失 "unrealisedInvestmentLosses": to_float(unrealisedInvestmentLosses[idx]), # 资产减值准备 "allowanceForAssetDevaluation": to_float(allowanceForAssetDevaluation[idx]), # 固定资产折旧、油气资产折耗、生产性物资折旧 "depreciationOfFixedAssets": to_float(depreciationOfFixedAssets[idx]), # 无形资产摊销 "amorizationOfIntangibleAssets": to_float(amorizationOfIntangibleAssets[idx]), # 长期待摊费用摊销 "longTermDeferredExpenses": to_float(longTermDeferredExpenses[idx]), # 待摊费用的减少 "decreaseOfDeferredExpenses": to_float(decreaseOfDeferredExpenses[idx]), # 预提费用的增加 "IncreaseOfwithholdingExpenses": to_float(IncreaseOfwithholdingExpenses[idx]), # 处置固定资产、无形资产和其他长期资产的损失 "lossOnDisposalOfFixedAssets": to_float(lossOnDisposalOfFixedAssets[idx]), # 固定资产报废损失 "lossOnFixedAssetsDamaged": to_float(lossOnFixedAssetsDamaged[idx]), # 公允价值变动损失 "lossOnFairValueChange": to_float(lossOnFairValueChange[idx]), # 递延收益增加(减:减少) "changeOnDeferredRevenue": to_float(changeOnDeferredRevenue[idx]), # 预计负债 "estimatedLiabilities": to_float(estimatedLiabilities[idx]), # 财务费用 "financingExpenses": to_float(financingExpenses[idx]), # 投资损失 "investmentLoss": to_float(investmentLoss[idx]), # 递延所得税资产减少 "decreaseOnDeferredIncomeTaxAssets": to_float(decreaseOnDeferredIncomeTaxAssets[idx]), # 递延所得税负债增加 "increaseOnDeferredIncomeTaxLiabilities": to_float(increaseOnDeferredIncomeTaxLiabilities[idx]), # 存货的减少 "decreaseInInventories": to_float(decreaseInInventories[idx]), # 经营性应收项目的减少 "decreaseInReceivablesUnderOperatingActivities": to_float(decreaseInReceivablesUnderOperatingActivities[idx]), # 经营性应付项目的增加 "increaseInReceivablesUnderOperatingActivities": to_float(increaseInReceivablesUnderOperatingActivities[idx]), # 已完工尚未结算款的减少(减:增加) "decreaseOnAmountDue": to_float(decreaseOnAmountDue[idx]), # 已结算尚未完工款的增加(减:减少) "increaseOnSettlementNotYetCompleted": to_float(increaseOnSettlementNotYetCompleted[idx]), # 其他 "other": to_float(other[idx]), # 经营活动产生现金流量净额 "netCashFlowFromOperatingActivities": to_float(netCashFlowFromOperatingActivities[idx]), # 债务转为资本 "debtsTransferToCapital": to_float(debtsTransferToCapital[idx]), # 一年内到期的可转换公司债券 "oneYearDueConvertibleBonds": to_float(oneYearDueConvertibleBonds[idx]), # 融资租入固定资产 "financingRentToFixedAsset": to_float(financingRentToFixedAsset[idx]), # 现金的期末余额 "cashAtTheEndOfPeriod": to_float(cashAtTheEndOfPeriod[idx]), # 现金的期初余额 "cashAtTheBeginningOfPeriod": to_float(cashAtTheBeginningOfPeriod[idx]), # 现金等价物的期末余额 "cashEquivalentsAtTheEndOfPeriod": to_float(cashEquivalentsAtTheEndOfPeriod[idx]), # 现金等价物的期初余额 "cashEquivalentsAtTheBeginningOfPeriod": to_float(cashEquivalentsAtTheBeginningOfPeriod[idx]), # 现金及现金等价物的净增加额 "netIncreaseInCashAndCashEquivalents": to_float(netIncreaseInCashAndCashEquivalents[idx]) } the_data = the_json if return_type == 'doc': the_data = CashFlowStatement(meta={'id': the_json['id']}) fill_doc_type(the_data, the_json) if report_period and is_same_date(report_period, reportDate[idx]): return the_data result_list.append(the_data) if result_list: result_list = sorted(result_list, key=lambda x: pd.Timestamp(x['reportDate'])) return result_list