def update_implied_vol(start_date, end_date, force_update): """ 根据日期循环导入数到数据平台otc_position_snapshot表 :param start_date: :param end_date: :param force_update: :return: """ db_session = create_db_session() try: for index in range(0, (end_date - start_date).days + 1): report_date = (start_date + timedelta(index)) logging.info('开始获取report_date: %s的数据' % report_date) clean_position_dict = ROTCPositionService.clean_one_day_implied_vol( db_session, report_date) position_snapshots = ROTCPositionService.convert_data_to_position_snapshot( clean_position_dict) exiting_trans_codes = OTCPositionSnapshotRepo.get_trans_codes_by_report_date( db_session, report_date) ROTCPositionService.save_position_snapshot( db_session, report_date, exiting_trans_codes, position_snapshots, force_update) except Exception as e: logging.error('运行错误: %s' % e) db_session.close() raise Exception(e)
def recon_all_instrument(self, start_date, end_date, fail_fast=False, dry_run=True, active_only=True, ignore_milestone=False): logging.info("开始recon, %s %s" % (start_date.strftime('%Y%m%d'), end_date.strftime('%Y%m%d'))) # 获取所有需要Recon的标的物,只处理状态为Active的标的物 db_session = create_db_session() # 获取标的物 all_instruments = self.load_instruments(db_session, active_only) logging.info("当前需要Recon的标的物总数为: %d:" % len(all_instruments)) holidays = TradingCalendarRepo.get_holiday_list(db_session) db_session.close() # 并发进行recon with ThreadPoolExecutor(max_workers=8) as executor: # 开始并行向远程服务器请求数据 all_task = [executor.submit(self.recon_one_instrument, (item, start_date, end_date, holidays, dry_run, ignore_milestone)) for item in all_instruments] # 处理Recon结果 success_instruments = [] failed_instruments = [] for future in as_completed(all_task): instrument, status = future.result() if status is True: success_instruments.append(instrument) else: failed_instruments.append(instrument) logging.info("标的物处理结束: %s,处理结果为:%s" % (instrument.instrumentId, status)) logging.info("当前处理成功的标的物:%d of %d,失败的标的物:%d of %d" % (len(success_instruments), len(all_instruments), len(failed_instruments), len(all_instruments))) if len(failed_instruments) != 0: err_msg = "处理失败的标的物为:%s" % ','.join([instrument.instrumentId for instrument in failed_instruments]) logging.info(err_msg) if fail_fast: raise Exception(err_msg) else: logging.info("完成Recon成功")
def backfill_all_instruments(self, start_date, end_date, active_only=True, force_update=False): db_session = create_db_session() # 获取标的物 all_instruments = self.load_instruments(db_session, active_only) logging.info("当前需要Backfill的标的物总数为: %d:" % len(all_instruments)) holidays = TradingCalendarRepo.get_holiday_list(db_session) db_session.close() # 并发进行 backfiller with ThreadPoolExecutor(max_workers=8) as executor: # 开始并行向远程服务器请求数据 all_task = [executor.submit(self.backfill_one_instrument, (item, start_date, end_date, holidays, force_update)) for item in all_instruments] # 处理Backfill结果 success_instruments = [] failed_instruments = [] for future in as_completed(all_task): instrument, status = future.result() if status is True: success_instruments.append(instrument) else: failed_instruments.append(instrument) logging.info("标的物处理结束: %s,处理结果为:%s" % (instrument.instrumentId, status)) logging.info("当前处理成功的标的物:%d of %d,失败的标的物:%d of %d" % (len(success_instruments), len(all_instruments), len(failed_instruments), len(all_instruments))) if len(failed_instruments) != 0: raise Exception( "处理失败的标的物为:%s" % ','.join([instrument.instrumentId for instrument in failed_instruments]))
def recon_one_instrument(self, params): # 返回True或Fasle, 当前标的物是Clean则返回True instrument, start_date, end_date, holidays, dry_run, ignore_milestone = params db_session = create_db_session() try: logging.info("开始Recon标的物: %s" % instrument.instrumentId) # 获取标的物Milestone milestone = self.get_milestone(db_session, instrument) # 只处理标的物上市日期和退市日期之间的数据 if instrument.listedDate is not None and start_date < instrument.listedDate: start_date = instrument.listedDate if instrument.delistedDate is not None and end_date > instrument.delistedDate: end_date = instrument.delistedDate # 计算需要Recon的日期,跳过节假日、周末和特殊日期 special_dates = TradingCalendarRepo.load_special_dates(db_session, instrument) unchecked_trading_dates = self.get_unchecked_trading_dates( start_date, end_date, milestone, holidays, special_dates, ignore_milestone) # 获取标的物已经拥有的交易日 existing_trading_dates = self.get_existing_trading_dates(db_session, instrument) # 计算出缺失数据 missing_trading_dates = self.get_missing_trading_dates(instrument, unchecked_trading_dates, existing_trading_dates) # 对缺数数据进行处理 if not dry_run: self.upsert_missing_trading_dates(db_session, instrument, missing_trading_dates) # TODO 需要对多余的数据进行处理 # 计算出多余数据 extra_trading_dates = [] logging.info("Recon找到多余数据:%s -> %d 个Break" % (instrument.instrumentId, len(extra_trading_dates))) # 如果没有差值,则比较开始日期和Milestone,更新Milestone has_missing_dates = len(missing_trading_dates) != 0 has_extra_dates = len(extra_trading_dates) != 0 if (not has_missing_dates) and (not has_extra_dates) and (not dry_run): if start_date <= end_date: self.upsert_milestone(db_session, instrument, start_date, end_date, milestone) else: logging.info("标的物开始日期大于结束日期,不更新Milestone:%s [%s %s]" % ( instrument.instrumentId, start_date.strftime('%Y%m%d'), end_date.strftime('%Y%m%d'))) return instrument, (not has_missing_dates) and (not has_extra_dates) except Exception as e: logging.error("Recon失败,标的物:%s, 异常:%s" % (instrument.instrumentId, e)) return instrument, False finally: db_session.close()
def backfill_one_instrument(self, params): instrument, start_date, end_date, holidays, force_update = params logging.info("开始Backfill标的物: %s, startDate: %s, endDate: %s" % (instrument.instrumentId, start_date.strftime('%Y%m%d'), end_date.strftime('%Y%m%d'))) db_session = create_db_session() try: special_dates = TradingCalendarRepo.load_special_dates( db_session, instrument) option_close_breaks = self.get_recon_breaks( db_session, start_date, end_date, instrument) ret = self.backfill_multiple_option_close(db_session, instrument, option_close_breaks, holidays, special_dates) return instrument, ret except Exception as e: return instrument, False finally: db_session.close()
def update_eod_date(start_date_str=EODDateConfig.eod_start_date, end_date_str=DateTimeUtils.date2str(date.today()), eod_cutoff_time=EODDateConfig.eod_cutoff_time): end_date = DateTimeUtils.str2date(end_date_str) start_date = DateTimeUtils.str2date(start_date_str) # 获取节假日信息 db_session = create_db_session() # TODO: 需要从配置文件读取 if not TradingDayService.is_trading_day(db_session, end_date): # 如果当天不是交易日则直接计算上一个交易日的值 end_date = TradingDayService.go_trading_day(db_session, end_date, 1, -1) else: # 如果是交易日,小于cutoff time还是返回上一个交易日 if datetime.now().time() < DateTimeUtils.str2time(eod_cutoff_time): end_date = TradingDayService.go_trading_day(db_session, end_date, 1, -1) else: end_date = end_date # 设置变量到airflow variable AirflowVariableUtils.set_eod_start_date(start_date) AirflowVariableUtils.set_eod_end_date(end_date)
def calc_all_instrument_quote_vol_surface(observed_date): """ 从otc_option_quote表里取华泰的数据,格式化写入到vol_surface表中 :return: """ db_session = create_db_session() try: # 取所有instrument_id instrument_ids = OTCOptionQuoteRepo.get_instrument_ids_by_observe_date( db_session, observed_date) vol_surface_dbo_dict = {} for index, instrument_id in enumerate(instrument_ids): # 计算一个标的的vol_surface logger.info('准备加载的标的为: %s, 进度为: %d of %d' % (instrument_id, index + 1, len(instrument_ids))) vol_surface_dbo = VolSurfaceSpiderService.calc_one_instrument_quote_vol_surface( db_session, instrument_id, observed_date) if vol_surface_dbo is None: continue vol_surface_dbo_dict[instrument_id] = vol_surface_dbo logger.info('成功计算了标的: %s的vol_surface, 进度为: %d of %d' % (instrument_id, index + 1, len(instrument_ids))) # 获取所有的合约种类及其对应的主力合约 holidays = TradingCalendarRepo.get_holiday_list(db_session) special_dates = [] # 取上一个交易日 last_trade_date = DateTimeUtils.get_trading_day( observed_date, holidays, special_dates, 1, -1) # 获取所有的主力合约 contract_list = db_session.query(FutureContractInfo).filter( FutureContractInfo.tradeDate == last_trade_date).all() # 获取当天所有活跃的大宗期货合约 active_instrument_list = db_session.query(Instrument).filter( and_( Instrument.status == enum_dto.InstrumentStatusType.ACTIVE.name, Instrument.assetClass == enum_dto.AssetClassType.COMMODITY.name, Instrument.instrumentType == enum_dto.InstrumentType.FUTURE.name, or_(Instrument.delistedDate is None, Instrument.delistedDate > observed_date))).all() active_instrument_dict = {} for instrument in active_instrument_list: if instrument.contractType not in active_instrument_dict: active_instrument_dict[instrument.contractType] = [] active_instrument_dict[instrument.contractType].append( instrument.instrumentId) # 所有标的都使用主力合约对应的波动率曲面 vol_surface_dbos = [] for contract in contract_list: primary_contract_id = contract.primaryContractId if primary_contract_id not in vol_surface_dbo_dict: logger.warning('标的没有主力合约对应的波动率曲面:%s' % primary_contract_id) continue variety_type = contract.contractType if variety_type not in active_instrument_dict: logger.warning('主力合约没有活跃的合约:%s' % variety_type) continue active_future_list = active_instrument_dict[variety_type] for instrument_id in active_future_list: vol_surface_dbo = copy.deepcopy( vol_surface_dbo_dict[primary_contract_id]) vol_surface_dbo.uuid = uuid.uuid1() vol_surface_dbo.instrumentId = instrument_id vol_surface_dbos.append(vol_surface_dbo) # 将波动率曲面写入数据库 VolSurfaceRepo.insert_vol_surfaces(db_session, vol_surface_dbos) logger.info('爬虫vol surface计算完成') except Exception as e: logger.error(e)
def update_all_instrument_vol_surface_to_terminal(observed_date): db_session = create_db_session() holidays = TradingCalendarRepo.get_holiday_list(db_session) special_dates = [] if not DateTimeUtils.is_trading_day(observed_date, holidays, special_dates): observed_date = DateTimeUtils.get_trading_day(observed_date, holidays, special_dates, 1, -1) # 登陆 logging.info('update all instrument vol surface to bct观察日为: %s' % DateTimeUtils.date2str(observed_date)) header = server_utils.login_terminal() # 从终端加载需要的标的物 terminal_instrument_ids = [ instrument['instrumentId'] for instrument in server_utils.get_instrument_list(header) ] logging.info( '从终端加载到的标的物为:%s, 长度为: %d' % (','.join(terminal_instrument_ids), len(terminal_instrument_ids))) # 获取数据库中标的物 vol_surface_list = VolSurfaceRepo.get_vol_surface_by_observed_date( db_session, observed_date) if not vol_surface_list: logging.info('observed_date为: %s没有可以导入到终端的vol surface' % observed_date) # 将instruments去重 unique_vol_surface_list = unique_instruments(vol_surface_list) logging.info('在trading_date为: %s时,vol surface 表获取到的标的数量为: %d' % (observed_date, len(unique_vol_surface_list))) # 写入vol surface 到 bct failed_instrument_ids = [] # hacky here, 由于bct接收的tenor是自然日,需要将vol surface中工作日转为自然日 for index, unique_vol_surface in enumerate(unique_vol_surface_list): # try: instrument_id = unique_vol_surface.instrumentId a = unique_vol_surface.modelInfo model_info = unique_vol_surface.modelInfo if model_info is None: continue # 获取到当前的vol surface的实时行情spot价格 market_data = get_real_time_market_data_by_instrument_id(instrument_id) if market_data is not None and market_data.get( 'lastPrice') is not None: now_price = market_data.get('lastPrice') else: logging.error('标的物%s当前行情为空' % instrument_id) continue if unique_vol_surface.instrumentId not in terminal_instrument_ids: logging.info('当前标的: %s的vol surface不在终端需要的列表里' % unique_vol_surface.instrumentId) continue logging.info('当前准备写入到bct的标的为: %s, 进度为: %d of %d' % (unique_vol_surface.instrumentId, index + 1, len(unique_vol_surface_list))) result = update_one_instrument_to_terminal(unique_vol_surface, header, now_price, observed_date) if not result: failed_instrument_ids.append(instrument_id) time.sleep(0.3) # except Exception as e: # logging.error('更新标的物%s波动率曲面失败, 异常:%s' % (instrument_id, str(e))) logging.error( '处理失败的标的物为:%d %s' % (len(failed_instrument_ids), ','.join(failed_instrument_ids))) db_session.close()