def update_implied_vol(start_date, end_date, force_update):
"""
根据日期循环导入数到数据平台otc_position_snapshot表
:param start_date:
:param end_date:
:param force_update:
:return:
"""
db_session = create_db_session()
try:
for index in range(0, (end_date - start_date).days + 1):
report_date = (start_date + timedelta(index))
logging.info('开始获取report_date: %s的数据' % report_date)
clean_position_dict = ROTCPositionService.clean_one_day_implied_vol(
db_session, report_date)
position_snapshots = ROTCPositionService.convert_data_to_position_snapshot(
clean_position_dict)
exiting_trans_codes = OTCPositionSnapshotRepo.get_trans_codes_by_report_date(
db_session, report_date)
ROTCPositionService.save_position_snapshot(
db_session, report_date, exiting_trans_codes,
position_snapshots, force_update)
except Exception as e:
logging.error('运行错误: %s' % e)
db_session.close()
raise Exception(e)
def recon_all_instrument(self, start_date, end_date, fail_fast=False, dry_run=True, active_only=True,
ignore_milestone=False):
logging.info("开始recon, %s %s" % (start_date.strftime('%Y%m%d'), end_date.strftime('%Y%m%d')))
# 获取所有需要Recon的标的物，只处理状态为Active的标的物
db_session = create_db_session()
# 获取标的物
all_instruments = self.load_instruments(db_session, active_only)
logging.info("当前需要Recon的标的物总数为: %d:" % len(all_instruments))
holidays = TradingCalendarRepo.get_holiday_list(db_session)
db_session.close()
# 并发进行recon
with ThreadPoolExecutor(max_workers=8) as executor:
# 开始并行向远程服务器请求数据
all_task = [executor.submit(self.recon_one_instrument, (item, start_date, end_date, holidays, dry_run,
ignore_milestone)) for item in all_instruments]
# 处理Recon结果
success_instruments = []
failed_instruments = []
for future in as_completed(all_task):
instrument, status = future.result()
if status is True:
success_instruments.append(instrument)
else:
failed_instruments.append(instrument)
logging.info("标的物处理结束: %s，处理结果为：%s" % (instrument.instrumentId, status))
logging.info("当前处理成功的标的物：%d of %d，失败的标的物：%d of %d" %
(len(success_instruments), len(all_instruments), len(failed_instruments),
len(all_instruments)))
if len(failed_instruments) != 0:
err_msg = "处理失败的标的物为：%s" % ','.join([instrument.instrumentId for instrument in failed_instruments])
logging.info(err_msg)
if fail_fast:
raise Exception(err_msg)
else:
logging.info("完成Recon成功")
def backfill_all_instruments(self, start_date, end_date, active_only=True, force_update=False):
db_session = create_db_session()
# 获取标的物
all_instruments = self.load_instruments(db_session, active_only)
logging.info("当前需要Backfill的标的物总数为: %d:" % len(all_instruments))
holidays = TradingCalendarRepo.get_holiday_list(db_session)
db_session.close()
# 并发进行 backfiller
with ThreadPoolExecutor(max_workers=8) as executor:
# 开始并行向远程服务器请求数据
all_task = [executor.submit(self.backfill_one_instrument, (item, start_date, end_date, holidays,
force_update)) for item in
all_instruments]
# 处理Backfill结果
success_instruments = []
failed_instruments = []
for future in as_completed(all_task):
instrument, status = future.result()
if status is True:
success_instruments.append(instrument)
else:
failed_instruments.append(instrument)
logging.info("标的物处理结束: %s，处理结果为：%s" % (instrument.instrumentId, status))
logging.info("当前处理成功的标的物：%d of %d，失败的标的物：%d of %d" %
(len(success_instruments), len(all_instruments), len(failed_instruments),
len(all_instruments)))
if len(failed_instruments) != 0:
raise Exception(
"处理失败的标的物为：%s" % ','.join([instrument.instrumentId for instrument in failed_instruments]))
def recon_one_instrument(self, params):
# 返回True或Fasle, 当前标的物是Clean则返回True
instrument, start_date, end_date, holidays, dry_run, ignore_milestone = params
db_session = create_db_session()
try:
logging.info("开始Recon标的物: %s" % instrument.instrumentId)
# 获取标的物Milestone
milestone = self.get_milestone(db_session, instrument)
# 只处理标的物上市日期和退市日期之间的数据
if instrument.listedDate is not None and start_date < instrument.listedDate:
start_date = instrument.listedDate
if instrument.delistedDate is not None and end_date > instrument.delistedDate:
end_date = instrument.delistedDate
# 计算需要Recon的日期，跳过节假日、周末和特殊日期
special_dates = TradingCalendarRepo.load_special_dates(db_session, instrument)
unchecked_trading_dates = self.get_unchecked_trading_dates(
start_date, end_date, milestone, holidays, special_dates, ignore_milestone)
# 获取标的物已经拥有的交易日
existing_trading_dates = self.get_existing_trading_dates(db_session, instrument)
# 计算出缺失数据
missing_trading_dates = self.get_missing_trading_dates(instrument, unchecked_trading_dates,
existing_trading_dates)
# 对缺数数据进行处理
if not dry_run:
self.upsert_missing_trading_dates(db_session, instrument, missing_trading_dates)
# TODO 需要对多余的数据进行处理
# 计算出多余数据
extra_trading_dates = []
logging.info("Recon找到多余数据：%s -> %d 个Break" % (instrument.instrumentId, len(extra_trading_dates)))
# 如果没有差值，则比较开始日期和Milestone，更新Milestone
has_missing_dates = len(missing_trading_dates) != 0
has_extra_dates = len(extra_trading_dates) != 0
if (not has_missing_dates) and (not has_extra_dates) and (not dry_run):
if start_date <= end_date:
self.upsert_milestone(db_session, instrument, start_date, end_date, milestone)
else:
logging.info("标的物开始日期大于结束日期，不更新Milestone：%s [%s %s]" % (
instrument.instrumentId, start_date.strftime('%Y%m%d'), end_date.strftime('%Y%m%d')))
return instrument, (not has_missing_dates) and (not has_extra_dates)
except Exception as e:
logging.error("Recon失败，标的物：%s, 异常：%s" % (instrument.instrumentId, e))
return instrument, False
finally:
db_session.close()
def backfill_one_instrument(self, params):
instrument, start_date, end_date, holidays, force_update = params
logging.info("开始Backfill标的物: %s, startDate: %s, endDate: %s" %
(instrument.instrumentId, start_date.strftime('%Y%m%d'),
end_date.strftime('%Y%m%d')))
db_session = create_db_session()
try:
special_dates = TradingCalendarRepo.load_special_dates(
db_session, instrument)
option_close_breaks = self.get_recon_breaks(
db_session, start_date, end_date, instrument)
ret = self.backfill_multiple_option_close(db_session, instrument,
option_close_breaks,
holidays, special_dates)
return instrument, ret
except Exception as e:
return instrument, False
finally:
db_session.close()
def update_eod_date(start_date_str=EODDateConfig.eod_start_date,
end_date_str=DateTimeUtils.date2str(date.today()),
eod_cutoff_time=EODDateConfig.eod_cutoff_time):
end_date = DateTimeUtils.str2date(end_date_str)
start_date = DateTimeUtils.str2date(start_date_str)
# 获取节假日信息
db_session = create_db_session()
# TODO: 需要从配置文件读取
if not TradingDayService.is_trading_day(db_session, end_date):
# 如果当天不是交易日则直接计算上一个交易日的值
end_date = TradingDayService.go_trading_day(db_session, end_date, 1, -1)
else:
# 如果是交易日，小于cutoff time还是返回上一个交易日
if datetime.now().time() < DateTimeUtils.str2time(eod_cutoff_time):
end_date = TradingDayService.go_trading_day(db_session, end_date, 1, -1)
else:
end_date = end_date
# 设置变量到airflow variable
AirflowVariableUtils.set_eod_start_date(start_date)
AirflowVariableUtils.set_eod_end_date(end_date)
def calc_all_instrument_quote_vol_surface(observed_date):
"""
从otc_option_quote表里取华泰的数据,格式化写入到vol_surface表中
:return:
"""
db_session = create_db_session()
try:
# 取所有instrument_id
instrument_ids = OTCOptionQuoteRepo.get_instrument_ids_by_observe_date(
db_session, observed_date)
vol_surface_dbo_dict = {}
for index, instrument_id in enumerate(instrument_ids):
# 计算一个标的的vol_surface
logger.info('准备加载的标的为: %s, 进度为: %d of %d' %
(instrument_id, index + 1, len(instrument_ids)))
vol_surface_dbo = VolSurfaceSpiderService.calc_one_instrument_quote_vol_surface(
db_session, instrument_id, observed_date)
if vol_surface_dbo is None:
continue
vol_surface_dbo_dict[instrument_id] = vol_surface_dbo
logger.info('成功计算了标的: %s的vol_surface, 进度为: %d of %d' %
(instrument_id, index + 1, len(instrument_ids)))
# 获取所有的合约种类及其对应的主力合约
holidays = TradingCalendarRepo.get_holiday_list(db_session)
special_dates = []
# 取上一个交易日
last_trade_date = DateTimeUtils.get_trading_day(
observed_date, holidays, special_dates, 1, -1)
# 获取所有的主力合约
contract_list = db_session.query(FutureContractInfo).filter(
FutureContractInfo.tradeDate == last_trade_date).all()
# 获取当天所有活跃的大宗期货合约
active_instrument_list = db_session.query(Instrument).filter(
and_(
Instrument.status ==
enum_dto.InstrumentStatusType.ACTIVE.name,
Instrument.assetClass ==
enum_dto.AssetClassType.COMMODITY.name,
Instrument.instrumentType ==
enum_dto.InstrumentType.FUTURE.name,
or_(Instrument.delistedDate is None,
Instrument.delistedDate > observed_date))).all()
active_instrument_dict = {}
for instrument in active_instrument_list:
if instrument.contractType not in active_instrument_dict:
active_instrument_dict[instrument.contractType] = []
active_instrument_dict[instrument.contractType].append(
instrument.instrumentId)
# 所有标的都使用主力合约对应的波动率曲面
vol_surface_dbos = []
for contract in contract_list:
primary_contract_id = contract.primaryContractId
if primary_contract_id not in vol_surface_dbo_dict:
logger.warning('标的没有主力合约对应的波动率曲面：%s' % primary_contract_id)
continue
variety_type = contract.contractType
if variety_type not in active_instrument_dict:
logger.warning('主力合约没有活跃的合约：%s' % variety_type)
continue
active_future_list = active_instrument_dict[variety_type]
for instrument_id in active_future_list:
vol_surface_dbo = copy.deepcopy(
vol_surface_dbo_dict[primary_contract_id])
vol_surface_dbo.uuid = uuid.uuid1()
vol_surface_dbo.instrumentId = instrument_id
vol_surface_dbos.append(vol_surface_dbo)
# 将波动率曲面写入数据库
VolSurfaceRepo.insert_vol_surfaces(db_session, vol_surface_dbos)
logger.info('爬虫vol surface计算完成')
except Exception as e:
logger.error(e)
def update_all_instrument_vol_surface_to_terminal(observed_date):
db_session = create_db_session()
holidays = TradingCalendarRepo.get_holiday_list(db_session)
special_dates = []
if not DateTimeUtils.is_trading_day(observed_date, holidays,
special_dates):
observed_date = DateTimeUtils.get_trading_day(observed_date, holidays,
special_dates, 1, -1)
# 登陆
logging.info('update all instrument vol surface to bct观察日为: %s' %
DateTimeUtils.date2str(observed_date))
header = server_utils.login_terminal()
# 从终端加载需要的标的物
terminal_instrument_ids = [
instrument['instrumentId']
for instrument in server_utils.get_instrument_list(header)
]
logging.info(
'从终端加载到的标的物为：%s, 长度为: %d' %
(','.join(terminal_instrument_ids), len(terminal_instrument_ids)))
# 获取数据库中标的物
vol_surface_list = VolSurfaceRepo.get_vol_surface_by_observed_date(
db_session, observed_date)
if not vol_surface_list:
logging.info('observed_date为: %s没有可以导入到终端的vol surface' % observed_date)
# 将instruments去重
unique_vol_surface_list = unique_instruments(vol_surface_list)
logging.info('在trading_date为: %s时,vol surface 表获取到的标的数量为: %d' %
(observed_date, len(unique_vol_surface_list)))
# 写入vol surface 到 bct
failed_instrument_ids = []
# hacky here, 由于bct接收的tenor是自然日，需要将vol surface中工作日转为自然日
for index, unique_vol_surface in enumerate(unique_vol_surface_list):
# try:
instrument_id = unique_vol_surface.instrumentId
a = unique_vol_surface.modelInfo
model_info = unique_vol_surface.modelInfo
if model_info is None:
continue
# 获取到当前的vol surface的实时行情spot价格
market_data = get_real_time_market_data_by_instrument_id(instrument_id)
if market_data is not None and market_data.get(
'lastPrice') is not None:
now_price = market_data.get('lastPrice')
else:
logging.error('标的物%s当前行情为空' % instrument_id)
continue
if unique_vol_surface.instrumentId not in terminal_instrument_ids:
logging.info('当前标的: %s的vol surface不在终端需要的列表里' %
unique_vol_surface.instrumentId)
continue
logging.info('当前准备写入到bct的标的为: %s, 进度为: %d of %d' %
(unique_vol_surface.instrumentId, index + 1,
len(unique_vol_surface_list)))
result = update_one_instrument_to_terminal(unique_vol_surface, header,
now_price, observed_date)
if not result:
failed_instrument_ids.append(instrument_id)
time.sleep(0.3)
# except Exception as e:
# logging.error('更新标的物%s波动率曲面失败， 异常:%s' % (instrument_id, str(e)))
logging.error(
'处理失败的标的物为：%d %s' %
(len(failed_instrument_ids), ','.join(failed_instrument_ids)))
db_session.close()